You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/09/25 23:38:00 UTC
[01/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903,
removal of org.apache.slider package. Contributed by Jian He [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/yarn-native-services 3ae5a4721 -> 3f7a50d8d (forced update)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
deleted file mode 100644
index 5493198..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.utils;
-
-import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
-import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
-import org.apache.commons.compress.utils.IOUtils;
-import org.junit.Assert;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-/**
- * Various utility methods
- * Byte comparison methods are from
- * <code>org.apache.hadoop.fs.contract.ContractTestUtils</code>
- */
-public class TestUtility {
- protected static final Logger log =
- LoggerFactory.getLogger(TestUtility.class);
-
- public static void addDir(File dirObj, ZipArchiveOutputStream zipFile, String prefix) throws IOException {
- for (File file : dirObj.listFiles()) {
- if (file.isDirectory()) {
- addDir(file, zipFile, prefix + file.getName() + File.separator);
- } else {
- log.info("Adding to zip - " + prefix + file.getName());
- zipFile.putArchiveEntry(new ZipArchiveEntry(prefix + file.getName()));
- IOUtils.copy(new FileInputStream(file), zipFile);
- zipFile.closeArchiveEntry();
- }
- }
- }
-
- public static void zipDir(String zipFile, String dir) throws IOException {
- File dirObj = new File(dir);
- ZipArchiveOutputStream out = new ZipArchiveOutputStream(new FileOutputStream(zipFile));
- log.info("Creating : {}", zipFile);
- try {
- addDir(dirObj, out, "");
- } finally {
- out.close();
- }
- }
-
- public static String createAppPackage(
- TemporaryFolder folder, String subDir, String pkgName, String srcPath) throws IOException {
- String zipFileName;
- File pkgPath = folder.newFolder(subDir);
- File zipFile = new File(pkgPath, pkgName).getAbsoluteFile();
- zipFileName = zipFile.getAbsolutePath();
- TestUtility.zipDir(zipFileName, srcPath);
- log.info("Created temporary zip file at {}", zipFileName);
- return zipFileName;
- }
-
-
- /**
- * Assert that tthe array original[0..len] and received[] are equal.
- * A failure triggers the logging of the bytes near where the first
- * difference surfaces.
- * @param original source data
- * @param received actual
- * @param len length of bytes to compare
- */
- public static void compareByteArrays(byte[] original,
- byte[] received,
- int len) {
- Assert.assertEquals("Number of bytes read != number written",
- len, received.length);
- int errors = 0;
- int first_error_byte = -1;
- for (int i = 0; i < len; i++) {
- if (original[i] != received[i]) {
- if (errors == 0) {
- first_error_byte = i;
- }
- errors++;
- }
- }
-
- if (errors > 0) {
- String message = String.format(" %d errors in file of length %d",
- errors, len);
- log.warn(message);
- // the range either side of the first error to print
- // this is a purely arbitrary number, to aid user debugging
- final int overlap = 10;
- for (int i = Math.max(0, first_error_byte - overlap);
- i < Math.min(first_error_byte + overlap, len);
- i++) {
- byte actual = received[i];
- byte expected = original[i];
- String letter = toChar(actual);
- String line = String.format("[%04d] %2x %s\n", i, actual, letter);
- if (expected != actual) {
- line = String.format("[%04d] %2x %s -expected %2x %s\n",
- i,
- actual,
- letter,
- expected,
- toChar(expected));
- }
- log.warn(line);
- }
- Assert.fail(message);
- }
- }
- /**
- * Convert a byte to a character for printing. If the
- * byte value is < 32 -and hence unprintable- the byte is
- * returned as a two digit hex value
- * @param b byte
- * @return the printable character string
- */
- public static String toChar(byte b) {
- if (b >= 0x20) {
- return Character.toString((char) b);
- } else {
- return String.format("%02x", b);
- }
- }
-
- /**
- * Convert a buffer to a string, character by character
- * @param buffer input bytes
- * @return a string conversion
- */
- public static String toChar(byte[] buffer) {
- StringBuilder builder = new StringBuilder(buffer.length);
- for (byte b : buffer) {
- builder.append(toChar(b));
- }
- return builder.toString();
- }
-
- public static byte[] toAsciiByteArray(String s) {
- char[] chars = s.toCharArray();
- int len = chars.length;
- byte[] buffer = new byte[len];
- for (int i = 0; i < len; i++) {
- buffer[i] = (byte) (chars[i] & 0xff);
- }
- return buffer;
- }
-
- /**
- * Create a dataset for use in the tests; all data is in the range
- * base to (base+modulo-1) inclusive
- * @param len length of data
- * @param base base of the data
- * @param modulo the modulo
- * @return the newly generated dataset
- */
- public static byte[] dataset(int len, int base, int modulo) {
- byte[] dataset = new byte[len];
- for (int i = 0; i < len; i++) {
- dataset[i] = (byte) (base + (i % modulo));
- }
- return dataset;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java
deleted file mode 100644
index 6cda9c1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java
+++ /dev/null
@@ -1,873 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.utils;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.service.ServiceOperations;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
-import org.apache.slider.client.SliderClient;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.common.params.ActionFreezeArgs;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.common.tools.Duration;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.main.ServiceLauncher;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import static org.apache.slider.utils.KeysForTests.*;
-
-/**
- * Base class for mini cluster tests -creates a field for the
- * mini yarn cluster.
- */
-public abstract class YarnMiniClusterTestBase extends SliderTestBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(YarnMiniClusterTestBase.class);
-
- /**
- * Mini YARN cluster only.
- */
- public static final int CLUSTER_GO_LIVE_TIME = 3 * 60 * 1000;
- public static final int CLUSTER_STOP_TIME = 1 * 60 * 1000;
- public static final int SIGTERM = -15;
- public static final int SIGKILL = -9;
- public static final int SIGSTOP = -17;
- public static final String NO_ARCHIVE_DEFINED = "Archive configuration " +
- "option not set: ";
- /**
- * RAM for the YARN containers: {@value}.
- */
- public static final String YRAM = "256";
- public static final String FIFO_SCHEDULER = "org.apache.hadoop.yarn.server" +
- ".resourcemanager.scheduler.fifo.FifoScheduler";
- public static final YarnConfiguration SLIDER_CONFIG =
- SliderUtils.createConfiguration();
- private static boolean killSupported;
-
- static {
- SLIDER_CONFIG.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, 1);
- SLIDER_CONFIG.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 100);
- SLIDER_CONFIG.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
- SLIDER_CONFIG.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
- SLIDER_CONFIG
- .setBoolean(SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED,
- true);
- SLIDER_CONFIG
- .setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
- }
-
-
- private int thawWaitTime = DEFAULT_THAW_WAIT_TIME_SECONDS * 1000;
- private int freezeWaitTime = DEFAULT_TEST_FREEZE_WAIT_TIME_SECONDS * 1000;
- private int sliderTestTimeout = DEFAULT_TEST_TIMEOUT_SECONDS * 1000;
- private boolean teardownKillall = DEFAULT_TEARDOWN_KILLALL;
-
- /**
- * This is set in a system property.
- */
- @Rule
- public Timeout testTimeout = new Timeout(
- getTimeOptionMillis(getTestConfiguration(),
- KEY_TEST_TIMEOUT,
- DEFAULT_TEST_TIMEOUT_SECONDS * 1000)
- );
- private MiniDFSCluster hdfsCluster;
- private MiniYARNCluster miniCluster;
- private boolean switchToImageDeploy = false;
- private boolean imageIsRemote = false;
- private URI remoteImageURI;
- private List<SliderClient> clustersToTeardown = new ArrayList<>();
- private int clusterCount = 1;
-
- /**
- * Clent side test: validate system env before launch.
- */
- @BeforeClass
- public static void checkClientEnv() throws IOException, SliderException {
- SliderUtils.validateSliderClientEnvironment(null);
- }
-
- /**
- * Work out if kill is supported.
- */
- @BeforeClass
- public static void checkKillSupport() {
- killSupported = !Shell.WINDOWS;
- }
-
- protected static boolean getKillSupported() {
- return killSupported;
- }
-
- protected MiniYARNCluster getMiniCluster() {
- return miniCluster;
- }
-
- /**
- * Probe for the disks being healthy in a mini cluster. Only the first
- * NM is checked.
- *
- * @param miniCluster
- */
- public static void assertMiniClusterDisksHealthy(
- MiniYARNCluster miniCluster) {
- boolean healthy = miniCluster.getNodeManager(
- 0).getNodeHealthChecker().getDiskHandler().areDisksHealthy();
- assertTrue("Disks on test cluster unhealthy —may be full", healthy);
- }
-
- /**
- * Inner work building the mini dfs cluster.
- *
- * @param name
- * @param conf
- * @return
- */
- public static MiniDFSCluster buildMiniHDFSCluster(
- String name,
- YarnConfiguration conf) throws IOException {
- assertNativeLibrariesPresent();
-
- File baseDir = new File("./target/hdfs", name).getAbsoluteFile();
- //use file: to rm it recursively
- FileUtil.fullyDelete(baseDir);
- conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
- MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
-
- MiniDFSCluster cluster = builder.build();
- return cluster;
- }
-
- public static String buildFsDefaultName(MiniDFSCluster miniDFSCluster) {
- if (miniDFSCluster != null) {
- return String.format("hdfs://localhost:%d/",
- miniDFSCluster.getNameNodePort());
- } else {
- return "file:///";
- }
- }
-
- /**
- * Assert that an operation failed because a cluster is in use.
- *
- * @param e exception
- */
- public static void assertFailureClusterInUse(SliderException e) {
- assertExceptionDetails(e,
- SliderExitCodes.EXIT_APPLICATION_IN_USE,
- ErrorStrings.E_CLUSTER_RUNNING);
- }
-
- protected String buildClustername(String clustername) {
- if (SliderUtils.isSet(clustername)) {
- return clustername;
- } else {
- return createClusterName();
- }
- }
-
- /**
- * Create the cluster name from the method name and an auto-incrementing
- * counter.
- *
- * @return a cluster name
- */
- protected String createClusterName() {
- String base = methodName.getMethodName().toLowerCase(Locale.ENGLISH);
- if (clusterCount++ > 1) {
- return String.format("%s-%d", base, clusterCount);
- }
- return base;
- }
-
- @Override
- public void setup() throws Exception {
- super.setup();
- Configuration testConf = getTestConfiguration();
- thawWaitTime = getTimeOptionMillis(testConf,
- KEY_TEST_THAW_WAIT_TIME,
- thawWaitTime);
- freezeWaitTime = getTimeOptionMillis(testConf,
- KEY_TEST_FREEZE_WAIT_TIME,
- freezeWaitTime);
- sliderTestTimeout = getTimeOptionMillis(testConf,
- KEY_TEST_TIMEOUT,
- sliderTestTimeout);
- teardownKillall =
- testConf.getBoolean(KEY_TEST_TEARDOWN_KILLALL,
- teardownKillall);
-
- }
-
- @After
- public void teardown() {
- describe("teardown");
- stopRunningClusters();
- stopMiniCluster();
- }
-
- protected void addToTeardown(SliderClient client) {
- clustersToTeardown.add(client);
- }
-
- protected void addToTeardown(ServiceLauncher<SliderClient> launcher) {
- if (launcher != null) {
- SliderClient sliderClient = launcher.getService();
- if (sliderClient != null) {
- addToTeardown(sliderClient);
- }
- }
- }
-
- /**
- * Kill any java process with the given grep pattern.
- *
- * @param grepString string to grep for
- */
- public int killJavaProcesses(String grepString, int signal)
- throws IOException, InterruptedException {
-
- String[] commandString;
- if (!Shell.WINDOWS) {
- String killCommand = String.format(
- "jps -l| grep %s | awk '{print $1}' | xargs kill %d", grepString,
- signal);
- LOG.info("Command command = {}", killCommand);
-
- commandString = new String[]{"bash", "-c", killCommand};
- } else {
- // windows
- if (!killSupported) {
- return -1;
- }
-
- // "jps -l | grep "String" | awk "{print $1}" | xargs -n 1 taskkill /PID"
- String killCommand = String.format(
- "jps -l | grep %s | gawk '{print $1}' | xargs -n 1 taskkill /f " +
- "/PID", grepString);
- commandString = new String[]{"CMD", "/C", killCommand};
- }
-
- Process command = new ProcessBuilder(commandString).start();
- int exitCode = command.waitFor();
-
- logStdOutStdErr(command);
- return exitCode;
- }
-
- /**
- * Kill all processes which match one of the list of grepstrings.
- *
- * @param greps
- * @param signal
- */
- public void killJavaProcesses(List<String> greps, int signal)
- throws IOException, InterruptedException {
- for (String grep : greps) {
- killJavaProcesses(grep, signal);
- }
- }
-
- protected YarnConfiguration getConfiguration() {
- return SLIDER_CONFIG;
- }
-
- /**
- * Stop any running cluster that has been added.
- */
- public void stopRunningClusters() {
- for (SliderClient client : clustersToTeardown) {
- maybeStopCluster(client, client.getDeployedClusterName(),
- "Teardown at end of test case", true);
- }
- }
-
- public void stopMiniCluster() {
- Log commonslog = LogFactory.getLog(YarnMiniClusterTestBase.class);
- ServiceOperations.stopQuietly(commonslog, miniCluster);
- if (hdfsCluster != null) {
- hdfsCluster.shutdown();
- }
- }
-
- /**
- * Create and start a minicluster.
- *
- * @param name cluster/test name; if empty one is created from
- * the junit method
- * @param conf configuration to use
- * @param noOfNodeManagers #of NMs
- * @param numLocalDirs #of local dirs
- * @param numLogDirs #of log dirs
- * @param startHDFS create an HDFS mini cluster
- * @return the name of the cluster
- */
- protected String createMiniCluster(String name,
- YarnConfiguration conf,
- int noOfNodeManagers,
- int numLocalDirs,
- int numLogDirs,
- boolean startHDFS) throws IOException {
- assertNativeLibrariesPresent();
- conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
- conf.set(YarnConfiguration.RM_SCHEDULER, FIFO_SCHEDULER);
- patchDiskCapacityLimits(conf);
- SliderUtils.patchConfiguration(conf);
- name = buildClustername(name);
- miniCluster = new MiniYARNCluster(
- name,
- noOfNodeManagers,
- numLocalDirs,
- numLogDirs);
- miniCluster.init(conf);
- miniCluster.start();
- // health check
- assertMiniClusterDisksHealthy(miniCluster);
- if (startHDFS) {
- createMiniHDFSCluster(name, conf);
- }
- return name;
- }
-
- public void patchDiskCapacityLimits(YarnConfiguration conf) {
- conf.setFloat(
- YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE,
- 99.0f);
- conf.setInt(SliderXmlConfKeys.DFS_NAMENODE_DU_RESERVED_KEY,
- 2 * 1024 * 1024);
- conf.setBoolean("yarn.nodemanager.disk-health-checker.enable", false);
- }
-
- /**
- * Create a mini HDFS cluster and save it to the hdfsClusterField.
- *
- * @param name
- * @param conf
- */
- public void createMiniHDFSCluster(String name, YarnConfiguration conf)
- throws IOException {
- hdfsCluster = buildMiniHDFSCluster(name, conf);
- }
-
- /**
- * Launch the client with the specific args against the MiniMR cluster
- * launcher i.e. expected to have successfully completed.
- *
- * @param conf configuration
- * @param args arg list
- * @return the return code
- */
- protected ServiceLauncher<SliderClient> launchClientAgainstMiniMR(
- Configuration conf,
- List args)
- throws Throwable {
- ServiceLauncher<SliderClient> launcher =
- launchClientNoExitCodeCheck(conf, args);
- int exited = launcher.getServiceExitCode();
- if (exited != 0) {
- throw new SliderException(exited, "Launch failed with exit code " +
- exited);
- }
- return launcher;
- }
-
- /**
- * Launch the client with the specific args against the MiniMR cluster
- * without any checks for exit codes.
- *
- * @param conf configuration
- * @param args arg list
- * @return the return code
- */
- public ServiceLauncher<SliderClient> launchClientNoExitCodeCheck(
- Configuration conf,
- List args) throws Throwable {
- assertNotNull(miniCluster);
- return launchClientAgainstRM(getRMAddr(), args, conf);
- }
-
- /**
- * Kill all Slider Services.
- *
- * @param signal
- */
- public int killAM(int signal) throws IOException, InterruptedException {
- return killJavaProcesses(SliderAppMaster.SERVICE_CLASSNAME_SHORT, signal);
- }
-
- public void logStdOutStdErr(Process p) throws IOException {
- try (BufferedReader br = new BufferedReader(new InputStreamReader(p
- .getInputStream()))) {
- String line = br.readLine();
- while (line != null) {
- LOG.info(line);
- line = br.readLine();
- }
- }
- try (BufferedReader br = new BufferedReader(new InputStreamReader(p
- .getErrorStream()))) {
- String line = br.readLine();
- while (line != null) {
- LOG.error(line);
- line = br.readLine();
- }
- }
- }
-
- /**
- * List any java process.
- */
- public void lsJavaProcesses() throws InterruptedException, IOException {
- Process bash = new ProcessBuilder("jps", "-v").start();
- bash.waitFor();
- logStdOutStdErr(bash);
- }
-
- public YarnConfiguration getTestConfiguration() {
- YarnConfiguration conf = getConfiguration();
- conf.addResource(SLIDER_TEST_XML);
- return conf;
- }
-
- protected String getRMAddr() {
- assertNotNull(miniCluster);
- String addr = miniCluster.getConfig().get(YarnConfiguration.RM_ADDRESS);
- assertNotNull(addr != null);
- assertNotEquals("", addr);
- return addr;
- }
-
- /**
- * Return the default filesystem, which is HDFS if the miniDFS cluster is
- * up, file:// if not.
- *
- * @return a filesystem string to pass down
- */
- protected String getFsDefaultName() {
- return buildFsDefaultName(hdfsCluster);
- }
-
- /**
- * Create or build a cluster (the action is set by the first verb).
- * @param action operation to invoke: SliderActions.ACTION_CREATE or
- * SliderActions.ACTION_BUILD
- * @param clustername cluster name
- * @param extraArgs list of extra args to add to the creation command
- * @param deleteExistingData should the data of any existing cluster
- * of this name be deleted
- * @param blockUntilRunning block until the AM is running
- * @return launcher which will have executed the command.
- */
- public ServiceLauncher<SliderClient> createOrBuildCluster(String action,
- String clustername, List<String> extraArgs, boolean deleteExistingData,
- boolean blockUntilRunning) throws Throwable {
- assertNotNull(clustername);
- assertNotNull(miniCluster);
- // update action should keep existing data
- Configuration config = miniCluster.getConfig();
- if (deleteExistingData && !SliderActions.ACTION_UPDATE.equals(action)) {
- FileSystem dfs = FileSystem.get(new URI(getFsDefaultName()), config);
-
- SliderFileSystem sliderFileSystem = new SliderFileSystem(dfs, config);
- Path clusterDir = sliderFileSystem.buildClusterDirPath(clustername);
- LOG.info("deleting instance data at {}", clusterDir);
- //this is a safety check to stop us doing something stupid like deleting /
- assertTrue(clusterDir.toString().contains("/.slider/"));
- rigorousDelete(sliderFileSystem, clusterDir, 60000);
- }
-
-
- List<String> argsList = new ArrayList<>();
- argsList.addAll(Arrays.asList(
- action, clustername,
- Arguments.ARG_MANAGER, getRMAddr(),
- Arguments.ARG_FILESYSTEM, getFsDefaultName(),
- Arguments.ARG_DEBUG));
-
- argsList.addAll(getExtraCLIArgs());
-
- if (extraArgs != null) {
- argsList.addAll(extraArgs);
- }
- ServiceLauncher<SliderClient> launcher = launchClientAgainstMiniMR(
- //config includes RM binding info
- new YarnConfiguration(config),
- //varargs list of command line params
- argsList
- );
- assertEquals(0, launcher.getServiceExitCode());
- SliderClient client = launcher.getService();
- if (blockUntilRunning) {
- client.monitorAppToRunning(new Duration(CLUSTER_GO_LIVE_TIME));
- }
- return launcher;
- }
-
- /**
- * Delete with some pauses and backoff; designed to handle slow delete
- * operation in windows.
- */
- public void rigorousDelete(
- SliderFileSystem sliderFileSystem,
- Path path, long timeout) throws IOException, SliderException {
-
- if (path.toUri().getScheme() == "file") {
- File dir = new File(path.toUri().getPath());
- rigorousDelete(dir, timeout);
- } else {
- Duration duration = new Duration(timeout);
- duration.start();
- FileSystem dfs = sliderFileSystem.getFileSystem();
- boolean deleted = false;
- while (!deleted && !duration.getLimitExceeded()) {
- dfs.delete(path, true);
- deleted = !dfs.exists(path);
- if (!deleted) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- LOG.info("ignoring interrupted sleep");
- }
- }
- }
- }
- sliderFileSystem.verifyDirectoryNonexistent(path);
- }
-
- /**
- * Delete with some pauses and backoff; designed to handle slow delete
- * operation in windows.
- *
- * @param dir dir to delete
- * @param timeout timeout in millis
- */
- public void rigorousDelete(File dir, long timeout) throws IOException {
- Duration duration = new Duration(timeout);
- duration.start();
- boolean deleted = false;
- while (!deleted && !duration.getLimitExceeded()) {
- FileUtils.deleteQuietly(dir);
- deleted = !dir.exists();
- if (!deleted) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- LOG.info("ignoring interrupted sleep");
- }
- }
- }
- if (!deleted) {
- // noisy delete raises an IOE
- FileUtils.deleteDirectory(dir);
- }
- }
-
- /**
- * Add arguments to launch Slider with.
- * <p>
- * Extra arguments are added after standard arguments and before roles.
- *
- * @return additional arguments to launch Slider with
- */
- protected List<String> getExtraCLIArgs() {
- return new ArrayList<>();
- }
-
- public String getConfDir() throws FileNotFoundException {
- return getResourceConfDirURI();
- }
-
- /**
- * Get the key for the application.
- *
- * @return
- */
- public String getApplicationHomeKey() {
- failNotImplemented();
- return null;
- }
-
- /**
- * Get the archive path -which defaults to the local one.
- *
- * @return
- */
- public String getArchivePath() {
- return getLocalArchive();
- }
-
- /**
- * Get the local archive -the one defined in the test configuration.
- *
- * @return a possibly null/empty string
- */
- public final String getLocalArchive() {
- return getTestConfiguration().getTrimmed(getArchiveKey());
- }
-
- /**
- * Get the key for archives in tests.
- *
- * @return
- */
- public String getArchiveKey() {
- failNotImplemented();
- return null;
- }
-
- /**
- * Merge a k-v pair into a simple k=v string; simple utility.
- *
- * @param key key
- * @param val value
- * @return the string to use after a -D option
- */
- public String define(String key, String val) {
- return String.format("%s=%s", key, val);
- }
-
- public void assumeTestEnabled(boolean flag) {
- assume(flag, "test disabled");
- }
-
- public void assumeArchiveDefined() {
- String archive = getArchivePath();
- boolean defined = archive != null && archive != "";
- if (!defined) {
- LOG.warn(NO_ARCHIVE_DEFINED + getArchiveKey());
- }
- assume(defined, NO_ARCHIVE_DEFINED + getArchiveKey());
- }
-
- /**
- * Assume that application home is defined. This does not check that the
- * path is valid -that is expected to be a failure on tests that require
- * application home to be set.
- */
- public void assumeApplicationHome() {
- String applicationHome = getApplicationHome();
- assume(applicationHome != null && applicationHome != "",
- "Application home dir option not set " + getApplicationHomeKey());
- }
-
- public String getApplicationHome() {
- return getTestConfiguration().getTrimmed(getApplicationHomeKey());
- }
-
- /**
- * Get the resource configuration dir in the source tree.
- *
- * @return
- */
- public File getResourceConfDir() throws FileNotFoundException {
- File f = new File(getTestConfigurationPath()).getAbsoluteFile();
- if (!f.exists()) {
- throw new FileNotFoundException(
- "Resource configuration directory " + f + " not found");
- }
- return f;
- }
-
- public String getTestConfigurationPath() {
- failNotImplemented();
- return null;
- }
-
- /**
- * Get a URI string to the resource conf dir that is suitable for passing down
- * to the AM -and works even when the default FS is hdfs.
- */
- public String getResourceConfDirURI() throws FileNotFoundException {
- return getResourceConfDir().getAbsoluteFile().toURI().toString();
- }
-
- /**
- * Log an application report.
- *
- * @param report
- */
- public void logReport(ApplicationReport report) {
- LOG.info(SliderUtils.reportToString(report));
- }
-
- /**
- * Stop the cluster via the stop action -and wait for
- * {@link #CLUSTER_STOP_TIME} for the cluster to stop. If it doesn't
- *
- * @param sliderClient client
- * @param clustername cluster
- * @return the exit code
- */
- public int clusterActionFreeze(SliderClient sliderClient, String clustername,
- String message, boolean force)
- throws IOException, YarnException {
- LOG.info("Stopping cluster {}: {}", clustername, message);
- ActionFreezeArgs freezeArgs = new ActionFreezeArgs();
- freezeArgs.setWaittime(CLUSTER_STOP_TIME);
- freezeArgs.message = message;
- freezeArgs.force = force;
- int exitCode = sliderClient.actionStop(clustername,
- freezeArgs);
- if (exitCode != 0) {
- LOG.warn("Cluster stop failed with error code {}", exitCode);
- }
- return exitCode;
- }
-
- /**
- * Teardown-time cluster termination; will stop the cluster iff the client
- * is not null.
- *
- * @param sliderClient client
- * @param clustername name of cluster to teardown
- * @return
- */
- public int maybeStopCluster(
- SliderClient sliderClient,
- String clustername,
- String message,
- boolean force) {
- if (sliderClient != null) {
- if (SliderUtils.isUnset(clustername)) {
- clustername = sliderClient.getDeployedClusterName();
- }
- //only stop a cluster that exists
- if (SliderUtils.isSet(clustername)) {
- try {
- clusterActionFreeze(sliderClient, clustername, message, force);
- } catch (Exception e) {
- LOG.warn("While stopping cluster " + e, e);
- }
- try {
- sliderClient.actionDestroy(clustername);
- } catch (Exception e) {
- LOG.warn("While destroying cluster " + e, e);
- }
- }
- }
- return 0;
- }
-
- public String roleMapToString(Map<String, Integer> roles) {
- StringBuilder builder = new StringBuilder();
- for (Entry<String, Integer> entry : roles.entrySet()) {
- builder.append(entry.getKey());
- builder.append("->");
- builder.append(entry.getValue());
- builder.append(" ");
- }
- return builder.toString();
- }
-
- /**
- * Turn on test runs against a copy of the archive that is
- * uploaded to HDFS -this method copies up the
- * archive then switches the tests into archive mode.
- */
- public void enableTestRunAgainstUploadedArchive() throws IOException {
- Path remotePath = copyLocalArchiveToHDFS(getLocalArchive());
- // image mode
- switchToRemoteImageDeploy(remotePath);
- }
-
- /**
- * Switch to deploying a remote image.
- *
- * @param remotePath the remote path to use
- */
- public void switchToRemoteImageDeploy(Path remotePath) {
- switchToImageDeploy = true;
- imageIsRemote = true;
- remoteImageURI = remotePath.toUri();
- }
-
- /**
- * Copy a local archive to HDFS.
- *
- * @param localArchive local archive
- * @return the path of the uploaded image
- */
- public Path copyLocalArchiveToHDFS(String localArchive) throws IOException {
- assertNotNull(localArchive);
- File localArchiveFile = new File(localArchive);
- assertTrue(localArchiveFile.exists());
- assertNotNull(hdfsCluster);
- Path remoteUnresolvedArchive = new Path(localArchiveFile.getName());
- assertTrue(FileUtil.copy(
- localArchiveFile,
- hdfsCluster.getFileSystem(),
- remoteUnresolvedArchive,
- false,
- getTestConfiguration()));
- Path remotePath = hdfsCluster.getFileSystem().resolvePath(
- remoteUnresolvedArchive);
- return remotePath;
- }
-
- /**
- * Create a SliderFileSystem instance bonded to the running FS.
- * The YARN cluster must be up and running already
- *
- * @return
- */
- public SliderFileSystem createSliderFileSystem()
- throws URISyntaxException, IOException {
- FileSystem dfs =
- FileSystem.get(new URI(getFsDefaultName()), getConfiguration());
- SliderFileSystem hfs = new SliderFileSystem(dfs, getConfiguration());
- return hfs;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java
deleted file mode 100644
index cf9e616..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.utils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.core.zk.BlockingZKWatcher;
-import org.apache.slider.core.zk.ZKIntegration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.apache.slider.utils.KeysForTests.USERNAME;
-
-/**
- * Base class for mini cluster tests that use Zookeeper.
- */
-public abstract class YarnZKMiniClusterTestBase extends
- YarnMiniClusterTestBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(YarnZKMiniClusterTestBase.class);
-
- private MicroZKCluster microZKCluster;
-
- public void stopMiniCluster() {
- super.stopMiniCluster();
- IOUtils.closeStream(microZKCluster);
- }
-
- public ZKIntegration createZKIntegrationInstance(String zkQuorum,
- String clusterName,
- boolean createClusterPath,
- boolean canBeReadOnly,
- int timeout) throws IOException, InterruptedException {
- int sessionTimeout = ZKIntegration.SESSION_TIMEOUT;
-
- BlockingZKWatcher watcher = new BlockingZKWatcher();
- ZKIntegration zki = ZKIntegration.newInstance(zkQuorum,
- USERNAME,
- clusterName,
- createClusterPath,
- canBeReadOnly,
- watcher,
- sessionTimeout);
- boolean fromCache = zki.init();
- //here the callback may or may not have occurred.
- //optionally wait for it
- if (timeout > 0 && !fromCache) {
- watcher.waitForZKConnection(timeout);
- }
- //if we get here, the binding worked
- LOG.info("Connected: {}", zki);
- return zki;
- }
-
- /**
- * Wait for a flag to go true.
- * @param connectedFlag
- */
- public void waitForZKConnection(AtomicBoolean connectedFlag, int timeout)
- throws InterruptedException {
- synchronized (connectedFlag) {
- if (!connectedFlag.get()) {
- LOG.info("waiting for ZK event");
- //wait a bit
- connectedFlag.wait(timeout);
- }
- }
- assertTrue(connectedFlag.get());
- }
-
- /**
- * Create and start a minicluster with ZK.
- * @param name cluster/test name
- * @param conf configuration to use
- * @param noOfNodeManagers #of NMs
- * @param numLocalDirs #of local dirs
- * @param numLogDirs #of log dirs
- * @param startZK create a ZK micro cluster *THIS IS IGNORED*
- * @param startHDFS create an HDFS mini cluster
- */
- protected String createMiniCluster(String name,
- YarnConfiguration conf,
- int noOfNodeManagers,
- int numLocalDirs,
- int numLogDirs,
- boolean startZK,
- boolean startHDFS) throws IOException {
- name = buildClustername(name);
- createMicroZKCluster("-" + name, conf);
- conf.setBoolean(RegistryConstants.KEY_REGISTRY_ENABLED, true);
- conf.set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, getZKBinding());
- //now create the cluster
- name = super.createMiniCluster(name, conf, noOfNodeManagers,
- numLocalDirs, numLogDirs, startHDFS);
-
- return name;
- }
-
- /**
- * Create and start a minicluster.
- * @param name cluster/test name
- * @param conf configuration to use
- * @param noOfNodeManagers #of NMs
- * @param startZK create a ZK micro cluster
- */
- protected String createMiniCluster(String name,
- YarnConfiguration conf,
- int noOfNodeManagers,
- boolean startZK) throws IOException {
- return createMiniCluster(name, conf, noOfNodeManagers, 1, 1, startZK,
- false);
- }
-
- /**
- * Create and start a minicluster with the name from the test method.
- * @param conf configuration to use
- * @param noOfNodeManagers #of NMs
- * @param startZK create a ZK micro cluster
- */
- protected String createMiniCluster(YarnConfiguration conf,
- int noOfNodeManagers,
- boolean startZK) throws IOException {
- return createMiniCluster("", conf, noOfNodeManagers, 1, 1, startZK,
- false);
- }
-
- public void createMicroZKCluster(String name, Configuration conf) {
- microZKCluster = new MicroZKCluster(new Configuration(conf));
- microZKCluster.createCluster(name);
- }
-
- public void assertHasZKCluster() {
- assertNotNull(microZKCluster);
- }
-
- public String getZKBinding() {
- if (microZKCluster == null) {
- return "localhost:1";
- } else {
- return microZKCluster.getZkBindingString();
- }
- }
-
- /**
- * CLI args include all the ZK bindings needed.
- * @return
- */
- protected List<String> getExtraCLIArgs() {
- return Arrays.asList(
- "-D", define(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, getZKBinding())
- );
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
index b1d73c5..a163b33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
@@ -6,7 +6,7 @@
"properties": {
"g1": "a",
"g2": "b",
- "internal.chaos.monkey.interval.seconds": "60"
+ "yarn.service.failure-count-reset.window": "60"
}
},
"resource": {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
deleted file mode 100644
index bc6429c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "nodes": ["/users/example/services/org-apache-slider/test-registry-rest-resources/components"], "service": {
- "description": "Slider Application Master",
- "yarn:id": "application_1411664296263_0001",
- "yarn:persistence": 1,
- "external": [],
- "internal": []
-}
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-3-role.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-3-role.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-3-role.json
deleted file mode 100644
index ceab0a5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-3-role.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{"entry":{"org.apache.slider.server.avro.RoleHistoryHeader":{"version":1,"saved":1415296260647,"savedx":"149863b1a27","savedate":"6 Nov 2014 17:51:00 GMT","roles":3}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":false,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":false,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":0,"active":false,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.86","role":2,"active":true,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.RoleHistoryFooter":{"count":4}}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-6-role.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-6-role.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-6-role.json
deleted file mode 100644
index f1c53d5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-6-role.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{"entry":{"org.apache.slider.server.avro.RoleHistoryHeader":{"version":1,"saved":1415296260647,"savedx":"149863b1a27","savedate":"6 Nov 2014 17:51:00 GMT","roles":6}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":false,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":false,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":0,"active":false,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.86","role":4,"active":true,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.86","role":5,"active":true,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.86","role":6,"active":true,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.RoleHistoryFooter":{"count":6}}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/avro/history_v01b_1_role.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/avro/history_v01b_1_role.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/avro/history_v01b_1_role.json
deleted file mode 100644
index 67d644f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/server/avro/history_v01b_1_role.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
- "entry": {
- "org.apache.slider.server.avro.RoleHistoryHeader": {
- "version": 1,
- "saved": 1450435691617,
- "savedx": "151b4b44461",
- "savedate": "18 Dec 2015 10:48:11 GMT",
- "roles": 2
- }
- }
-}
-{
- "entry": {
- "org.apache.slider.server.avro.RoleHistoryMapping": {
- "rolemap": {
- "echo": 1,
- "slider-appmaster": 0
- }
- }
- }
-}
-{
- "entry": {
- "org.apache.slider.server.avro.NodeEntryRecord": {
- "host": "192.168.56.1",
- "role": 1,
- "active": true,
- "last_used": 0
- }
- }
-}
-{
- "entry": {
- "org.apache.slider.server.avro.RoleHistoryFooter": {
- "count": 1
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-services.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-services.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-services.js
index fb01138..0f74f2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-services.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-services.js
@@ -23,7 +23,7 @@ export default AbstractRoute.extend({
model() {
return Ember.RSVP.hash({
apps: this.store.query('yarn-app', {
- applicationTypes: "org-apache-slider"
+ applicationTypes: "yarn-native-service"
}),
});
},
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java
deleted file mode 100644
index 8686479..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java
+++ /dev/null
@@ -1,430 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority;
-import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole;
-
-/**
- * Test app state RM operations.
- */
-public class TestMockAppStateRMOperations extends BaseMockAppStateTest
- implements MockRoles {
- private static final Logger LOG =
- LoggerFactory.getLogger(BaseMockAppStateTest.class);
-
- @Override
- public String getTestName() {
- return "TestMockAppStateRMOperations";
- }
-
- //@Test
- public void testPriorityOnly() throws Throwable {
- assertEquals(5, extractRole(buildPriority(5, false)));
- }
-
- //@Test
- public void testPriorityRoundTrip() throws Throwable {
- assertEquals(5, extractRole(buildPriority(5, false)));
- }
-
- //@Test
- public void testPriorityRoundTripWithRequest() throws Throwable {
- int priority = buildPriority(5, false);
- assertEquals(5, extractRole(priority));
- }
-
- //@Test
- public void testMockAddOp() throws Throwable {
- getRole0Status().setDesired(1);
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- assertListLength(ops, 1);
- ContainerRequestOperation operation = (ContainerRequestOperation)ops.get(0);
- int priority = operation.getRequest().getPriority().getPriority();
- assertEquals(extractRole(priority), getRole0Status().getKey());
- MockRMOperationHandler handler = new MockRMOperationHandler();
- handler.execute(ops);
-
- AbstractRMOperation op = handler.getFirstOp();
- assertTrue(op instanceof ContainerRequestOperation);
- }
-
- /**
- * Test of a flex up and down op which verifies that outstanding
- * requests are cancelled first.
- * <ol>
- * <li>request 5 nodes, assert 5 request made</li>
- * <li>allocate 1 of them</li>
- * <li>flex cluster size to 3</li>
- * <li>assert this generates 2 cancel requests</li>
- * </ol>
- */
- //@Test
- public void testRequestThenCancelOps() throws Throwable {
- RoleStatus role0 = getRole0Status();
- role0.setDesired(5);
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- assertListLength(ops, 5);
- // now 5 outstanding requests.
- assertEquals(5, role0.getRequested());
-
- // allocate one
- List<AbstractRMOperation> processed = new ArrayList<>();
- processed.add(ops.get(0));
- List<ContainerId> released = new ArrayList<>();
- List<AppState.NodeCompletionResult> completionResults = new ArrayList<>();
- submitOperations(processed, released);
- List<RoleInstance> instances = createAndSubmitNodes(released);
- processSubmissionOperations(instances, completionResults, released);
-
-
- // four outstanding
- assertEquals(4, role0.getRequested());
-
- // flex cluster to 3
- role0.setDesired(3);
- ops = appState.reviewRequestAndReleaseNodes();
-
- // expect two cancel operation from review
- assertListLength(ops, 2);
- for (AbstractRMOperation op : ops) {
- assertTrue(op instanceof CancelSingleRequest);
- }
-
- MockRMOperationHandler handler = new MockRMOperationHandler();
- handler.setAvailableToCancel(4);
- handler.execute(ops);
- assertEquals(2, handler.getAvailableToCancel());
- assertEquals(2, role0.getRequested());
-
- // flex down one more
- role0.setDesired(2);
- ops = appState.reviewRequestAndReleaseNodes();
- assertListLength(ops, 1);
- for (AbstractRMOperation op : ops) {
- assertTrue(op instanceof CancelSingleRequest);
- }
- handler.execute(ops);
- assertEquals(1, handler.getAvailableToCancel());
- assertEquals(1, role0.getRequested());
- }
-
- //@Test
- public void testCancelNoActualContainers() throws Throwable {
- RoleStatus role0 = getRole0Status();
- role0.setDesired(5);
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- assertListLength(ops, 5);
- // now 5 outstanding requests.
- assertEquals(5, role0.getRequested());
- role0.setDesired(0);
- ops = appState.reviewRequestAndReleaseNodes();
- assertListLength(ops, 5);
-
- }
-
-
- //@Test
- public void testFlexDownOutstandingRequests() throws Throwable {
- // engine only has two nodes, so > 2 will be outstanding
- engine = new MockYarnEngine(1, 2);
- List<AbstractRMOperation> ops;
- // role: desired = 2, requested = 1, actual=1
- RoleStatus role0 = getRole0Status();
- role0.setDesired(4);
- createAndSubmitNodes();
-
- assertEquals(2, role0.getRequested());
- assertEquals(2, role0.getRunning());
- // there are now two outstanding, two actual
- // Release 3 and verify that the two
- // cancellations were combined with a release
- role0.setDesired(1);
- assertEquals(-3, role0.getDelta());
- ops = appState.reviewRequestAndReleaseNodes();
- assertListLength(ops, 3);
- int numCancel = 0;
- int numRelease = 0;
- for (AbstractRMOperation op : ops) {
- if (op instanceof CancelSingleRequest) {
- numCancel++;
- }
- if (op instanceof ContainerReleaseOperation) {
- numRelease++;
- }
- }
- assertEquals(2, numCancel);
- assertEquals(1, numRelease);
- assertEquals(0, role0.getRequested());
- // TODO releasing?
-// assertEquals(1, role0.getReleasing());
- }
-
- //@Test
- public void testCancelAllOutstandingRequests() throws Throwable {
-
- // role: desired = 2, requested = 1, actual=1
- RoleStatus role0 = getRole0Status();
- role0.setDesired(2);
- List<AbstractRMOperation> ops;
- ops = appState.reviewRequestAndReleaseNodes();
- int count = 0;
- for (AbstractRMOperation op : ops) {
- if (op instanceof ContainerRequestOperation) {
- count++;
- }
- }
- assertEquals(2, count);
-
- // there are now two outstanding, two actual
- // Release 3 and verify that the two
- // cancellations were combined with a release
- role0.setDesired(0);
- ops = appState.reviewRequestAndReleaseNodes();
- assertEquals(2, ops.size());
-
- for (AbstractRMOperation op : ops) {
- assertTrue(op instanceof CancelSingleRequest);
- }
- }
-
-
- //@Test
- public void testFlexUpOutstandingRequests() throws Throwable {
-
- List<AbstractRMOperation> ops;
- // role: desired = 2, requested = 1, actual=1
- RoleStatus role0 = getRole0Status();
- role0.setDesired(2);
- appState.incRunningContainers(role0);
- appState.incRequestedContainers(role0);
-
- // flex up 2 nodes, yet expect only one node to be requested,
- // as the outstanding request is taken into account
- role0.setDesired(4);
- appState.incRequestedContainers(role0);
-
- assertEquals(1, role0.getRunning());
- assertEquals(2, role0.getRequested());
- assertEquals(3, role0.getActualAndRequested());
- assertEquals(1, role0.getDelta());
- ops = appState.reviewRequestAndReleaseNodes();
- assertListLength(ops, 1);
- assertTrue(ops.get(0) instanceof ContainerRequestOperation);
- assertEquals(3, role0.getRequested());
- }
-
- //@Test
- public void testFlexUpNoSpace() throws Throwable {
- // engine only has two nodes, so > 2 will be outstanding
- engine = new MockYarnEngine(1, 2);
- // role: desired = 2, requested = 1, actual=1
- RoleStatus role0 = getRole0Status();
- role0.setDesired(4);
- createAndSubmitNodes();
-
- assertEquals(2, role0.getRequested());
- assertEquals(2, role0.getRunning());
- role0.setDesired(8);
- assertEquals(4, role0.getDelta());
- createAndSubmitNodes();
- assertEquals(6, role0.getRequested());
- }
-
-
- //@Test
- public void testAllocateReleaseOp() throws Throwable {
- getRole0Status().setDesired(1);
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- ContainerRequestOperation operation = (ContainerRequestOperation)ops.get(0);
- AMRMClient.ContainerRequest request = operation.getRequest();
- Container cont = engine.allocateContainer(request);
- List<Container> allocated = new ArrayList<>();
- allocated.add(cont);
- List<ContainerAssignment> assignments = new ArrayList<>();
- List<AbstractRMOperation> operations = new ArrayList<>();
- appState.onContainersAllocated(allocated, assignments, operations);
-
- assertListLength(ops, 1);
- assertListLength(assignments, 1);
- ContainerAssignment assigned = assignments.get(0);
- Container target = assigned.container;
- assertEquals(target.getId(), cont.getId());
- int roleId = assigned.role.getPriority();
- assertEquals(roleId, extractRole(request.getPriority()));
- assertEquals(assigned.role.getName(), ROLE0);
- RoleInstance ri = roleInstance(assigned);
- //tell the app it arrived
- appState.containerStartSubmitted(target, ri);
- appState.innerOnNodeManagerContainerStarted(target.getId());
- assertEquals(1, getRole0Status().getRunning());
-
- //now release it by changing the role status
- getRole0Status().setDesired(0);
- ops = appState.reviewRequestAndReleaseNodes();
- assertListLength(ops, 1);
-
- assertTrue(ops.get(0) instanceof ContainerReleaseOperation);
- ContainerReleaseOperation release = (ContainerReleaseOperation) ops.get(0);
- assertEquals(release.getContainerId(), cont.getId());
- }
-
- //@Test
- public void testComplexAllocation() throws Throwable {
- getRole0Status().setDesired(1);
- getRole1Status().setDesired(3);
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- List<Container> allocations = engine.execute(ops);
- List<ContainerAssignment> assignments = new ArrayList<>();
- List<AbstractRMOperation> releases = new ArrayList<>();
- appState.onContainersAllocated(allocations, assignments, releases);
- // we expect four release requests here for all the allocated containers
- assertListLength(releases, 4);
- for (AbstractRMOperation op : releases) {
- assertTrue(op instanceof CancelSingleRequest);
- }
- assertListLength(assignments, 4);
- for (ContainerAssignment assigned : assignments) {
- Container target = assigned.container;
- RoleInstance ri = roleInstance(assigned);
- appState.containerStartSubmitted(target, ri);
- }
- //insert some async operation here
- for (ContainerAssignment assigned : assignments) {
- Container target = assigned.container;
- appState.innerOnNodeManagerContainerStarted(target.getId());
- }
- assertEquals(4, engine.containerCount());
- getRole1Status().setDesired(0);
- ops = appState.reviewRequestAndReleaseNodes();
- assertListLength(ops, 3);
- allocations = engine.execute(ops);
- assertEquals(1, engine.containerCount());
-
- appState.onContainersAllocated(allocations, assignments, releases);
- assertTrue(assignments.isEmpty());
- assertTrue(releases.isEmpty());
- }
-
- //@Test
- public void testDoubleNodeManagerStartEvent() throws Throwable {
- getRole0Status().setDesired(1);
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- List<Container> allocations = engine.execute(ops);
- List<ContainerAssignment> assignments = new ArrayList<>();
- List<AbstractRMOperation> releases = new ArrayList<>();
- appState.onContainersAllocated(allocations, assignments, releases);
- assertListLength(assignments, 1);
- ContainerAssignment assigned = assignments.get(0);
- Container target = assigned.container;
- RoleInstance ri = roleInstance(assigned);
- appState.containerStartSubmitted(target, ri);
- RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target
- .getId());
- assertEquals(ri2, ri);
- //try a second time, expect an error
- try {
- appState.innerOnNodeManagerContainerStarted(target.getId());
- fail("Expected an exception");
- } catch (RuntimeException expected) {
- // expected
- }
- //and non-faulter should not downgrade to a null
- LOG.warn("Ignore any exception/stack trace that appears below");
- LOG.warn("===============================================================");
- RoleInstance ri3 = appState.onNodeManagerContainerStarted(target.getId());
- LOG.warn("===============================================================");
- LOG.warn("Ignore any exception/stack trace that appeared above");
- assertNull(ri3);
- }
-
- //@Test
- public void testDoubleAllocate() throws Throwable {
- getRole0Status().setDesired(1);
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- ContainerRequestOperation operation = (ContainerRequestOperation)ops.get(0);
- AMRMClient.ContainerRequest request = operation.getRequest();
- Container cont = engine.allocateContainer(request);
- List<Container> allocated = new ArrayList<>();
- allocated.add(cont);
- List<ContainerAssignment> assignments = new ArrayList<>();
- List<AbstractRMOperation> operations = new ArrayList<>();
- assertEquals(0L, getRole0Status().getRunning());
- assertEquals(1L, getRole0Status().getRequested());
- appState.onContainersAllocated(allocated, assignments, operations);
-
- assertListLength(ops, 1);
- assertListLength(assignments, 1);
- ContainerAssignment assigned = assignments.get(0);
- Container target = assigned.container;
- assertEquals(target.getId(), cont.getId());
- int roleId = assigned.role.getPriority();
- assertEquals(roleId, extractRole(request.getPriority()));
- assertEquals(assigned.role.getName(), ROLE0);
- RoleInstance ri = roleInstance(assigned);
- //tell the app it arrived
- appState.containerStartSubmitted(target, ri);
- appState.innerOnNodeManagerContainerStarted(target.getId());
- assertEquals(1L, getRole0Status().getRunning());
- assertEquals(0L, getRole0Status().getRequested());
-
- // now get an extra allocation that should be released
- cont = engine.allocateContainer(request);
- allocated = new ArrayList<>();
- allocated.add(cont);
- assignments = new ArrayList<>();
- operations = new ArrayList<>();
- appState.onContainersAllocated(allocated, assignments, operations);
-
- assertListLength(operations, 1);
- assertTrue(operations.get(0) instanceof ContainerReleaseOperation);
- ContainerReleaseOperation release = (ContainerReleaseOperation)
- operations.get(0);
- assertEquals(release.getContainerId(), cont.getId());
-
- assertEquals(1L, getRole0Status().getRunning());
- assertEquals(0L, getRole0Status().getRequested());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java
deleted file mode 100644
index d257248..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockAppState;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
-import org.apache.slider.server.appmaster.state.NodeEntry;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.NodeMap;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Test that app state is rebuilt on a restart.
- */
-public class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- public String getTestName() {
- return "TestMockAppStateRebuildOnAMRestart";
- }
-
- //@Test
- public void testRebuild() throws Throwable {
-
- int r0 = 1;
- int r1 = 2;
- int r2 = 3;
- getRole0Status().setDesired(r0);
- getRole1Status().setDesired(r1);
- getRole2Status().setDesired(r2);
- List<RoleInstance> instances = createAndStartNodes();
-
- int clusterSize = r0 + r1 + r2;
- assertEquals(instances.size(), clusterSize);
-
- //clone the list
- List<Container> containers = new ArrayList<>();
- for (RoleInstance ri : instances) {
- containers.add(ri.container);
- }
- NodeMap nodemap = appState.getRoleHistory().cloneNodemap();
-
- //and rebuild
-
- AppStateBindingInfo bindingInfo = buildBindingInfo();
- bindingInfo.application = factory.newApplication(r0, r1, r2)
- .name(getValidTestName());
- bindingInfo.liveContainers = containers;
- appState = new MockAppState(bindingInfo);
-
- assertEquals(appState.getLiveContainers().size(), clusterSize);
-
- appState.getRoleHistory().dump();
-
- //check that the app state direct structures match
- List<RoleInstance> r0live = appState.enumLiveNodesInRole(ROLE0);
- List<RoleInstance> r1live = appState.enumLiveNodesInRole(ROLE1);
- List<RoleInstance> r2live = appState.enumLiveNodesInRole(ROLE2);
-
- assertEquals(r0, r0live.size());
- assertEquals(r1, r1live.size());
- assertEquals(r2, r2live.size());
-
- //now examine the role history
- NodeMap newNodemap = appState.getRoleHistory().cloneNodemap();
-
- for (NodeInstance nodeInstance : newNodemap.values()) {
- String hostname = nodeInstance.hostname;
- NodeInstance orig = nodemap.get(hostname);
- assertNotNull("Null entry in original nodemap for " + hostname, orig);
-
- for (int i : Arrays.asList(getRole0Status().getKey(), getRole1Status()
- .getKey(), getRole2Status().getKey())) {
- assertEquals(nodeInstance.getActiveRoleInstances(i), orig
- .getActiveRoleInstances(i));
- NodeEntry origRE = orig.getOrCreate(i);
- NodeEntry newRE = nodeInstance.getOrCreate(i);
- assertEquals(origRE.getLive(), newRE.getLive());
- assertEquals(0, newRE.getStarting());
- }
- }
- assertEquals(0, appState.reviewRequestAndReleaseNodes().size());
-
- Application application = appState.getClusterStatus();
- // verify the AM restart container count was set
- Long restarted = application.getNumberOfRunningContainers();
- assertNotNull(restarted);
- //and that the count == 1 master + the region servers
- assertEquals(restarted.longValue(), (long)containers.size());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java
deleted file mode 100644
index 2eccd1b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.state.RoleHistoryUtils;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole;
-
-/**
- * Test that the app state lets you ask for nodes, get a specific host,
- * release it and then get that one back again.
- */
-public class TestMockAppStateRolePlacement extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- public String getTestName() {
- return "TestMockAppStateRolePlacement";
- }
-
-
- //@Test
- public void testAllocateReleaseRealloc() throws Throwable {
- getRole0Status().setDesired(1);
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- ContainerRequestOperation operation = (ContainerRequestOperation)ops
- .get(0);
- AMRMClient.ContainerRequest request = operation.getRequest();
- assertTrue(request.getRelaxLocality());
- assertNull(request.getNodes());
- assertNull(request.getRacks());
- assertNotNull(request.getCapability());
-
- Container allocated = engine.allocateContainer(request);
- List<ContainerAssignment> assignments = new ArrayList<>();
- List<AbstractRMOperation> releaseOperations = new ArrayList<>();
- appState.onContainersAllocated(Arrays.asList((Container)allocated),
- assignments, releaseOperations);
- // verify the release matches the allocation
- assertEquals(releaseOperations.size(), 1);
- CancelSingleRequest cancelOp = (CancelSingleRequest)releaseOperations
- .get(0);
- assertNotNull(cancelOp.getRequest());
- assertNotNull(cancelOp.getRequest().getCapability());
- assertEquals(cancelOp.getRequest().getCapability(), allocated
- .getResource());
- // now the assignment
- assertEquals(assignments.size(), 1);
- ContainerAssignment assigned = assignments.get(0);
- Container container = assigned.container;
- assertEquals(container.getId(), allocated.getId());
- int roleId = assigned.role.getPriority();
- assertEquals(roleId, extractRole(request.getPriority()));
- assertEquals(assigned.role.getName(), ROLE0);
- String containerHostname = RoleHistoryUtils.hostnameOf(container);
- RoleInstance ri = roleInstance(assigned);
- //tell the app it arrived
- appState.containerStartSubmitted(container, ri);
- assertNotNull(appState.onNodeManagerContainerStarted(container.getId()));
- assertEquals(getRole0Status().getRunning(), 1);
- ops = appState.reviewRequestAndReleaseNodes();
- assertEquals(ops.size(), 0);
-
- //now it is surplus
- getRole0Status().setDesired(0);
- ops = appState.reviewRequestAndReleaseNodes();
- ContainerReleaseOperation release = (ContainerReleaseOperation) ops.get(0);
-
- assertEquals(release.getContainerId(), container.getId());
- engine.execute(ops);
- assertNotNull(appState.onCompletedContainer(containerStatus(container
- .getId())).roleInstance);
-
- //view the world
- appState.getRoleHistory().dump();
-
- //now ask for a new one
- getRole0Status().setDesired(1);
- ops = appState.reviewRequestAndReleaseNodes();
- assertEquals(ops.size(), 1);
- operation = (ContainerRequestOperation) ops.get(0);
- AMRMClient.ContainerRequest request2 = operation.getRequest();
- assertNotNull(request2);
- assertEquals(request2.getNodes().get(0), containerHostname);
- assertFalse(request2.getRelaxLocality());
- engine.execute(ops);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java
deleted file mode 100644
index b6c8526..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Test that if you have >1 role, the right roles are chosen for release.
- */
-public class TestMockAppStateRoleRelease extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- public String getTestName() {
- return "TestMockAppStateRoleRelease";
- }
-
- /**
- * Small cluster with multiple containers per node,
- * to guarantee many container allocations on each node.
- * @return
- */
- @Override
- public MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(4, 4);
- }
-
- //@Test
- public void testAllocateReleaseRealloc() throws Throwable {
- /**
- * Allocate to all nodes
- */
- getRole0Status().setDesired(6);
- getRole1Status().setDesired(5);
- getRole2Status().setDesired(4);
- List<RoleInstance> instances = createAndStartNodes();
- assertEquals(instances.size(), 15);
-
- //now it is surplus
- getRole0Status().setDesired(0);
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
-
- List<ContainerId> released = new ArrayList<>();
- engine.execute(ops, released);
- List<ContainerId> ids = extractContainerIds(instances, ROLE0);
- for (ContainerId cid : released) {
- assertNotNull(appState.onCompletedContainer(containerStatus(cid))
- .roleInstance);
- assertTrue(ids.contains(cid));
- }
-
- //view the world
- appState.getRoleHistory().dump();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
deleted file mode 100644
index 8dae8e7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.Resource;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
-import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
-import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.TreeMap;
-
-/**
- * Test that if you have more than one role, the right roles are chosen for
- * release.
- */
-public class TestMockAppStateUniqueNames extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- public String getTestName() {
- return "TestMockAppStateUniqueNames";
- }
-
- /**
- * Small cluster with multiple containers per node,
- * to guarantee many container allocations on each node.
- * @return
- */
- @Override
- public MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(4, 4);
- }
-
- @Override
- public AppStateBindingInfo buildBindingInfo() throws IOException {
- AppStateBindingInfo bindingInfo = super.buildBindingInfo();
- bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector();
- return bindingInfo;
- }
-
- @Override
- public Application buildApplication() {
- Application application = super.buildApplication();
-
- Component component = new Component().name("group1").numberOfContainers(2L)
- .resource(new Resource().memory("1024").cpus(2))
- .uniqueComponentSupport(true);
-
- application.getComponents().add(component);
- return application;
- }
-
- public static Map<String, RoleInstance> organize(List<RoleInstance>
- instances) {
- Map<String, RoleInstance> map = new TreeMap<>();
- for (RoleInstance instance : instances) {
- assertFalse("Multiple role instances for unique name " + instance
- .compInstanceName, map.containsKey(instance.compInstanceName));
- System.out.println("Adding to map " + instance.compInstanceName + " for" +
- instance.role);
- map.put(instance.compInstanceName, instance);
- }
- return map;
- }
-
- public static void verifyInstances(List<RoleInstance> instances, String
- group, String... roles) {
- assertEquals(roles.length, instances.size());
- Map<String, RoleInstance> map = organize(instances);
- int i = 0;
- for (Entry<String, RoleInstance> entry : map.entrySet()) {
- assertEquals(roles[i], entry.getKey());
- RoleInstance instance = entry.getValue();
- assertEquals(roles[i], instance.compInstanceName);
- assertEquals(i, instance.componentId);
- assertEquals(group, instance.role);
- assertEquals(group, instance.providerRole.name);
- i++;
- }
- }
-
- //@Test
- public void testDynamicFlexDown() throws Throwable {
- createAndStartNodes();
- List<RoleInstance> instances = appState.cloneOwnedContainerList();
- verifyInstances(instances, "group1", "group10", "group11");
-
- appState.updateComponents(Collections.singletonMap("group1", 0L));
- createAndStartNodes();
- instances = appState.cloneOwnedContainerList();
- assertEquals(0, instances.size());
-
- RoleStatus roleStatus = appState.lookupRoleStatus("group1");
- assertEquals(0, roleStatus.getDesired());
- assertEquals(1024L, roleStatus.getResourceRequirements().getMemorySize());
- assertEquals(2, roleStatus.getResourceRequirements().getVirtualCores());
-
- // now flex back up
- appState.updateComponents(Collections.singletonMap("group1", 3L));
- createAndStartNodes();
- instances = appState.cloneOwnedContainerList();
- verifyInstances(instances, "group1", "group10", "group11", "group12");
- }
-
- //@Test
- public void testDynamicFlexUp() throws Throwable {
- createAndStartNodes();
- List<RoleInstance> instances = appState.cloneOwnedContainerList();
- verifyInstances(instances, "group1", "group10", "group11");
-
- appState.updateComponents(Collections.singletonMap("group1", 3L));
- createAndStartNodes();
- instances = appState.cloneOwnedContainerList();
- verifyInstances(instances, "group1", "group10", "group11", "group12");
-
- RoleStatus group1 = appState.lookupRoleStatus("group1");
- assertEquals(3, group1.getDesired());
- assertEquals(1024L, group1.getResourceRequirements().getMemorySize());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
deleted file mode 100644
index 8bf2742..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockAppState;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Test the container resource allocation logic.
- */
-public class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
-
- @Override
- public Application buildApplication() {
- return factory.newApplication(1, 0, 0).name(getValidTestName());
- }
-
- //@Test
- public void testNormalAllocations() throws Throwable {
- Component role0 = appState.getClusterStatus().getComponent(MockRoles.ROLE0);
- role0.resource(new org.apache.slider.api.resource.Resource().memory("512")
- .cpus(2));
- // hack - because role0 is created before the test run
- RoleStatus role0Status =
- appState.getRoleStatusMap().get(appState.getRoleMap().get(ROLE0).id);
- role0Status.setResourceRequirements(
- appState.buildResourceRequirements(role0Status));
- appState.updateComponents(Collections.singletonMap(role0.getName(),
- role0.getNumberOfContainers()));
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- assertEquals(1, ops.size());
- ContainerRequestOperation operation = (ContainerRequestOperation) ops
- .get(0);
- Resource requirements = operation.getRequest().getCapability();
- assertEquals(512L, requirements.getMemorySize());
- assertEquals(2, requirements.getVirtualCores());
- }
-
- //TODO replace with resource profile feature in yarn
- //@Test
- public void testMaxMemAllocations() throws Throwable {
- // max core allocations no longer supported
- Component role0 = appState.getClusterStatus().getComponent(MockRoles.ROLE0);
- role0.resource(new org.apache.slider.api.resource.Resource()
- .memory(ResourceKeys.YARN_RESOURCE_MAX).cpus(2));
- RoleStatus role0Status =
- appState.getRoleStatusMap().get(appState.getRoleMap().get(ROLE0).id);
- role0Status.setResourceRequirements(
- appState.buildResourceRequirements(role0Status));
- appState.updateComponents(Collections.singletonMap(role0.getName(),
- role0.getNumberOfContainers()));
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- assertEquals(1, ops.size());
- ContainerRequestOperation operation = (ContainerRequestOperation) ops
- .get(0);
- Resource requirements = operation.getRequest().getCapability();
- assertEquals(MockAppState.RM_MAX_RAM, requirements.getMemorySize());
- assertEquals(2, requirements.getVirtualCores());
- }
-
- //@Test
- public void testMaxDefaultAllocations() throws Throwable {
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- assertEquals(ops.size(), 1);
- ContainerRequestOperation operation = (ContainerRequestOperation) ops
- .get(0);
- Resource requirements = operation.getRequest().getCapability();
- assertEquals(ResourceKeys.DEF_YARN_MEMORY, requirements.getMemorySize());
- assertEquals(ResourceKeys.DEF_YARN_CORES, requirements.getVirtualCores());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java
deleted file mode 100644
index 453b14c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.slider.server.appmaster.model.mock.MockFactory;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.AppState.NodeUpdatedOutcome;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Test Anti-affine placement.
- */
-public class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
- implements MockRoles {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestMockLabelledAAPlacement.class);
-
- private static final int NODES = 3;
- private static final int GPU_NODES = 2;
- private static final String HOST0 = "00000000";
- private static final String HOST1 = "00000001";
-
- @Override
- public void setup() throws Exception {
- super.setup();
-
- updateNodes(MockFactory.INSTANCE.newNodeReport(HOST0, NodeState.RUNNING,
- LABEL_GPU));
- updateNodes(MockFactory.INSTANCE.newNodeReport(HOST1, NodeState.RUNNING,
- LABEL_GPU));
- }
-
- @Override
- public MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(NODES, 8);
- }
-
- void assertAllContainersAA() {
- assertAllContainersAA(getGpuRole().getKey());
- }
-
- /**
- *
- * @throws Throwable
- */
- //@Test
- public void testAskForTooMany() throws Throwable {
- RoleStatus gpuRole = getGpuRole();
-
- describe("Ask for 1 more than the no of available nodes;" +
- " expect the final request to be unsatisfied until the cluster " +
- "changes size");
- //more than expected
- int size = GPU_NODES;
- gpuRole.setDesired(size + 1);
-
- List<AbstractRMOperation > operations = appState
- .reviewRequestAndReleaseNodes();
- assertTrue(gpuRole.isAARequestOutstanding());
-
- assertEquals(gpuRole.getAAPending(), size);
- for (int i = 0; i < size; i++) {
- String iter = "Iteration " + i + " role = " + getAaRole();
- describe(iter);
- List<AbstractRMOperation > operationsOut = new ArrayList<>();
-
- List<RoleInstance> roleInstances = submitOperations(operations,
- EMPTY_ID_LIST, operationsOut);
- // one instance per request
- assertEquals(1, roleInstances.size());
- appState.onNodeManagerContainerStarted(roleInstances.get(0)
- .getContainerId());
- assertAllContainersAA();
- // there should be none left
- LOG.debug(nodeInformationSnapshotAsString());
- operations = operationsOut;
- if (i + 1 < size) {
- assertEquals(2, operations.size());
- } else {
- assertEquals(1, operations.size());
- }
- }
- // expect an outstanding AA request to be unsatisfied
- assertTrue(gpuRole.getRunning() < gpuRole.getDesired());
- assertEquals(0, gpuRole.getRequested());
- assertFalse(gpuRole.isAARequestOutstanding());
- List<Container> allocatedContainers = engine.execute(operations,
- EMPTY_ID_LIST);
- assertEquals(0, allocatedContainers.size());
- // in a review now, no more requests can be generated, as there is no
- // space for AA placements, even though there is cluster capacity
- assertEquals(0, appState.reviewRequestAndReleaseNodes().size());
-
- // switch node 2 into being labelled
- NodeUpdatedOutcome outcome = updateNodes(MockFactory.INSTANCE.
- newNodeReport("00000002", NodeState.RUNNING, "gpu"));
-
- assertEquals(NODES, cloneNodemap().size());
- assertTrue(outcome.clusterChanged);
- // no active calls to empty
- assertTrue(outcome.operations.isEmpty());
- assertEquals(1, appState.reviewRequestAndReleaseNodes().size());
- }
-
- protected AppState.NodeUpdatedOutcome addNewNode() {
- return updateNodes(MockFactory.INSTANCE.newNodeReport("00000004",
- NodeState.RUNNING, "gpu"));
- }
-
- //@Test
- public void testClusterSizeChangesDuringRequestSequence() throws Throwable {
- RoleStatus gpuRole = getGpuRole();
- describe("Change the cluster size where the cluster size changes during " +
- "a test sequence.");
- gpuRole.setDesired(GPU_NODES + 1);
- List<AbstractRMOperation> operations = appState
- .reviewRequestAndReleaseNodes();
- assertTrue(gpuRole.isAARequestOutstanding());
- assertEquals(GPU_NODES, gpuRole.getAAPending());
- NodeUpdatedOutcome outcome = addNewNode();
- assertTrue(outcome.clusterChanged);
- // one call to cancel
- assertEquals(1, outcome.operations.size());
- // and on a review, one more to rebuild
- assertEquals(1, appState.reviewRequestAndReleaseNodes().size());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java
deleted file mode 100644
index 65d1d86..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
-import org.apache.slider.server.appmaster.state.ContainerPriority;
-import org.apache.slider.server.appmaster.state.OutstandingRequest;
-import org.apache.slider.utils.SliderTestBase;
-import org.junit.Test;
-
-/**
- * Test outstanding request validation.
- */
-public class TestOutstandingRequestValidation extends SliderTestBase {
-
- private static final String[] H1 = hosts("one");
-
- //@Test
- public void testRelaxedNohostsOrLabels() throws Throwable {
- createAndValidate(null, null, true);
- }
-
- //@Test
- public void testRelaxedLabels() throws Throwable {
- createAndValidate(null, "gpu", true);
- }
-
- //@Test
- public void testNonRelaxedLabels() throws Throwable {
- expectCreationFailure(null, "gpu", false);
- }
-
- //@Test
- public void testRelaxedHostNoLabel() throws Throwable {
- createAndValidate(H1, "", true);
- }
-
- /**
- * Use varargs for simple list to array conversion.
- * @param hostnames host names
- * @return
- */
- public static String[] hosts(String...hostnames) {
- return hostnames;
- }
-
- void expectCreationFailure(
- String[] hosts,
- String labels,
- boolean relaxLocality) {
- try {
- ContainerRequest result = createAndValidate(hosts, labels, relaxLocality);
- fail("Expected an exception, got " + result);
- } catch (IllegalArgumentException expected) {
- assertTrue(expected.toString()
- .contains("Can't turn off locality relaxation on a request with no " +
- "location constraints"));
- }
- }
-
- AMRMClient.ContainerRequest createAndValidate(
- String[] hosts,
- String labels,
- boolean relaxLocality) {
- int cores = 1;
- int memory = 64;
- int p = 1;
- Priority pri = ContainerPriority.createPriority(p, !relaxLocality);
- ContainerRequest issuedRequest =
- newRequest(pri, hosts, labels, relaxLocality);
- OutstandingRequest.validateContainerRequest(issuedRequest, p, "");
- return issuedRequest;
- }
-
- AMRMClient.ContainerRequest newRequest(
- Priority pri,
- String[] hosts,
- String labels,
- boolean relaxLocality) {
- int cores = 1;
- int memory = 64;
- Resource resource = Resource.newInstance(memory, cores);
- return new AMRMClient.ContainerRequest(resource,
- hosts,
- null,
- pri,
- relaxLocality,
- labels);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java
deleted file mode 100644
index 0cdb952..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history;
-
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.slider.api.proto.Messages;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.api.types.NodeInformationList;
-import org.apache.slider.api.types.RestTypeMarshalling;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.server.appmaster.model.mock.MockFactory;
-import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
-import org.apache.slider.server.appmaster.state.NodeEntry;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.NodeMap;
-import org.apache.slider.server.appmaster.state.RoleHistory;
-import org.apache.slider.utils.SliderTestBase;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Test anti-affine placement.
- */
-public class TestRoleHistoryAA extends SliderTestBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestRoleHistoryAA.class);
-
- private List<String> hostnames = Arrays.asList("1", "2", "3");
- private NodeMap nodeMap, gpuNodeMap;
- private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
-
- public TestRoleHistoryAA() throws BadConfigException {
- }
-
- @Override
- public void setup() throws Exception {
- super.setup();
- nodeMap = createNodeMap(hostnames, NodeState.RUNNING, "");
- gpuNodeMap = createNodeMap(hostnames, NodeState.RUNNING, "GPU");
- }
-
- //@Test
- public void testFindNodesInFullCluster() throws Throwable {
- // all three will surface at first
- verifyResultSize(3, nodeMap.findAllNodesForRole(1, ""));
- }
-
- //@Test
- public void testFindNodesInUnhealthyCluster() throws Throwable {
- // all three will surface at first
- markNodeOneUnhealthy();
- verifyResultSize(2, nodeMap.findAllNodesForRole(1, ""));
- }
-
- public boolean markNodeOneUnhealthy() {
- return setNodeState(nodeMap.get("1"), NodeState.UNHEALTHY);
- }
-
- protected boolean setNodeState(NodeInstance node, NodeState state) {
- return node.updateNode(MockFactory.INSTANCE.newNodeReport(node.hostname,
- state, ""));
- }
-
- //@Test
- public void testFindNoNodesWrongLabel() throws Throwable {
- // all three will surface at first
- verifyResultSize(0, nodeMap.findAllNodesForRole(1, "GPU"));
- }
-
- //@Test
- public void testFindSomeNodesSomeLabel() throws Throwable {
- // all three will surface at first
- update(nodeMap,
- Arrays.asList(MockFactory.INSTANCE.newNodeReport("1", NodeState
- .RUNNING, "GPU")));
- List<NodeInstance> gpuNodes = nodeMap.findAllNodesForRole(1, "GPU");
- verifyResultSize(1, gpuNodes);
- NodeInstance instance = gpuNodes.get(0);
- instance.getOrCreate(1).onStarting();
- assertFalse(instance.canHost(1, "GPU"));
- assertFalse(instance.canHost(1, ""));
- verifyResultSize(0, nodeMap.findAllNodesForRole(1, "GPU"));
-
- }
-
- //@Test
- public void testFindNoNodesRightLabel() throws Throwable {
- // all three will surface at first
- verifyResultSize(3, gpuNodeMap.findAllNodesForRole(1, "GPU"));
- }
-
- //@Test
- public void testFindNoNodesNoLabel() throws Throwable {
- // all three will surface at first
- verifyResultSize(3, gpuNodeMap.findAllNodesForRole(1, ""));
- }
-
- //@Test
- public void testFindNoNodesClusterRequested() throws Throwable {
- // all three will surface at first
- for (NodeInstance ni : nodeMap.values()) {
- ni.getOrCreate(1).request();
- }
- assertNoAvailableNodes(1);
- }
-
- //@Test
- public void testFindNoNodesClusterBusy() throws Throwable {
- // all three will surface at first
- for (NodeInstance ni : nodeMap.values()) {
- ni.getOrCreate(1).request();
- }
- assertNoAvailableNodes(1);
- }
-
- /**
- * Tag all nodes as starting, then walk one through a bit
- * more of its lifecycle.
- */
- //@Test
- public void testFindNoNodesLifecycle() throws Throwable {
- // all three will surface at first
- for (NodeInstance ni : nodeMap.values()) {
- ni.getOrCreate(1).onStarting();
- }
- assertNoAvailableNodes(1);
-
- // walk one of the nodes through the lifecycle
- NodeInstance node1 = nodeMap.get("1");
- assertFalse(node1.canHost(1, ""));
- node1.get(1).onStartCompleted();
- assertFalse(node1.canHost(1, ""));
- assertNoAvailableNodes(1);
- node1.get(1).release();
- assertTrue(node1.canHost(1, ""));
- List<NodeInstance> list2 =
- verifyResultSize(1, nodeMap.findAllNodesForRole(1, ""));
- assertEquals(list2.get(0).hostname, "1");
-
- // now tag that node as unhealthy and expect it to go away
- markNodeOneUnhealthy();
- assertNoAvailableNodes(1);
- }
-
- //@Test
- public void testRolesIndependent() throws Throwable {
- NodeInstance node1 = nodeMap.get("1");
- NodeEntry role1 = node1.getOrCreate(1);
- NodeEntry role2 = node1.getOrCreate(2);
- for (NodeInstance ni : nodeMap.values()) {
- ni.updateNode(MockFactory.INSTANCE.newNodeReport("0", NodeState
- .UNHEALTHY, ""));
- }
- assertNoAvailableNodes(1);
- assertNoAvailableNodes(2);
- assertTrue(setNodeState(node1, NodeState.RUNNING));
- // tag role 1 as busy
- role1.onStarting();
- assertNoAvailableNodes(1);
-
- verifyResultSize(1, nodeMap.findAllNodesForRole(2, ""));
- assertTrue(node1.canHost(2, ""));
- }
-
- //@Test
- public void testNodeEntryAvailablity() throws Throwable {
- NodeEntry entry = new NodeEntry(1);
- assertTrue(entry.isAvailable());
- entry.onStarting();
- assertFalse(entry.isAvailable());
- entry.onStartCompleted();
- assertFalse(entry.isAvailable());
- entry.release();
- assertTrue(entry.isAvailable());
- entry.onStarting();
- assertFalse(entry.isAvailable());
- entry.onStartFailed();
- assertTrue(entry.isAvailable());
- }
-
- //@Test
- public void testNodeInstanceSerialization() throws Throwable {
- MockRoleHistory rh2 = new MockRoleHistory(new ArrayList<>());
- rh2.getOrCreateNodeInstance("localhost");
- NodeInstance instance = rh2.getOrCreateNodeInstance("localhost");
- instance.getOrCreate(1).onStartCompleted();
- Map<Integer, String> naming = Collections.singletonMap(1, "manager");
- NodeInformation ni = instance.serialize(naming);
- assertEquals(1, ni.entries.get("manager").live);
- NodeInformation ni2 = rh2.getNodeInformation("localhost", naming);
- assertEquals(1, ni2.entries.get("manager").live);
- Map<String, NodeInformation> info = rh2.getNodeInformationSnapshot(naming);
- assertEquals(1, info.get("localhost").entries.get("manager").live);
- NodeInformationList nil = new NodeInformationList(info.values());
- assertEquals(1, nil.get(0).entries.get("manager").live);
-
- Messages.NodeInformationProto nodeInformationProto =
- RestTypeMarshalling.marshall(ni);
- Messages.NodeEntryInformationProto entryProto = nodeInformationProto
- .getEntries(0);
- assertNotNull(entryProto);
- assertEquals(1, entryProto.getPriority());
- NodeInformation unmarshalled =
- RestTypeMarshalling.unmarshall(nodeInformationProto);
- assertEquals(unmarshalled.hostname, ni.hostname);
- assertTrue(unmarshalled.entries.keySet().containsAll(ni.entries.keySet()));
-
- }
-
- //@Test
- public void testBuildRolenames() throws Throwable {
-
- }
- public List<NodeInstance> assertNoAvailableNodes(int role) {
- String label = "";
- return verifyResultSize(0, nodeMap.findAllNodesForRole(role, label));
- }
-
- List<NodeInstance> verifyResultSize(int size, List<NodeInstance> list) {
- if (list.size() != size) {
- for (NodeInstance ni : list) {
- LOG.error(ni.toFullString());
- }
- }
- assertEquals(size, list.size());
- return list;
- }
-
- NodeMap createNodeMap(List<NodeReport> nodeReports)
- throws BadConfigException {
- NodeMap newNodeMap = new NodeMap(1);
- update(newNodeMap, nodeReports);
- return newNodeMap;
- }
-
- protected boolean update(NodeMap nm, List<NodeReport> nodeReports) {
- return nm.buildOrUpdate(nodeReports);
- }
-
- NodeMap createNodeMap(List<String> hosts, NodeState state,
- String label) throws BadConfigException {
- return createNodeMap(MockFactory.INSTANCE.createNodeReports(hosts, state,
- label));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java
deleted file mode 100644
index bbe95b9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java
+++ /dev/null
@@ -1,447 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockContainer;
-import org.apache.slider.server.appmaster.model.mock.MockNodeId;
-import org.apache.slider.server.appmaster.state.ContainerOutcome;
-import org.apache.slider.server.appmaster.state.ContainerPriority;
-import org.apache.slider.server.appmaster.state.NodeEntry;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.NodeMap;
-import org.apache.slider.server.appmaster.state.RoleHistory;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Test container events at the role history level -one below
- * the App State.
- */
-public class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestRoleHistoryContainerEvents.class);
-
- @Override
- public String getTestName() {
- return "TestRoleHistoryContainerEvents";
- }
-
- private NodeInstance age1Active4;
- private NodeInstance age2Active2;
- private NodeInstance age3Active0;
- private NodeInstance age4Active1;
- private NodeInstance age2Active0;
-
- private RoleHistory roleHistory;
-
- private Resource resource;
-
- AMRMClient.ContainerRequest requestContainer(RoleStatus roleStatus) {
- return roleHistory.requestContainerForRole(roleStatus).getIssuedRequest();
- }
-
- @Override
- public void setup() throws Exception {
- super.setup();
-
- age1Active4 = nodeInstance(1, 4, 0, 0);
- age2Active2 = nodeInstance(2, 2, 0, 1);
- age3Active0 = nodeInstance(3, 0, 0, 0);
- age4Active1 = nodeInstance(4, 1, 0, 0);
- age2Active0 = nodeInstance(2, 0, 0, 0);
-
- roleHistory = appState.getRoleHistory();
- roleHistory.insert(Arrays.asList(age2Active2, age2Active0,
- age4Active1, age1Active4, age3Active0));
- roleHistory.buildRecentNodeLists();
- resource = Resource.newInstance(ResourceKeys.DEF_YARN_CORES,
- ResourceKeys.DEF_YARN_MEMORY);
- }
-
- //@Test
- public void testFindAndCreate() throws Throwable {
- RoleStatus roleStatus = getRole0Status();
-
- AMRMClient.ContainerRequest request =
- requestContainer(roleStatus);
-
- List<String> requestNodes = request.getNodes();
- assertNotNull(requestNodes);
- assertEquals(1, requestNodes.size());
- String hostname = requestNodes.get(0);
- assertEquals(hostname, age3Active0.hostname);
-
- //build a container
- MockContainer container = factory.newContainer();
- container.setNodeId(new MockNodeId(hostname, 0));
- container.setPriority(request.getPriority());
- roleHistory.onContainerAssigned(container);
-
- NodeMap nodemap = roleHistory.cloneNodemap();
- NodeInstance allocated = nodemap.get(hostname);
- NodeEntry roleEntry = allocated.get(roleStatus.getKey());
- assertEquals(1, roleEntry.getStarting());
- assertFalse(roleEntry.isAvailable());
- RoleInstance ri = new RoleInstance(container);
- //start it
- roleHistory.onContainerStartSubmitted(container, ri);
- //later, declare that it started
- roleHistory.onContainerStarted(container);
- assertEquals(0, roleEntry.getStarting());
- assertFalse(roleEntry.isAvailable());
- assertEquals(1, roleEntry.getActive());
- assertEquals(1, roleEntry.getLive());
- }
-
- //@Test
- public void testCreateAndRelease() throws Throwable {
- RoleStatus roleStatus = getRole1Status();
-
- //verify it is empty
- assertTrue(roleHistory.listActiveNodes(roleStatus.getKey()).isEmpty());
-
- AMRMClient.ContainerRequest request =
- requestContainer(roleStatus);
-
- assertNull(request.getNodes());
-
- //pick an idle host
- String hostname = age3Active0.hostname;
-
- //build a container
- MockContainer container = factory.newContainer(new MockNodeId(hostname,
- 0), request.getPriority());
- roleHistory.onContainerAssigned(container);
-
- NodeMap nodemap = roleHistory.cloneNodemap();
- NodeInstance allocated = nodemap.get(hostname);
- NodeEntry roleEntry = allocated.get(roleStatus.getKey());
- assertEquals(1, roleEntry.getStarting());
- assertFalse(roleEntry.isAvailable());
- RoleInstance ri = new RoleInstance(container);
- //start it
- roleHistory.onContainerStartSubmitted(container, ri);
- //later, declare that it started
- roleHistory.onContainerStarted(container);
- assertEquals(0, roleEntry.getStarting());
- assertFalse(roleEntry.isAvailable());
- assertEquals(1, roleEntry.getActive());
- assertEquals(1, roleEntry.getLive());
-
- // now pick that instance to destroy
- List<NodeInstance> activeNodes = roleHistory.listActiveNodes(roleStatus
- .getKey());
-
-
- assertEquals(1, activeNodes.size());
- NodeInstance target = activeNodes.get(0);
- assertEquals(target, allocated);
- roleHistory.onContainerReleaseSubmitted(container);
- assertEquals(1, roleEntry.getReleasing());
- assertEquals(1, roleEntry.getLive());
- assertEquals(0, roleEntry.getActive());
-
- // release completed
- roleHistory.onReleaseCompleted(container);
- assertEquals(0, roleEntry.getReleasing());
- assertEquals(0, roleEntry.getLive());
- assertEquals(0, roleEntry.getActive());
-
- // verify it is empty
- assertTrue(roleHistory.listActiveNodes(roleStatus.getKey()).isEmpty());
-
- // ask for a container and expect to get the recently released one
- AMRMClient.ContainerRequest request2 =
- requestContainer(roleStatus);
-
- List<String> nodes2 = request2.getNodes();
- assertNotNull(nodes2);
- String hostname2 = nodes2.get(0);
-
- //pick an idle host
- assertEquals(hostname2, age3Active0.hostname);
- }
-
-
- //@Test
- public void testStartWithoutWarning() throws Throwable {
- //pick an idle host
- String hostname = age3Active0.hostname;
- //build a container
- MockContainer container = factory.newContainer(
- new MockNodeId(hostname, 0),
- ContainerPriority.createPriority(getRole0Status().getKey(), false));
-
- NodeMap nodemap = roleHistory.cloneNodemap();
- NodeInstance allocated = nodemap.get(hostname);
- NodeEntry roleEntry = allocated.get(getRole0Status().getKey());
-
- //tell RH that it started
- roleHistory.onContainerStarted(container);
- assertEquals(0, roleEntry.getStarting());
- assertFalse(roleEntry.isAvailable());
- assertEquals(1, roleEntry.getActive());
- assertEquals(1, roleEntry.getLive());
- }
-
- //@Test
- public void testStartFailed() throws Throwable {
- RoleStatus roleStatus = getRole0Status();
-
- AMRMClient.ContainerRequest request =
- requestContainer(roleStatus);
-
- LOG.info("req {}", request);
- LOG.info("{}", request.getNodes());
- String hostname = request.getNodes().get(0);
- assertEquals(hostname, age3Active0.hostname);
-
- //build a container
- MockContainer container = factory.newContainer(new MockNodeId(hostname,
- 0), request.getPriority());
- roleHistory.onContainerAssigned(container);
-
- NodeMap nodemap = roleHistory.cloneNodemap();
- NodeInstance allocated = nodemap.get(hostname);
- NodeEntry roleEntry = allocated.get(roleStatus.getKey());
- assertEquals(1, roleEntry.getStarting());
- assertFalse(roleEntry.isAvailable());
- RoleInstance ri = new RoleInstance(container);
- //start it
- roleHistory.onContainerStartSubmitted(container, ri);
- //later, declare that it failed on startup
- assertFalse(roleHistory.onNodeManagerContainerStartFailed(container));
- assertEquals(0, roleEntry.getStarting());
- assertEquals(1, roleEntry.getStartFailed());
- assertEquals(1, roleEntry.getFailed());
- assertTrue(roleEntry.isAvailable());
- assertEquals(0, roleEntry.getActive());
- assertEquals(0, roleEntry.getLive());
- }
-
- //@Test
- public void testStartFailedWithoutWarning() throws Throwable {
- RoleStatus roleStatus = getRole0Status();
-
- AMRMClient.ContainerRequest request =
- requestContainer(roleStatus);
-
- String hostname = request.getNodes().get(0);
- assertEquals(hostname, age3Active0.hostname);
-
- //build a container
- MockContainer container = factory.newContainer();
- container.setNodeId(new MockNodeId(hostname, 0));
- container.setPriority(request.getPriority());
-
- NodeMap nodemap = roleHistory.cloneNodemap();
- NodeInstance allocated = nodemap.get(hostname);
- NodeEntry roleEntry = allocated.get(roleStatus.getKey());
-
- assertFalse(roleHistory.onNodeManagerContainerStartFailed(container));
- assertEquals(0, roleEntry.getStarting());
- assertEquals(1, roleEntry.getStartFailed());
- assertEquals(1, roleEntry.getFailed());
- assertTrue(roleEntry.isAvailable());
- assertEquals(0, roleEntry.getActive());
- assertEquals(0, roleEntry.getLive());
- }
-
- //@Test
- public void testContainerFailed() throws Throwable {
- describe("fail a container without declaring it as starting");
-
- RoleStatus roleStatus = getRole0Status();
-
- AMRMClient.ContainerRequest request =
- requestContainer(roleStatus);
-
- String hostname = request.getNodes().get(0);
- assertEquals(hostname, age3Active0.hostname);
-
- //build a container
- MockContainer container = factory.newContainer();
- container.setNodeId(new MockNodeId(hostname, 0));
- container.setPriority(request.getPriority());
- roleHistory.onContainerAssigned(container);
-
- NodeMap nodemap = roleHistory.cloneNodemap();
- NodeInstance allocated = nodemap.get(hostname);
- NodeEntry roleEntry = allocated.get(roleStatus.getKey());
- assertEquals(1, roleEntry.getStarting());
- assertFalse(roleEntry.isAvailable());
- RoleInstance ri = new RoleInstance(container);
- //start it
- roleHistory.onContainerStartSubmitted(container, ri);
- roleHistory.onContainerStarted(container);
-
- //later, declare that it failed
- roleHistory.onFailedContainer(
- container,
- false,
- ContainerOutcome.Failed);
- assertEquals(0, roleEntry.getStarting());
- assertTrue(roleEntry.isAvailable());
- assertEquals(0, roleEntry.getActive());
- assertEquals(0, roleEntry.getLive());
- }
-
- //@Test
- public void testContainerFailedWithoutWarning() throws Throwable {
- describe("fail a container without declaring it as starting");
- RoleStatus roleStatus = getRole0Status();
-
- AMRMClient.ContainerRequest request =
- requestContainer(roleStatus);
-
- String hostname = request.getNodes().get(0);
- assertEquals(hostname, age3Active0.hostname);
-
- //build a container
- MockContainer container = factory.newContainer();
- container.setNodeId(new MockNodeId(hostname, 0));
- container.setPriority(request.getPriority());
-
-
- NodeMap nodemap = roleHistory.cloneNodemap();
- NodeInstance allocated = nodemap.get(hostname);
- NodeEntry roleEntry = allocated.get(roleStatus.getKey());
- assertTrue(roleEntry.isAvailable());
- roleHistory.onFailedContainer(
- container,
- false,
- ContainerOutcome.Failed);
- assertEquals(0, roleEntry.getStarting());
- assertEquals(1, roleEntry.getFailed());
- assertTrue(roleEntry.isAvailable());
- assertEquals(0, roleEntry.getActive());
- assertEquals(0, roleEntry.getLive());
- }
-
- //@Test
- public void testAllocationListPrep() throws Throwable {
- describe("test prepareAllocationList");
- RoleStatus roleStatus = getRole0Status();
-
- AMRMClient.ContainerRequest request =
- requestContainer(roleStatus);
-
- String hostname = request.getNodes().get(0);
- assertEquals(hostname, age3Active0.hostname);
-
- MockContainer container1 = factory.newContainer();
- container1.setNodeId(new MockNodeId(hostname, 0));
- container1.setPriority(Priority.newInstance(getRole0Status().getKey()));
-
- MockContainer container2 = factory.newContainer();
- container2.setNodeId(new MockNodeId(hostname, 0));
- container2.setPriority(Priority.newInstance(getRole1Status().getKey()));
-
- // put containers in List with role == 1 first
- List<Container> containers = Arrays.asList((Container) container2,
- (Container) container1);
- List<Container> sortedContainers = roleHistory.prepareAllocationList(
- containers);
-
- // verify that the first container has role == 0 after sorting
- MockContainer c1 = (MockContainer) sortedContainers.get(0);
- assertEquals(getRole0Status().getKey(), c1.getPriority().getPriority());
- MockContainer c2 = (MockContainer) sortedContainers.get(1);
- assertEquals(getRole1Status().getKey(), c2.getPriority().getPriority());
- }
-
- //@Test
- public void testNodeUpdated() throws Throwable {
- describe("fail a node");
-
- RoleStatus roleStatus = getRole0Status();
-
- AMRMClient.ContainerRequest request =
- roleHistory.requestContainerForRole(roleStatus).getIssuedRequest();
-
- String hostname = request.getNodes().get(0);
- assertEquals(age3Active0.hostname, hostname);
-
- // build a container
- MockContainer container = factory.newContainer(new MockNodeId(hostname,
- 0), request.getPriority());
-
- roleHistory.onContainerAssigned(container);
-
- NodeMap nodemap = roleHistory.cloneNodemap();
- NodeInstance allocated = nodemap.get(hostname);
- NodeEntry roleEntry = allocated.get(roleStatus.getKey());
- assertEquals(1, roleEntry.getStarting());
- assertFalse(roleEntry.isAvailable());
- RoleInstance ri = new RoleInstance(container);
- // start it
- roleHistory.onContainerStartSubmitted(container, ri);
- roleHistory.onContainerStarted(container);
-
- int startSize = nodemap.size();
-
- // now send a list of updated (failed) nodes event
- List<NodeReport> nodesUpdated = new ArrayList<>();
- NodeReport nodeReport = NodeReport.newInstance(
- NodeId.newInstance(hostname, 0),
- NodeState.LOST,
- null, null, null, null, 1, null, 0);
- nodesUpdated.add(nodeReport);
- roleHistory.onNodesUpdated(nodesUpdated);
-
- nodemap = roleHistory.cloneNodemap();
- int endSize = nodemap.size();
- // as even unused nodes are added to the list, we expect the map size to
- // be >1
- assertTrue(startSize <= endSize);
- assertNotNull(nodemap.get(hostname));
- assertFalse(nodemap.get(hostname).isOnline());
-
- // add a failure of a node we've never head of
- String newhost = "newhost";
- nodesUpdated = Arrays.asList(
- NodeReport.newInstance(
- NodeId.newInstance(newhost, 0),
- NodeState.LOST,
- null, null, null, null, 1, null, 0)
- );
- roleHistory.onNodesUpdated(nodesUpdated);
-
- NodeMap nodemap2 = roleHistory.cloneNodemap();
- assertNotNull(nodemap2.get(newhost));
- assertFalse(nodemap2.get(newhost).isOnline());
-
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[66/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
new file mode 100644
index 0000000..3e53418
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
+import java.io.File;
+
+/**
+ * Registry actions
+ *
+ * --instance {app name}, if a / is in it, refers underneath?
+ * --dest {destfile}
+ * --list : list instances of slider service
+ * --listfiles
+ */
+@Parameters(commandNames = {ACTION_REGISTRY},
+ commandDescription = DESCRIBE_ACTION_REGISTRY)
+
+public class ActionRegistryArgs extends AbstractActionArgs {
+
+ public static final String USAGE =
+ "Usage: " + SliderActions.ACTION_REGISTRY
+ + " ("
+ + Arguments.ARG_LIST + "|"
+ + Arguments.ARG_LISTCONF + "|"
+ + Arguments.ARG_LISTEXP + "|"
+ + Arguments.ARG_LISTFILES + "|"
+ + Arguments.ARG_GETCONF + "|"
+ + Arguments.ARG_GETEXP + "> "
+ + Arguments.ARG_NAME + " <name> "
+ + " )"
+ + "[" + Arguments.ARG_VERBOSE + "] "
+ + "[" + Arguments.ARG_USER + "] "
+ + "[" + Arguments.ARG_OUTPUT + " <filename> ] "
+ + "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] "
+ + "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] "
+ + System.getProperty("line.separator")
+ + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
+ ;
+ public ActionRegistryArgs() {
+ }
+
+ public ActionRegistryArgs(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String getActionName() {
+ return ACTION_REGISTRY;
+ }
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Parameter(names = {ARG_LIST},
+ description = "list services")
+ public boolean list;
+
+ @Parameter(names = {ARG_LISTCONF},
+ description = "list configurations")
+ public boolean listConf;
+
+ @Parameter(names = {ARG_GETCONF},
+ description = "get configuration")
+ public String getConf;
+
+ @Parameter(names = {ARG_LISTEXP},
+ description = "list exports")
+ public boolean listExports;
+
+ @Parameter(names = {ARG_GETEXP},
+ description = "get export")
+ public String getExport;
+
+ @Parameter(names = {ARG_LISTFILES},
+ description = "list files")
+ public String listFiles;
+
+ @Parameter(names = {ARG_GETFILES},
+ description = "get files")
+ public String getFiles;
+
+ //--format
+ @Parameter(names = ARG_FORMAT,
+ description = "Format for a response: <xml|json|properties>")
+ public String format = ConfigFormat.XML.toString() ;
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
+ description = "Output destination")
+ public File out;
+
+ @Parameter(names = {ARG_NAME},
+ description = "name of an instance")
+ public String name;
+
+ @Parameter(names = {ARG_SERVICETYPE},
+ description = "optional service type")
+ public String serviceType = YarnServiceConstants.APP_TYPE;
+
+ @Parameter(names = {ARG_VERBOSE},
+ description = "verbose output")
+ public boolean verbose;
+
+ @Parameter(names = {ARG_INTERNAL},
+ description = "fetch internal registry entries")
+ public boolean internal;
+
+ @Parameter(names = {ARG_USER},
+ description = "the name of the user whose service is being resolved")
+ public String user;
+
+ /**
+ * validate health of all the different operations
+ * @throws BadCommandArgumentsException
+ */
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+
+ //verify that at most one of the operations is set
+ int gets = s(getConf) + s(getFiles) + s(getExport);
+ int lists = s(list) + s(listConf) + s(listFiles) + s(listExports);
+ int set = lists + gets;
+ if (set > 1) {
+ throw new UsageException(USAGE);
+ }
+
+ if (out != null && ( set == 0)) {
+ throw new UsageException("output path"
+ + " is only supported on 'get' operations: ");
+ }
+ if (!list && !is(name)) {
+ throw new UsageException("Argument " + ARG_NAME
+ +" missing: ");
+
+ }
+ }
+
+ private int s(String arg) {
+ return is(arg) ? 1 : 0;
+ }
+
+ private boolean is(String arg) {
+ return arg != null;
+ }
+
+ private int s(boolean arg) {
+ return arg ? 1 : 0;
+ }
+
+ private String ifdef(String arg, boolean val) {
+ return val ? (arg + " "): "";
+ }
+
+ private String ifdef(String arg, String val) {
+ if (is(val)) {
+ return arg + " " + val + " ";
+ } else {
+ return "";
+ }
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder(ACTION_REGISTRY);
+ sb.append(' ');
+ sb.append(ifdef(ARG_LIST, list));
+ sb.append(ifdef(ARG_LISTCONF, listConf));
+ sb.append(ifdef(ARG_LISTFILES, listFiles));
+ sb.append(ifdef(ARG_GETCONF, getConf));
+ sb.append(ifdef(ARG_GETFILES, getFiles));
+
+ sb.append(ifdef(ARG_NAME, name));
+ sb.append(ifdef(ARG_SERVICETYPE, serviceType));
+
+
+ sb.append(ifdef(ARG_VERBOSE, verbose));
+ sb.append(ifdef(ARG_INTERNAL, internal));
+
+ if (out != null) {
+ sb.append(ifdef(ARG_OUTPUT, out.toString()));
+ }
+ sb.append(ifdef(ARG_FORMAT, format));
+
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
new file mode 100644
index 0000000..65f0472
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+
+import java.io.File;
+
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_RESOLVE;
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
+
+/**
+ * Resolve registry entries
+ *
+ * --path {path}
+ * --out {destfile}
+ * --verbose
+ * --list
+ */
+@Parameters(commandNames = {ACTION_RESOLVE},
+ commandDescription = DESCRIBE_ACTION_REGISTRY)
+public class ActionResolveArgs extends AbstractActionArgs {
+
+ public static final String USAGE =
+ "Usage: " + SliderActions.ACTION_RESOLVE
+ + " "
+ + ARG_PATH + " <path> "
+ + "[" + ARG_LIST + "] "
+ + "[" + ARG_OUTPUT + " <filename> ] "
+ + "[" + ARG_DESTDIR + " <directory> ] "
+ ;
+ public ActionResolveArgs() {
+ }
+
+ @Override
+ public String getActionName() {
+ return ACTION_RESOLVE;
+ }
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Parameter(names = {ARG_LIST},
+ description = "list services")
+ public boolean list;
+
+ @Parameter(names = {ARG_PATH},
+ description = "resolve a path")
+ public String path;
+
+ @Parameter(names = {ARG_DESTDIR},
+ description = "destination directory for operations")
+ public File destdir;
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "dest file")
+ public File out;
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder(ACTION_RESOLVE).append(" ");
+ sb.append(ARG_PATH).append(" ").append(path).append(" ");
+ if (list) {
+ sb.append(ARG_LIST).append(" ");
+ }
+ if (destdir != null) {
+ sb.append(ARG_DESTDIR).append(" ").append(destdir).append(" ");
+ }
+ if (out != null) {
+ sb.append(ARG_OUTPUT).append(" ").append(out).append(" ");
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+ if (StringUtils.isEmpty(path)) {
+ throw new BadCommandArgumentsException("Missing mandatory argument "
+ + ARG_PATH);
+ }
+ if (list && out != null) {
+ throw new BadCommandArgumentsException("Argument "
+ + ARG_OUTPUT +
+ " not supported for " + ARG_LIST);
+ }
+ if (out != null && destdir != null) {
+ throw new BadCommandArgumentsException(
+ ARG_OUTPUT + " and " + ARG_DESTDIR + " cannot be used together"
+ );
+ }
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ public boolean isList() {
+ return list;
+ }
+
+ public void setList(boolean list) {
+ this.list = list;
+ }
+
+ public File getDestdir() {
+ return destdir;
+ }
+
+ public void setDestdir(File destdir) {
+ this.destdir = destdir;
+ }
+
+ public File getOut() {
+ return out;
+ }
+
+ public void setOut(File out) {
+ this.out = out;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
new file mode 100644
index 0000000..b03dc92
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+@Parameters(commandNames = { SliderActions.ACTION_RESOURCE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_RESOURCE)
+
+public class ActionResourceArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_RESOURCE;
+ }
+
+ @Parameter(names = {ARG_INSTALL},
+ description = "Install the resource(s)")
+ public boolean install;
+
+ @Parameter(names = {ARG_DELETE},
+ description = "Delete the file")
+ public boolean delete;
+
+ @Parameter(names = {ARG_LIST},
+ description = "List of installed files")
+ public boolean list;
+
+ @Parameter(names = {ARG_RESOURCE},
+ description = "Name of the file or directory")
+ public String resource;
+
+ @Parameter(names = {ARG_DESTDIR},
+ description = "The name of the folder in which to store the resources")
+ public String folder;
+
+ @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing resource(s)")
+ public boolean overwrite = false;
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 3;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
new file mode 100644
index 0000000..31f25ef
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+@Parameters(commandNames = { SliderActions.ACTION_STATUS},
+ commandDescription = SliderActions.DESCRIBE_ACTION_STATUS)
+
+public class ActionStatusArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_STATUS;
+ }
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "Output file for the status information")
+ public String output;
+
+ @Parameter(names = {ARG_LIFETIME},
+ description = "Lifetime of the service from the time of request")
+ public boolean lifetime;
+
+ public String getOutput() {
+ return output;
+ }
+
+ public void setOutput(String output) {
+ this.output = output;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
new file mode 100644
index 0000000..175e367
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+import java.io.File;
+
+@Parameters(commandNames = { SliderActions.ACTION_START },
+ commandDescription = SliderActions.DESCRIBE_ACTION_THAW)
+public class ActionThawArgs extends AbstractActionArgs implements
+ WaitTimeAccessor,
+ LaunchArgsAccessor {
+
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_START;
+ }
+
+ @Override
+ public int getWaittime() {
+ return launchArgs.getWaittime();
+ }
+
+ @ParametersDelegate
+ LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
+
+ @Parameter(names = {ARG_LIFETIME},
+ description = "Life time of the service since service started at"
+ + " running state")
+ public long lifetime;
+
+ @Override
+ public String getRmAddress() {
+ return launchArgs.getRmAddress();
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ launchArgs.setWaittime(waittime);
+ }
+
+
+ @Override
+ public File getOutputFile() {
+ return launchArgs.getOutputFile();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
new file mode 100644
index 0000000..cf48513
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+
+import java.io.File;
+
+@Parameters(commandNames = { SliderActions.ACTION_TOKENS},
+ commandDescription = "save tokens to a file or list tokens in a file")
+public class ActionTokensArgs extends AbstractActionArgs {
+
+ public static final String DUPLICATE_ARGS = "Only one of " +
+ ARG_SOURCE + " and " + ARG_OUTPUT + " allowed";
+
+ public static final String MISSING_KT_PROVIDER =
+ "Both " + ARG_KEYTAB + " and " + ARG_PRINCIPAL
+ + " must be provided";
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_TOKENS;
+ }
+
+ @Parameter(names = {ARG_OUTPUT},
+ description = "File to write")
+ public File output;
+
+ @Parameter(names = {ARG_SOURCE},
+ description = "source file")
+ public File source;
+
+ @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
+ public File keytab;
+
+ @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
+ public String principal="";
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+ if (output != null && source != null) {
+ throw new BadCommandArgumentsException(DUPLICATE_ARGS);
+ }
+
+ // this is actually a !xor
+ if (keytab != null ^ !principal.isEmpty()) {
+ throw new BadCommandArgumentsException(MISSING_KT_PROVIDER);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
new file mode 100644
index 0000000..e310f45
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_UPDATE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE)
+
+public class ActionUpdateArgs extends AbstractClusterBuildingActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_UPDATE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
new file mode 100644
index 0000000..00151f4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Static argument manipulation operations
+ */
+public class ArgOps {
+
+ private static final Logger
+ log = LoggerFactory.getLogger(ArgOps.class);
+
+ /**
+ * create a 3-tuple
+ */
+ public static List<Object> triple(String msg, int min, int max) {
+ List<Object> l = new ArrayList<>(3);
+ l.add(msg);
+ l.add(min);
+ l.add(max);
+ return l;
+ }
+
+ public static void applyFileSystemBinding(String filesystemBinding,
+ Configuration conf) {
+ if (filesystemBinding != null) {
+ //filesystem argument was set -this overwrites any defaults in the
+ //configuration
+ FileSystem.setDefaultUri(conf, filesystemBinding);
+ }
+ }
+
+ public static void splitPairs(Collection<String> pairs,
+ Map<String, String> dest) {
+ for (String prop : pairs) {
+ String[] keyval = prop.split("=", 2);
+ if (keyval.length == 2) {
+ dest.put(keyval[0], keyval[1]);
+ }
+ }
+ }
+
+
+ public static void applyDefinitions(Map<String, String> definitionMap,
+ Configuration conf) {
+ for (Map.Entry<String, String> entry : definitionMap.entrySet()) {
+ String key = entry.getKey();
+ String val = entry.getValue();
+ log.debug("configuration[{}]<=\"{}\"", key, val);
+ conf.set(key, val, "command line");
+ }
+ }
+
+ /**
+ * Create a map from a tuple list like ['worker','2','master','1] into a map
+ * ['worker':'2',"master":'1'];
+ * Duplicate entries also trigger errors
+ * @param description description for errors
+ * @param list list to conver to tuples
+ * @return the map of key value pairs -unordered.
+ * @throws BadCommandArgumentsException odd #of arguments received
+ */
+ public static Map<String, String> convertTupleListToMap(String description,
+ List<String> list) throws
+ BadCommandArgumentsException {
+ Map<String, String> results = new HashMap<>();
+ if (list != null && !list.isEmpty()) {
+ int size = list.size();
+ if (size % 2 != 0) {
+ //odd number of elements, not permitted
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_PARSE_FAILURE + description);
+ }
+ for (int count = 0; count < size; count += 2) {
+ String key = list.get(count);
+ String val = list.get(count + 1);
+ if (results.get(key) != null) {
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_DUPLICATE_ENTRY + description
+ + ": " + key);
+ }
+ results.put(key, val);
+ }
+ }
+ return results;
+ }
+
+ /**
+ * Create a map from a tuple list like
+ * ['worker','heapsize','5G','master','heapsize','2M'] into a map
+ * ['worker':'2',"master":'1'];
+ * Duplicate entries also trigger errors
+
+ * @throws BadCommandArgumentsException odd #of arguments received
+ */
+ public static Map<String, Map<String, String>> convertTripleListToMaps(String description,
+ List<String> list) throws BadCommandArgumentsException {
+
+ Map<String, Map<String, String>> results = new HashMap<>();
+ if (list != null && !list.isEmpty()) {
+ int size = list.size();
+ if (size % 3 != 0) {
+ //wrong number of elements, not permitted
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_PARSE_FAILURE + description);
+ }
+ for (int count = 0; count < size; count += 3) {
+ String role = list.get(count);
+ String key = list.get(count + 1);
+ String val = list.get(count + 2);
+ Map<String, String> roleMap = results.get(role);
+ if (roleMap == null) {
+ //demand create new role map
+ roleMap = new HashMap<>();
+ results.put(role, roleMap);
+ }
+ if (roleMap.get(key) != null) {
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_DUPLICATE_ENTRY + description
+ + ": for key " + key + " under " + role);
+ }
+ roleMap.put(key, val);
+ }
+ }
+ return results;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
new file mode 100644
index 0000000..204149b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+/**
+ * Here are all the arguments that may be parsed by the client or server
+ * command lines.
+ *
+ * Important: Please keep the main list in alphabetical order
+ * so it is easier to see what arguments are there
+ */
+public interface Arguments {
+
+ String ARG_APPDEF = "--appdef";
+ String ARG_BASE_PATH = "--basepath";
+ String ARG_COMPONENT = "--component";
+ String ARG_COMPONENT_SHORT = "--comp";
+ String ARG_COMPONENTS = "--components";
+ String ARG_COMP_OPT= "--compopt";
+ String ARG_COMP_OPT_SHORT = "--co";
+ String ARG_CONFIG = "--config";
+ String ARG_CONTAINERS = "--containers";
+ String ARG_DEBUG = "--debug";
+ String ARG_DEFINE = "-D";
+ String ARG_DELETE = "--delete";
+ String ARG_DEST = "--dest";
+ String ARG_DESTDIR = "--destdir";
+ String ARG_FILESYSTEM = "--fs";
+ String ARG_FILESYSTEM_LONG = "--filesystem";
+ String ARG_FOLDER = "--folder";
+ String ARG_FORCE = "--force";
+ String ARG_FORMAT = "--format";
+ String ARG_GETCONF = "--getconf";
+ String ARG_GETEXP = "--getexp";
+ String ARG_GETFILES = "--getfiles";
+ String ARG_HELP = "--help";
+ String ARG_IMAGE = "--image";
+ String ARG_INSTALL = "--install";
+ String ARG_INTERNAL = "--internal";
+ String ARG_KEYLEN = "--keylen";
+ String ARG_KEYTAB = "--keytab";
+ String ARG_KEYTABINSTALL = ARG_INSTALL;
+ String ARG_KEYTABDELETE = ARG_DELETE;
+ String ARG_KEYTABLIST = "--list";
+ String ARG_LIST = "--list";
+ String ARG_LISTCONF = "--listconf";
+ String ARG_LISTEXP = "--listexp";
+ String ARG_LISTFILES = "--listfiles";
+ String ARG_LIVE = "--live";
+ String ARG_MANAGER = "--manager";
+ String ARG_MANAGER_SHORT = "--m";
+ String ARG_MESSAGE = "--message";
+ String ARG_NAME = "--name";
+ String ARG_OPTION = "--option";
+ String ARG_OPTION_SHORT = "-O";
+ String ARG_OUTPUT = "--out";
+ String ARG_OUTPUT_SHORT = "-o";
+ String ARG_OVERWRITE = "--overwrite";
+ String ARG_PACKAGE = "--package";
+ String ARG_PATH = "--path";
+ String ARG_PRINCIPAL = "--principal";
+ String ARG_QUEUE = "--queue";
+ String ARG_LIFETIME = "--lifetime";
+ String ARG_RESOURCE = "--resource";
+ String ARG_RESOURCE_MANAGER = "--rm";
+ String ARG_SECURE = "--secure";
+ String ARG_SERVICETYPE = "--servicetype";
+ String ARG_SERVICES = "--services";
+ String ARG_SOURCE = "--source";
+ String ARG_STATE = "--state";
+ String ARG_SYSPROP = "-S";
+ String ARG_USER = "--user";
+ String ARG_UPLOAD = "--upload";
+ String ARG_VERBOSE = "--verbose";
+ String ARG_VERSION = "--version";
+ String ARG_WAIT = "--wait";
+/*
+ STOP: DO NOT ADD YOUR ARGUMENTS HERE. GO BACK AND INSERT THEM IN THE
+ RIGHT PLACE IN THE LIST
+ */
+
+ /**
+ * server: URI for the cluster
+ */
+ String ARG_CLUSTER_URI = "-cluster-uri";
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
new file mode 100644
index 0000000..7b957fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+
+import java.util.Collection;
+
+/**
+ * Client CLI Args
+ */
+
+public class ClientArgs extends CommonArgs {
+
+ // =========================================================
+ // Keep all of these in alphabetical order. Thanks.
+ // =========================================================
+
+ private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
+ private final ActionClientArgs actionClientArgs = new ActionClientArgs();
+ private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
+ private final ActionDependencyArgs actionDependencyArgs = new ActionDependencyArgs();
+ private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs();
+ private final ActionExistsArgs actionExistsArgs = new ActionExistsArgs();
+ private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs();
+ private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
+ private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
+ private final ActionKDiagArgs actionKDiagArgs = new ActionKDiagArgs();
+ private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs();
+ private final ActionListArgs actionListArgs = new ActionListArgs();
+ private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
+ private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
+ private final ActionResourceArgs actionResourceArgs = new ActionResourceArgs();
+ private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
+ private final ActionThawArgs actionThawArgs = new ActionThawArgs();
+ private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs();
+ private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
+
+ public ClientArgs(String[] args) {
+ super(args);
+ }
+
+ public ClientArgs(Collection args) {
+ super(args);
+ }
+
+ @Override
+ protected void addActionArguments() {
+
+ addActions(
+ actionBuildArgs,
+ actionCreateArgs,
+ actionDependencyArgs,
+ actionDestroyArgs,
+ actionFlexArgs,
+ actionFreezeArgs,
+ actionHelpArgs,
+ actionStatusArgs,
+ actionThawArgs
+ );
+ }
+
+ @Override
+ public void applyDefinitions(Configuration conf) throws
+ BadCommandArgumentsException {
+ super.applyDefinitions(conf);
+ //RM
+ if (getManager() != null) {
+ log.debug("Setting RM to {}", getManager());
+ conf.set(YarnConfiguration.RM_ADDRESS, getManager());
+ }
+ if (getBasePath() != null) {
+ log.debug("Setting basePath to {}", getBasePath());
+ conf.set(YarnServiceConf.YARN_SERVICE_BASE_PATH,
+ getBasePath().toString());
+ }
+ }
+
+
+ public ActionBuildArgs getActionBuildArgs() {
+ return actionBuildArgs;
+ }
+
+ public ActionUpdateArgs getActionUpdateArgs() {
+ return actionUpdateArgs;
+ }
+
+ public ActionCreateArgs getActionCreateArgs() {
+ return actionCreateArgs;
+ }
+
+ public ActionDependencyArgs getActionDependencyArgs() {
+ return actionDependencyArgs;
+ }
+
+ public ActionDestroyArgs getActionDestroyArgs() {
+ return actionDestroyArgs;
+ }
+
+ public ActionExistsArgs getActionExistsArgs() {
+ return actionExistsArgs;
+ }
+
+ public ActionFlexArgs getActionFlexArgs() {
+ return actionFlexArgs;
+ }
+
+ public ActionFreezeArgs getActionFreezeArgs() {
+ return actionFreezeArgs;
+ }
+
+ public ActionListArgs getActionListArgs() {
+ return actionListArgs;
+ }
+
+
+ public ActionRegistryArgs getActionRegistryArgs() {
+ return actionRegistryArgs;
+ }
+
+ public ActionResolveArgs getActionResolveArgs() {
+ return actionResolveArgs;
+ }
+
+ public ActionResourceArgs getActionResourceArgs() {
+ return actionResourceArgs;
+ }
+
+ public ActionStatusArgs getActionStatusArgs() {
+ return actionStatusArgs;
+ }
+
+ public ActionThawArgs getActionThawArgs() {
+ return actionThawArgs;
+ }
+
+ public ActionTokensArgs getActionTokenArgs() {
+ return actionTokenArgs;
+ }
+
+ /**
+ * Look at the chosen action and bind it as the core action for the operation.
+ * @throws SliderException bad argument or similar
+ */
+ @Override
+ public void applyAction() throws SliderException {
+ String action = getAction();
+ if (SliderUtils.isUnset(action)) {
+ action = ACTION_HELP;
+ }
+ switch (action) {
+ case ACTION_BUILD:
+ bindCoreAction(actionBuildArgs);
+ break;
+
+ case ACTION_CREATE:
+ bindCoreAction(actionCreateArgs);
+ break;
+
+ case ACTION_STOP:
+ bindCoreAction(actionFreezeArgs);
+ break;
+
+ case ACTION_START:
+ bindCoreAction(actionThawArgs);
+ break;
+
+ case ACTION_DEPENDENCY:
+ bindCoreAction(actionDependencyArgs);
+ break;
+
+ case ACTION_DESTROY:
+ bindCoreAction(actionDestroyArgs);
+ break;
+
+ case ACTION_EXISTS:
+ bindCoreAction(actionExistsArgs);
+ break;
+
+ case ACTION_FLEX:
+ bindCoreAction(actionFlexArgs);
+ break;
+
+ case ACTION_HELP:
+ bindCoreAction(actionHelpArgs);
+ break;
+
+ case ACTION_KDIAG:
+ bindCoreAction(actionKDiagArgs);
+ break;
+
+ case ACTION_KEYTAB:
+ bindCoreAction(actionKeytabArgs);
+ break;
+
+ case ACTION_LIST:
+ bindCoreAction(actionListArgs);
+ break;
+
+ case ACTION_REGISTRY:
+ bindCoreAction(actionRegistryArgs);
+ break;
+
+ case ACTION_RESOLVE:
+ bindCoreAction(actionResolveArgs);
+ break;
+
+ case ACTION_RESOURCE:
+ bindCoreAction(actionResourceArgs);
+ break;
+
+ case ACTION_STATUS:
+ bindCoreAction(actionStatusArgs);
+ break;
+
+ case ACTION_TOKENS:
+ bindCoreAction(actionTokenArgs);
+ break;
+
+ case ACTION_UPDATE:
+ bindCoreAction(actionUpdateArgs);
+ break;
+
+ default:
+ throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
+ + " " + action);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
new file mode 100644
index 0000000..e1197ea
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
@@ -0,0 +1,282 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterDescription;
+import com.beust.jcommander.ParameterException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This class contains the common argument set for all tne entry points,
+ * and the core parsing logic to verify that the action is on the list
+ * of allowed actions -and that the remaining number of arguments is
+ * in the range allowed
+ */
+
+public abstract class CommonArgs extends ArgOps implements SliderActions,
+ Arguments {
+
+ protected static final Logger log = LoggerFactory.getLogger(CommonArgs.class);
+
+
+ private static final int DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME = 30;
+
+
+ @Parameter(names = ARG_HELP, help = true)
+ public boolean help;
+
+
+ /**
+ -D name=value
+
+ Define an HBase configuration option which overrides any options in
+ the configuration XML files of the image or in the image configuration
+ directory. The values will be persisted.
+ Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
+
+ */
+
+ public Map<String, String> definitionMap = new HashMap<String, String>();
+ /**
+ * System properties
+ */
+ public Map<String, String> syspropsMap = new HashMap<String, String>();
+
+
+ /**
+ * fields
+ */
+ public final JCommander commander;
+ private final String[] args;
+
+ private AbstractActionArgs coreAction;
+
+ /**
+ * get the name: relies on arg 1 being the cluster name in all operations
+ * @return the name argument, null if there is none
+ */
+ public String getClusterName() {
+ return coreAction.getClusterName();
+ }
+
+ protected CommonArgs(String[] args) {
+ this.args = args;
+ commander = new JCommander(this);
+ }
+
+ protected CommonArgs(Collection args) {
+ List<String> argsAsStrings = SliderUtils.collectionToStringList(args);
+ this.args = argsAsStrings.toArray(new String[argsAsStrings.size()]);
+ commander = new JCommander(this);
+ }
+
+ public String usage() {
+ return usage(this, null);
+ }
+
+ public static String usage(CommonArgs serviceArgs, String commandOfInterest) {
+ String result = null;
+ StringBuilder helperMessage = new StringBuilder();
+ if (commandOfInterest == null) {
+ // JCommander.usage is too verbose for a command with many options like
+ // slider no short version of that is found Instead, we compose our msg by
+ helperMessage.append("\nUsage: service COMMAND [options]\n");
+ helperMessage.append("where COMMAND is one of\n");
+ for (String jcommand : serviceArgs.commander.getCommands().keySet()) {
+ helperMessage.append(String.format("\t%-"
+ + DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", jcommand,
+ serviceArgs.commander.getCommandDescription(jcommand) + "\n"));
+ }
+ helperMessage
+ .append("Most commands print help when invoked without parameters or with --help");
+ result = helperMessage.toString();
+ } else {
+ helperMessage.append("\nUsage: service ").append(commandOfInterest);
+ helperMessage.append(serviceArgs.coreAction.getMinParams() > 0 ? " <service>" : "");
+ helperMessage.append("\n");
+ for (ParameterDescription paramDesc : serviceArgs.commander.getCommands()
+ .get(commandOfInterest).getParameters()) {
+ String optional = paramDesc.getParameter().required() ? " (required)"
+ : " (optional)";
+ String paramName = paramDesc.getParameterized().getType() == Boolean.TYPE ? paramDesc
+ .getLongestName() : paramDesc.getLongestName() + " <"
+ + paramDesc.getParameterized().getName() + ">";
+ helperMessage.append(String.format("\t%-"
+ + DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", paramName,
+ paramDesc.getDescription() + optional + "\n"));
+ result = helperMessage.toString();
+ }
+ }
+ return result;
+ }
+
+ public static String usage(CommonArgs serviceArgs) {
+ return usage(serviceArgs, null);
+ }
+
+ /**
+ * Parse routine -includes registering the action-specific argument classes
+ * and postprocess it
+ * @throws SliderException on any problem
+ */
+ public void parse() throws SliderException {
+ addActionArguments();
+ try {
+ commander.parse(args);
+ } catch (ParameterException e) {
+ throw new BadCommandArgumentsException(e, "%s in %s",
+ e.toString(),
+ (args != null
+ ? (SliderUtils.join(args,
+ " ", false))
+ : "[]"));
+ }
+ //now copy back to this class some of the attributes that are common to all
+ //actions
+ postProcess();
+ }
+
+ /**
+ * Add a command
+ * @param name action
+ * @param arg value
+ */
+ protected void addAction(String name, Object arg) {
+ commander.addCommand(name, arg);
+ }
+
+ protected void addActions(Object... actions) {
+ for (Object action : actions) {
+ commander.addCommand(action);
+ }
+ }
+
+ /**
+ * Override point to add a set of actions
+ */
+ protected void addActionArguments() {
+
+ }
+
+ /**
+ * validate args via {@link #validate()}
+ * then postprocess the arguments
+ */
+ public void postProcess() throws SliderException {
+ applyAction();
+ validate();
+
+ //apply entry set
+ for (Map.Entry<String, String> entry : syspropsMap.entrySet()) {
+ System.setProperty(entry.getKey(), entry.getValue());
+ }
+ }
+
+
+ /**
+ * Implementors must implement their action apply routine here
+ */
+ public abstract void applyAction() throws SliderException;
+
+
+ /**
+ * Bind the core action; this extracts any attributes that are used
+ * across routines
+ * @param action action to bind
+ */
+ protected void bindCoreAction(AbstractActionArgs action) {
+ coreAction = action;
+
+ splitPairs(coreAction.definitions, definitionMap);
+ splitPairs(coreAction.sysprops, syspropsMap);
+ }
+
+ /**
+ * Validate the arguments against the action requested
+ */
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ if (coreAction == null) {
+ throw new UsageException(ErrorStrings.ERROR_NO_ACTION + usage());
+ }
+ log.debug("action={}", getAction());
+ // let the action validate itself
+ try {
+ coreAction.validate();
+ } catch (BadCommandArgumentsException e) {
+ String badArgMsgBuilder =
+ e.getMessage() + System.lineSeparator() + usage(this,
+ coreAction.getActionName());
+ throw new BadCommandArgumentsException(badArgMsgBuilder);
+ }
+ }
+
+ /**
+ * Apply all the definitions on the command line to the configuration
+ * @param conf config
+ */
+ public void applyDefinitions(Configuration conf) throws
+ BadCommandArgumentsException {
+ applyDefinitions(definitionMap, conf);
+ }
+
+
+ /**
+ * If the Filesystem binding was provided, it overrides anything in
+ * the configuration
+ * @param conf configuration
+ */
+ public void applyFileSystemBinding(Configuration conf) {
+ ArgOps.applyFileSystemBinding(getFilesystemBinding(), conf);
+ }
+
+ public boolean isDebug() {
+ return coreAction.debug;
+ }
+
+
+ public String getFilesystemBinding() {
+ return coreAction.filesystemBinding;
+ }
+
+ public Path getBasePath() { return coreAction.basePath; }
+
+ public String getManager() {
+ return coreAction.manager;
+ }
+
+ public String getAction() {
+ return commander.getParsedCommand();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
new file mode 100644
index 0000000..b6cd0a1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class ComponentArgsDelegate extends AbstractArgsDelegate {
+
+ /**
+ * This is a listing of the roles to create
+ */
+ @Parameter(names = {ARG_COMPONENT, ARG_COMPONENT_SHORT},
+ arity = 2,
+ description = "--component <name> <count> e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3",
+ splitter = DontSplitArguments.class)
+ public List<String> componentTuples = new ArrayList<>(0);
+
+
+ /**
+ * Get the role mapping (may be empty, but never null)
+ * @return role mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, String> getComponentMap() throws BadCommandArgumentsException {
+ return convertTupleListToMap("component", componentTuples);
+ }
+
+ public List<String> getComponentTuples() {
+ return componentTuples;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
new file mode 100644
index 0000000..85de615
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.converters.IParameterSplitter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DontSplitArguments implements IParameterSplitter {
+
+ @Override
+ public List<String> split(String value) {
+ List<String> list = new ArrayList<>(1);
+ list.add(value);
+ return list;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
new file mode 100644
index 0000000..bf194b6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import java.io.File;
+
+/**
+ * Launch args for create and start and anything else that can start something
+ */
+public interface LaunchArgsAccessor extends WaitTimeAccessor {
+ String getRmAddress();
+
+ File getOutputFile();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
new file mode 100644
index 0000000..d42510c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+
+import java.io.File;
+
+/**
+ * Any launch-time args
+ */
+public class LaunchArgsDelegate extends WaitArgsDelegate implements
+ LaunchArgsAccessor {
+
+
+ //TODO: do we need this?
+ @Parameter(names = ARG_RESOURCE_MANAGER,
+ description = "Resource manager hostname:port ",
+ required = false)
+ private String rmAddress;
+
+ @Override
+ public String getRmAddress() {
+ return rmAddress;
+ }
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for any service report")
+ public File outputFile;
+
+ @Override
+ public File getOutputFile() {
+ return outputFile;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
new file mode 100644
index 0000000..7972716
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Delegate for application and resource options.
+ */
+public class OptionArgsDelegate extends AbstractArgsDelegate {
+
+ /**
+ * Options key value.
+ */
+ @Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
+ description = ARG_OPTION + "<name> <value>",
+ splitter = DontSplitArguments.class)
+ public List<String> optionTuples = new ArrayList<>(0);
+
+
+ /**
+ * All the app component option triples.
+ */
+ @Parameter(names = {ARG_COMP_OPT, ARG_COMP_OPT_SHORT}, arity = 3,
+ description = "Component option " + ARG_COMP_OPT +
+ " <component> <name> <option>",
+ splitter = DontSplitArguments.class)
+ public List<String> compOptTriples = new ArrayList<>(0);
+
+ public Map<String, String> getOptionsMap() throws
+ BadCommandArgumentsException {
+ return convertTupleListToMap(ARG_OPTION, optionTuples);
+ }
+
+ /**
+ * Get the role heap mapping (may be empty, but never null).
+ * @return role heap mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, Map<String, String>> getCompOptionMap()
+ throws BadCommandArgumentsException {
+ return convertTripleListToMaps(ARG_COMP_OPT, compOptTriples);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
new file mode 100644
index 0000000..040ac64
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.converters.BaseConverter;
+import org.apache.hadoop.fs.Path;
+
+public class PathArgumentConverter extends BaseConverter<Path> {
+
+ public PathArgumentConverter(String optionName) {
+ super(optionName);
+ }
+
+ @Override
+ public Path convert(String value) {
+ return new Path(value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
new file mode 100644
index 0000000..1c38213
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.yarn.service.client.params;
+
+/**
+ * Parameters sent by the Client to the AM
+ */
+public class SliderAMArgs extends CommonArgs {
+
+ SliderAMCreateAction createAction = new SliderAMCreateAction();
+
+ public SliderAMArgs(String[] args) {
+ super(args);
+ }
+
+ @Override
+ protected void addActionArguments() {
+ addActions(createAction);
+ }
+
+ public String getImage() {
+ return createAction.image;
+ }
+
+ /**
+ * This is the URI in the FS to the Slider cluster; the conf file (and any
+ * other cluster-specifics) can be picked up here
+ */
+ public String getAppDefPath() {
+ return createAction.sliderClusterURI;
+ }
+
+ /**
+ * Am binding is simple: there is only one action
+ */
+ @Override
+ public void applyAction() {
+ bindCoreAction(createAction);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
new file mode 100644
index 0000000..a446665
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+import java.io.File;
+
+
+@Parameters(commandNames = { SliderActions.ACTION_CREATE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
+
+public class SliderAMCreateAction extends AbstractActionArgs implements
+ LaunchArgsAccessor {
+
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_CREATE;
+ }
+
+ @Parameter(names = ARG_IMAGE, description = "image", required = false)
+ public String image;
+
+ /**
+ * This is the URI in the FS to the Slider cluster; the conf file (and any
+ * other cluster-specifics) can be picked up here
+ */
+ @Parameter(names = ARG_CLUSTER_URI,
+ description = "URI to the Slider cluster", required = true)
+ public String sliderClusterURI;
+
+ @ParametersDelegate LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
+
+ @Override
+ public String getRmAddress() {
+ return launchArgs.getRmAddress();
+ }
+
+ @Override
+ public int getWaittime() {
+ return launchArgs.getWaittime();
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ launchArgs.setWaittime(waittime);
+ }
+
+ @Override
+ public File getOutputFile() {
+ return launchArgs.getOutputFile();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
new file mode 100644
index 0000000..1b2a92d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+/**
+ * Actions.
+ * Only some of these are supported by specific Slider Services; they
+ * are listed here to ensure the names are consistent
+ */
+public interface SliderActions {
+ String ACTION_BUILD = "build";
+ String ACTION_CLIENT = "client";
+ String ACTION_CREATE = "create";
+ String ACTION_DEPENDENCY = "dependency";
+ String ACTION_UPDATE = "update";
+ String ACTION_UPGRADE = "upgrade";
+ String ACTION_DESTROY = "destroy";
+ String ACTION_EXISTS = "exists";
+ String ACTION_FLEX = "flex";
+ String ACTION_STOP = "stop";
+ String ACTION_HELP = "help";
+ String ACTION_INSTALL_KEYTAB = "install-keytab";
+ String ACTION_KDIAG = "kdiag";
+ String ACTION_KEYTAB = "keytab";
+ String ACTION_LIST = "list";
+
+ String ACTION_REGISTRY = "registry";
+ String ACTION_RESOLVE = "resolve";
+ String ACTION_RESOURCE = "resource";
+ String ACTION_STATUS = "status";
+ String ACTION_START = "start";
+ String ACTION_TOKENS = "tokens";
+
+ String DESCRIBE_ACTION_BUILD =
+ "Build a service specification, but do not start it";
+ String DESCRIBE_ACTION_CREATE =
+ "Build and start a service, it's equivalent to first invoke build and then start";
+ String DESCRIBE_ACTION_DEPENDENCY =
+ "Yarn service framework dependency (libraries) management";
+ String DESCRIBE_ACTION_UPDATE =
+ "Update template for service";
+ String DESCRIBE_ACTION_UPGRADE =
+ "Rolling upgrade/downgrade the component/containerto a newer/previous version";
+ String DESCRIBE_ACTION_DESTROY =
+ "Destroy a stopped service, service must be stopped first before destroying.";
+ String DESCRIBE_ACTION_EXISTS =
+ "Probe for a service running";
+ String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
+ String DESCRIBE_ACTION_FREEZE =
+ "Stop a running service";
+ String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems";
+ String DESCRIBE_ACTION_HELP = "Print help information";
+ String DESCRIBE_ACTION_LIST =
+ "List running services";
+ String DESCRIBE_ACTION_REGISTRY =
+ "Query the registry of a service";
+ String DESCRIBE_ACTION_STATUS =
+ "Get the status of a service";
+ String DESCRIBE_ACTION_THAW =
+ "Start a service with pre-built specification or a previously stopped service";
+ String DESCRIBE_ACTION_CLIENT = "Install the service client in the specified directory or obtain a client keystore or truststore";
+ String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
+ String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory";
+
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
new file mode 100644
index 0000000..86f3709
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+
+public class WaitArgsDelegate extends AbstractArgsDelegate implements
+ WaitTimeAccessor {
+
+
+ //--wait [timeout]
+ @Parameter(names = {ARG_WAIT},
+ description = "time to wait for an action to complete")
+ public int waittime = 0;
+
+
+ @Override
+ public int getWaittime() {
+ return waittime;
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ this.waittime = waittime;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
new file mode 100644
index 0000000..f6afae6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+public interface WaitTimeAccessor {
+ int getWaittime();
+ void setWaittime(int waittime);
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[57/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
deleted file mode 100644
index 8c968dc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ /dev/null
@@ -1,655 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.UpdatedContainer;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.service.component.ComponentEvent;
-import org.apache.hadoop.yarn.service.component.ComponentEventType;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
-import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
-import org.apache.hadoop.yarn.service.provider.ProviderUtils;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
-import org.apache.hadoop.yarn.util.BoundedAppender;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.nio.ByteBuffer;
-import java.text.MessageFormat;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
-import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
-import static org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants.*;
-import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
-
-/**
- *
- */
-public class ServiceScheduler extends CompositeService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceScheduler.class);
- private Application app;
-
- // component_name -> component
- private final Map<String, Component> componentsByName =
- new ConcurrentHashMap<>();
-
- // id - > component
- protected final Map<Long, Component> componentsById =
- new ConcurrentHashMap<>();
-
- private final Map<ContainerId, ComponentInstance> liveInstances =
- new ConcurrentHashMap<>();
-
- private ServiceMetrics serviceMetrics;
-
- private ServiceTimelinePublisher serviceTimelinePublisher;
-
- // Global diagnostics that will be reported to RM on eRxit.
- // The unit the number of characters. This will be limited to 64 * 1024
- // characters.
- private BoundedAppender diagnostics = new BoundedAppender(64 * 1024);
-
- // A cache for loading config files from remote such as hdfs
- public LoadingCache<ConfigFile, Object> configFileCache = null;
-
- public ScheduledExecutorService executorService;
- public Map<String, String> globalTokens = new HashMap<>();
-
- private AMRMClientAsync<AMRMClient.ContainerRequest> amRMClient;
- private NMClientAsync nmClient;
- private AsyncDispatcher dispatcher;
- AsyncDispatcher compInstanceDispatcher;
- private YarnRegistryViewForProviders yarnRegistryOperations;
- private ServiceContext context;
- private ContainerLaunchService containerLaunchService;
-
- public ServiceScheduler(ServiceContext context) {
- super(context.application.getName());
- this.context = context;
- }
-
- public void buildInstance(ServiceContext context, Configuration configuration)
- throws YarnException {
- app = context.application;
- executorService = Executors.newScheduledThreadPool(10);
- RegistryOperations registryClient = RegistryOperationsFactory
- .createInstance("ServiceScheduler", configuration);
- addIfService(registryClient);
- yarnRegistryOperations =
- createYarnRegistryOperations(context, registryClient);
-
- // register metrics
- serviceMetrics = ServiceMetrics
- .register(app.getName(), "Metrics for service");
- serviceMetrics.tag("type", "Metrics type [component or service]", "service");
- serviceMetrics.tag("appId", "Application id for service", app.getId());
-
- amRMClient = createAMRMClient();
- addIfService(amRMClient);
-
- nmClient = createNMClient();
- addIfService(nmClient);
-
- dispatcher = new AsyncDispatcher("Component dispatcher");
- dispatcher.register(ComponentEventType.class,
- new ComponentEventHandler());
- dispatcher.setDrainEventsOnStop();
- addIfService(dispatcher);
-
- compInstanceDispatcher =
- new AsyncDispatcher("CompInstance dispatcher");
- compInstanceDispatcher.register(ComponentInstanceEventType.class,
- new ComponentInstanceEventHandler());
- addIfService(compInstanceDispatcher);
- containerLaunchService = new ContainerLaunchService(context.fs);
- addService(containerLaunchService);
-
- if (YarnConfiguration.timelineServiceV2Enabled(configuration)) {
- TimelineV2Client timelineClient = TimelineV2Client
- .createTimelineClient(context.attemptId.getApplicationId());
- amRMClient.registerTimelineV2Client(timelineClient);
- serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
- addService(serviceTimelinePublisher);
- DefaultMetricsSystem.instance().register("ServiceMetricsSink",
- "For processing metrics to ATS",
- new ServiceMetricsSink(serviceTimelinePublisher));
- LOG.info("Timeline v2 is enabled.");
- }
-
- initGlobalTokensForSubstitute(context);
- //substitute quicklinks
- ProviderUtils.substituteMapWithTokens(app.getQuicklinks(), globalTokens);
- createConfigFileCache(context.fs.getFileSystem());
-
- createAllComponents();
- }
-
- protected YarnRegistryViewForProviders createYarnRegistryOperations(
- ServiceContext context, RegistryOperations registryClient) {
- return new YarnRegistryViewForProviders(registryClient,
- RegistryUtils.currentUser(), YarnServiceConstants.APP_TYPE, app.getName(),
- context.attemptId);
- }
-
- protected NMClientAsync createNMClient() {
- return NMClientAsync.createNMClientAsync(new NMClientCallback());
- }
-
- protected AMRMClientAsync<AMRMClient.ContainerRequest> createAMRMClient() {
- return AMRMClientAsync
- .createAMRMClientAsync(1000, new AMRMClientCallback());
- }
-
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- try {
- buildInstance(context, conf);
- } catch (YarnException e) {
- throw new YarnRuntimeException(e);
- }
- super.serviceInit(conf);
- }
-
- @Override
- public void serviceStop() throws Exception {
- LOG.info("Stopping service scheduler");
-
- if (executorService != null) {
- executorService.shutdownNow();
- }
-
- DefaultMetricsSystem.shutdown();
- if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
- serviceTimelinePublisher
- .serviceAttemptUnregistered(context, diagnostics.toString());
- }
- // Cleanup each component instance. no need to release containers as
- // they will be automatically released by RM
- for (ComponentInstance instance : liveInstances.values()) {
- instance.cleanupRegistryAndCompHdfsDir();
- }
- String msg = diagnostics.toString()
- + "Navigate to the failed component for more details.";
- amRMClient
- .unregisterApplicationMaster(FinalApplicationStatus.ENDED, msg, "");
- LOG.info("Application " + app.getName()
- + " unregistered with RM, with attemptId = " + context.attemptId
- + ", diagnostics = " + diagnostics);
- super.serviceStop();
- }
-
- @Override
- public void serviceStart() throws Exception {
- super.serviceStart();
- InetSocketAddress bindAddress = context.clientAMService.getBindAddress();
- RegisterApplicationMasterResponse response = amRMClient
- .registerApplicationMaster(bindAddress.getHostName(),
- bindAddress.getPort(), "N/A");
- if (response.getClientToAMTokenMasterKey() != null
- && response.getClientToAMTokenMasterKey().remaining() != 0) {
- context.secretManager
- .setMasterKey(response.getClientToAMTokenMasterKey().array());
- }
- registerServiceInstance(context.attemptId, app);
-
- //TODO handle containers recover
- }
-
- private void recover() {
-
- }
-
- private void initGlobalTokensForSubstitute(ServiceContext context) {
- // ZK
- globalTokens.put(ServiceApiConstants.CLUSTER_ZK_QUORUM, getConfig()
- .getTrimmed(KEY_REGISTRY_ZK_QUORUM, DEFAULT_REGISTRY_ZK_QUORUM));
- String user = null;
- try {
- user = UserGroupInformation.getCurrentUser().getShortUserName();
- } catch (IOException e) {
- LOG.error("Failed to get user.", e);
- }
- globalTokens
- .put(SERVICE_ZK_PATH, ServiceRegistryUtils.mkClusterPath(user, app.getName()));
-
- globalTokens.put(ServiceApiConstants.USER, user);
- String dnsDomain = getConfig().getTrimmed(KEY_DNS_DOMAIN);
- if (dnsDomain != null && !dnsDomain.isEmpty()) {
- globalTokens.put(ServiceApiConstants.DOMAIN, dnsDomain);
- }
- // HDFS
- String clusterFs = getConfig().getTrimmed(FS_DEFAULT_NAME_KEY);
- if (clusterFs != null && !clusterFs.isEmpty()) {
- globalTokens.put(ServiceApiConstants.CLUSTER_FS_URI, clusterFs);
- globalTokens.put(ServiceApiConstants.CLUSTER_FS_HOST,
- URI.create(clusterFs).getHost());
- }
- globalTokens.put(SERVICE_HDFS_DIR, context.serviceHdfsDir);
- // service name
- globalTokens.put(SERVICE_NAME_LC, app.getName().toLowerCase());
- globalTokens.put(SERVICE_NAME, app.getName());
- }
-
- private void createConfigFileCache(final FileSystem fileSystem) {
- this.configFileCache =
- CacheBuilder.newBuilder().expireAfterAccess(10, TimeUnit.MINUTES)
- .build(new CacheLoader<ConfigFile, Object>() {
- @Override public Object load(ConfigFile key) throws Exception {
- switch (key.getType()) {
- case HADOOP_XML:
- try (FSDataInputStream input = fileSystem
- .open(new Path(key.getSrcFile()))) {
- org.apache.hadoop.conf.Configuration confRead =
- new org.apache.hadoop.conf.Configuration(false);
- confRead.addResource(input);
- Map<String, String> map = new HashMap<>(confRead.size());
- for (Map.Entry<String, String> entry : confRead) {
- map.put(entry.getKey(), entry.getValue());
- }
- return map;
- }
- case TEMPLATE:
- try (FSDataInputStream fileInput = fileSystem
- .open(new Path(key.getSrcFile()))) {
- return IOUtils.toString(fileInput);
- }
- default:
- return null;
- }
- }
- });
- context.configCache = configFileCache;
- }
-
- private void registerServiceInstance(ApplicationAttemptId attemptId,
- Application application) throws IOException {
- LOG.info("Registering " + attemptId + ", " + application.getName()
- + " into registry");
- ServiceRecord serviceRecord = new ServiceRecord();
- serviceRecord.set(YarnRegistryAttributes.YARN_ID,
- attemptId.getApplicationId().toString());
- serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
- PersistencePolicies.APPLICATION);
- serviceRecord.description = "Yarn Service Master";
-
- serviceRecord.addExternalEndpoint(RegistryTypeUtils
- .ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc",
- context.clientAMService.getBindAddress()));
-
- // set any provided attributes
- setUserProvidedServiceRecordAttributes(application.getConfiguration(),
- serviceRecord);
-
- executorService.submit(new Runnable() {
- @Override public void run() {
- try {
- yarnRegistryOperations.registerSelf(serviceRecord, true);
- LOG.info("Registered service under {}; absolute path {}",
- yarnRegistryOperations.getSelfRegistrationPath(),
- yarnRegistryOperations.getAbsoluteSelfRegistrationPath());
- boolean isFirstAttempt = 1 == attemptId.getAttemptId();
- // delete the children in case there are any and this is an AM startup.
- // just to make sure everything underneath is purged
- if (isFirstAttempt) {
- yarnRegistryOperations.deleteChildren(
- yarnRegistryOperations.getSelfRegistrationPath(), true);
- }
- } catch (IOException e) {
- LOG.error(
- "Failed to register app " + app.getName() + " in registry");
- }
- }
- });
- if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
- serviceTimelinePublisher.serviceAttemptRegistered(app, getConfig());
- }
- }
-
- private void setUserProvidedServiceRecordAttributes(
- org.apache.hadoop.yarn.service.api.records.Configuration conf, ServiceRecord record) {
- String prefix = "service.record.attribute";
- for (Map.Entry<String, String> entry : conf.getProperties().entrySet()) {
- if (entry.getKey().startsWith(prefix)) {
- String key = entry.getKey().substring(prefix.length() + 1);
- record.set(key, entry.getValue().trim());
- }
- }
- }
-
- private void createAllComponents() {
- long allocateId = 0;
-
- // sort components by dependencies
- Collection<org.apache.hadoop.yarn.service.api.records.Component> sortedComponents =
- ServiceApiUtil.sortByDependencies(app.getComponents());
-
- for (org.apache.hadoop.yarn.service.api.records.Component compSpec : sortedComponents) {
- Component component = new Component(compSpec, allocateId, context);
- componentsById.put(allocateId, component);
- componentsByName.put(component.getName(), component);
- allocateId++;
-
- // Trigger the component without dependencies
- if (component.areDependenciesReady()) {
- ComponentEvent event = new ComponentEvent(compSpec.getName(), FLEX)
- .setDesired(compSpec.getNumberOfContainers());
- component.handle(event);
- }
- }
- }
-
- private final class ComponentEventHandler
- implements EventHandler<ComponentEvent> {
- @Override
- public void handle(ComponentEvent event) {
- Component component = componentsByName.get(event.getName());
-
- if (component == null) {
- LOG.error("No component exists for " + event.getName());
- return;
- }
- try {
- component.handle(event);
- } catch (Throwable t) {
- LOG.error(MessageFormat
- .format("[COMPONENT {0}]: Error in handling event type {1}",
- component.getName(), event.getType()), t);
- }
- }
- }
-
- private final class ComponentInstanceEventHandler
- implements EventHandler<ComponentInstanceEvent> {
- @Override
- public void handle(ComponentInstanceEvent event) {
- ComponentInstance instance =
- liveInstances.get(event.getContainerId());
- if (instance == null) {
- LOG.error("No component instance exists for " + event.getContainerId());
- return;
- }
- try {
- instance.handle(event);
- } catch (Throwable t) {
- LOG.error(instance.getCompInstanceId() +
- ": Error in handling event type " + event.getType(), t);
- }
- }
- }
-
- class AMRMClientCallback extends AMRMClientAsync.AbstractCallbackHandler {
-
- @Override
- public void onContainersAllocated(List<Container> containers) {
- LOG.info(containers.size() + " containers allocated. ");
- for (Container container : containers) {
- Component comp = componentsById.get(container.getAllocationRequestId());
- ComponentEvent event =
- new ComponentEvent(comp.getName(), CONTAINER_ALLOCATED)
- .setContainer(container);
- dispatcher.getEventHandler().handle(event);
- LOG.info("[COMPONENT {}]: {} outstanding container requests.",
- comp.getName(),
- amRMClient.getMatchingRequests(container.getAllocationRequestId()).size());
- // remove the corresponding request
- Collection<AMRMClient.ContainerRequest> collection = amRMClient
- .getMatchingRequests(container.getAllocationRequestId());
- if (collection.iterator().hasNext()) {
- AMRMClient.ContainerRequest request = collection.iterator().next();
- amRMClient.removeContainerRequest(request);
- }
-
- }
- }
-
- @Override
- public void onContainersCompleted(List<ContainerStatus> statuses) {
- for (ContainerStatus status : statuses) {
- ContainerId containerId = status.getContainerId();
- ComponentInstance instance = liveInstances.get(status.getContainerId());
- if (instance == null) {
- LOG.error(
- "Container {} Completed. No component instance exists. exitStatus={}. diagnostics={} ",
- containerId, status.getExitStatus(), status.getDiagnostics());
- return;
- }
- ComponentEvent event =
- new ComponentEvent(instance.getCompName(), CONTAINER_COMPLETED)
- .setStatus(status).setInstance(instance);
- dispatcher.getEventHandler().handle(event);
- }
- }
-
- @Override
- public void onContainersUpdated(List<UpdatedContainer> containers) {
- }
-
- @Override public void onShutdownRequest() {
- //Was used for non-work-preserving restart in YARN, should be deprecated.
- }
-
- @Override public void onNodesUpdated(List<NodeReport> updatedNodes) {
- StringBuilder str = new StringBuilder();
- str.append("Nodes updated info: ").append(System.lineSeparator());
- for (NodeReport report : updatedNodes) {
- str.append(report.getNodeId()).append(", state = ")
- .append(report.getNodeState()).append(", healthDiagnostics = ")
- .append(report.getHealthReport()).append(System.lineSeparator());
- }
- LOG.warn(str.toString());
- }
-
- @Override public float getProgress() {
- // get running containers over desired containers
- long total = 0;
- for (org.apache.hadoop.yarn.service.api.records.Component component : app
- .getComponents()) {
- total += component.getNumberOfContainers();
- }
- // Probably due to user flexed down to 0
- if (total == 0) {
- return 100;
- }
- return Math.max((float) liveInstances.size() / total * 100, 100);
- }
-
- @Override public void onError(Throwable e) {
- LOG.error("Error in AMRMClient callback handler ", e);
- }
- }
-
-
- private class NMClientCallback extends NMClientAsync.AbstractCallbackHandler {
-
- @Override public void onContainerStarted(ContainerId containerId,
- Map<String, ByteBuffer> allServiceResponse) {
- ComponentInstance instance = liveInstances.get(containerId);
- if (instance == null) {
- LOG.error("No component instance exists for " + containerId);
- return;
- }
- ComponentEvent event =
- new ComponentEvent(instance.getCompName(), CONTAINER_STARTED)
- .setInstance(instance);
- dispatcher.getEventHandler().handle(event);
- }
-
- @Override public void onContainerStatusReceived(ContainerId containerId,
- ContainerStatus containerStatus) {
-
- }
-
- @Override public void onContainerStopped(ContainerId containerId) {
-
- }
-
- @Override
- public void onStartContainerError(ContainerId containerId, Throwable t) {
- ComponentInstance instance = liveInstances.get(containerId);
- if (instance == null) {
- LOG.error("No component instance exists for " + containerId);
- return;
- }
- amRMClient.releaseAssignedContainer(containerId);
- // After container released, it'll get CONTAINER_COMPLETED event from RM
- // automatically which will trigger stopping COMPONENT INSTANCE
- }
-
- @Override public void onContainerResourceIncreased(ContainerId containerId,
- Resource resource) {
-
- }
-
- @Override public void onGetContainerStatusError(ContainerId containerId,
- Throwable t) {
-
- }
-
- @Override
- public void onIncreaseContainerResourceError(ContainerId containerId,
- Throwable t) {
-
- }
-
- @Override
- public void onStopContainerError(ContainerId containerId, Throwable t) {
-
- }
- }
-
- public ServiceMetrics getServiceMetrics() {
- return serviceMetrics;
- }
-
- public AMRMClientAsync<AMRMClient.ContainerRequest> getAmRMClient() {
- return amRMClient;
- }
-
- public NMClientAsync getNmClient() {
- return nmClient;
- }
-
- public void addLiveCompInstance(ContainerId containerId,
- ComponentInstance instance) {
- liveInstances.put(containerId, instance);
- }
-
- public void removeLiveCompInstance(ContainerId containerId) {
- liveInstances.remove(containerId);
- }
-
- public AsyncDispatcher getCompInstanceDispatcher() {
- return compInstanceDispatcher;
- }
-
- public YarnRegistryViewForProviders getYarnRegistryOperations() {
- return yarnRegistryOperations;
- }
-
- public ServiceTimelinePublisher getServiceTimelinePublisher() {
- return serviceTimelinePublisher;
- }
-
- public Map<ContainerId, ComponentInstance> getLiveInstances() {
- return liveInstances;
- }
-
- public ContainerLaunchService getContainerLaunchService() {
- return containerLaunchService;
- }
-
- public ServiceContext getContext() {
- return context;
- }
-
- public Map<String, Component> getAllComponents() {
- return componentsByName;
- }
-
- public Application getApp() {
- return app;
- }
-
- public AsyncDispatcher getDispatcher() {
- return dispatcher;
- }
-
- public BoundedAppender getDiagnostics() {
- return diagnostics;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java
deleted file mode 100644
index cf9e31f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.constants;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
-
-/**
- * This class defines constants that can be used in input spec for
- * variable substitutions
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public interface ServiceApiConstants {
-
- // Constants for service
- String SERVICE_NAME = $("SERVICE_NAME");
-
- String SERVICE_NAME_LC = $("SERVICE_NAME.lc");
-
- String USER = $("USER");
-
- String DOMAIN = $("DOMAIN");
-
- // Constants for component
- String COMPONENT_NAME = $("COMPONENT_NAME");
-
- String COMPONENT_NAME_LC = $("COMPONENT_NAME.lc");
-
- String COMPONENT_INSTANCE_NAME = $("COMPONENT_INSTANCE_NAME");
-
- // Constants for component instance
- String COMPONENT_ID = $("COMPONENT_ID");
-
- String CONTAINER_ID = $("CONTAINER_ID");
-
- // Constants for default cluster ZK
- String CLUSTER_ZK_QUORUM = $("CLUSTER_ZK_QUORUM");
-
- // URI for the default cluster fs
- String CLUSTER_FS_URI = $("CLUSTER_FS_URI");
-
- // the host component of the cluster fs UI
- String CLUSTER_FS_HOST = $("CLUSTER_FS_HOST");
-
- // Path in zookeeper for a specific service
- String SERVICE_ZK_PATH = $("SERVICE_ZK_PATH");
-
- // Constants for service specific hdfs dir
- String SERVICE_HDFS_DIR = $("SERVICE_HDFS_DIR");
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java
deleted file mode 100644
index f9e5154..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java
+++ /dev/null
@@ -1,466 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonPropertyOrder;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * An Application resource has the following attributes.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "An Application resource has the following attributes.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@XmlRootElement
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonPropertyOrder({ "name", "state", "resource", "number_of_containers",
- "lifetime", "containers" })
-public class Application extends BaseResource {
- private static final long serialVersionUID = -4491694636566094885L;
-
- private String name = null;
- private String id = null;
- private Artifact artifact = null;
- private Resource resource = null;
- private String launchCommand = null;
- private Date launchTime = null;
- private Long numberOfContainers = null;
- private Long numberOfRunningContainers = null;
- private Long lifetime = null;
- private PlacementPolicy placementPolicy = null;
- private List<Component> components = new ArrayList<>();
- private Configuration configuration = new Configuration();
- private List<Container> containers = new ArrayList<>();
- private ApplicationState state = null;
- private Map<String, String> quicklinks = new HashMap<>();
- private String queue = null;
-
- /**
- * A unique application name.
- **/
- public Application name(String name) {
- this.name = name;
- return this;
- }
-
- @ApiModelProperty(example = "null", required = true, value = "A unique application name.")
- @JsonProperty("name")
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- /**
- * A unique application id.
- **/
- public Application id(String id) {
- this.id = id;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A unique application id.")
- @JsonProperty("id")
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- /**
- * Artifact of single-component applications. Mandatory if components
- * attribute is not specified.
- **/
- public Application artifact(Artifact artifact) {
- this.artifact = artifact;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact of single-component applications. Mandatory if components attribute is not specified.")
- @JsonProperty("artifact")
- public Artifact getArtifact() {
- return artifact;
- }
-
- public void setArtifact(Artifact artifact) {
- this.artifact = artifact;
- }
-
- /**
- * Resource of single-component applications or the global default for
- * multi-component applications. Mandatory if it is a single-component
- * application and if cpus and memory are not specified at the Application
- * level.
- **/
- public Application resource(Resource resource) {
- this.resource = resource;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Resource of single-component applications or the global default for multi-component applications. Mandatory if it is a single-component application and if cpus and memory are not specified at the Application level.")
- @JsonProperty("resource")
- public Resource getResource() {
- return resource;
- }
-
- public void setResource(Resource resource) {
- this.resource = resource;
- }
-
- /**
- * The custom launch command of an application component (optional). If not
- * specified for applications with docker images say, it will default to the
- * default start command of the image. If there is a single component in this
- * application, you can specify this without the need to have a 'components'
- * section.
- **/
- public Application launchCommand(String launchCommand) {
- this.launchCommand = launchCommand;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The custom launch command of an application component (optional). If not specified for applications with docker images say, it will default to the default start command of the image. If there is a single component in this application, you can specify this without the need to have a 'components' section.")
- @JsonProperty("launch_command")
- public String getLaunchCommand() {
- return launchCommand;
- }
-
- @XmlElement(name = "launch_command")
- public void setLaunchCommand(String launchCommand) {
- this.launchCommand = launchCommand;
- }
-
- /**
- * The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.
- **/
- public Application launchTime(Date launchTime) {
- this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.")
- @JsonProperty("launch_time")
- public Date getLaunchTime() {
- return launchTime == null ? null : (Date) launchTime.clone();
- }
-
- @XmlElement(name = "launch_time")
- public void setLaunchTime(Date launchTime) {
- this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
- }
-
- /**
- * Number of containers for each app-component in the application. Each
- * app-component can further override this app-level global default.
- **/
- public Application numberOfContainers(Long numberOfContainers) {
- this.numberOfContainers = numberOfContainers;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Number of containers for each app-component in the application. Each app-component can further override this app-level global default.")
- @JsonProperty("number_of_containers")
- public Long getNumberOfContainers() {
- return numberOfContainers;
- }
-
- @XmlElement(name = "number_of_containers")
- public void setNumberOfContainers(Long numberOfContainers) {
- this.numberOfContainers = numberOfContainers;
- }
-
- /**
- * In get response this provides the total number of running containers for
- * this application (across all components) at the time of request. Note, a
- * subsequent request can return a different number as and when more
- * containers get allocated until it reaches the total number of containers or
- * if a flex request has been made between the two requests.
- **/
- public Application numberOfRunningContainers(Long numberOfRunningContainers) {
- this.numberOfRunningContainers = numberOfRunningContainers;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "In get response this provides the total number of running containers for this application (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.")
- @JsonProperty("number_of_running_containers")
- public Long getNumberOfRunningContainers() {
- return numberOfRunningContainers;
- }
-
- @XmlElement(name = "number_of_running_containers")
- public void setNumberOfRunningContainers(Long numberOfRunningContainers) {
- this.numberOfRunningContainers = numberOfRunningContainers;
- }
-
- /**
- * Life time (in seconds) of the application from the time it reaches the
- * RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For
- * unlimited lifetime do not set a lifetime value.
- **/
- public Application lifetime(Long lifetime) {
- this.lifetime = lifetime;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Life time (in seconds) of the application from the time it reaches the RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.")
- @JsonProperty("lifetime")
- public Long getLifetime() {
- return lifetime;
- }
-
- public void setLifetime(Long lifetime) {
- this.lifetime = lifetime;
- }
-
- /**
- * Advanced scheduling and placement policies (optional). If not specified, it
- * defaults to the default placement policy of the app owner. The design of
- * placement policies are in the works. It is not very clear at this point,
- * how policies in conjunction with labels be exposed to application owners.
- * This is a placeholder for now. The advanced structure of this attribute
- * will be determined by YARN-4902.
- **/
- public Application placementPolicy(PlacementPolicy placementPolicy) {
- this.placementPolicy = placementPolicy;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the app owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to application owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.")
- @JsonProperty("placement_policy")
- public PlacementPolicy getPlacementPolicy() {
- return placementPolicy;
- }
-
- @XmlElement(name = "placement_policy")
- public void setPlacementPolicy(PlacementPolicy placementPolicy) {
- this.placementPolicy = placementPolicy;
- }
-
- /**
- * Components of an application.
- **/
- public Application components(List<Component> components) {
- this.components = components;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Components of an application.")
- @JsonProperty("components")
- public List<Component> getComponents() {
- return components;
- }
-
- public void setComponents(List<Component> components) {
- this.components = components;
- }
-
- public void addComponent(Component component) {
- components.add(component);
- }
-
- public Component getComponent(String name) {
- for (Component component : components) {
- if (component.getName().equals(name)) {
- return component;
- }
- }
- return null;
- }
-
- /**
- * Config properties of an application. Configurations provided at the
- * application/global level are available to all the components. Specific
- * properties can be overridden at the component level.
- **/
- public Application configuration(Configuration configuration) {
- this.configuration = configuration;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Config properties of an application. Configurations provided at the application/global level are available to all the components. Specific properties can be overridden at the component level.")
- @JsonProperty("configuration")
- public Configuration getConfiguration() {
- return configuration;
- }
-
- public void setConfiguration(Configuration configuration) {
- this.configuration = configuration;
- }
-
- /**
- * Containers of a started application. Specifying a value for this attribute
- * for the POST payload raises a validation error. This blob is available only
- * in the GET response of a started application.
- **/
- public Application containers(List<Container> containers) {
- this.containers = containers;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Containers of a started application. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.")
- @JsonProperty("containers")
- public List<Container> getContainers() {
- return containers;
- }
-
- public void setContainers(List<Container> containers) {
- this.containers = containers;
- }
-
- public void addContainer(Container container) {
- this.containers.add(container);
- }
-
- /**
- * State of the application. Specifying a value for this attribute for the
- * POST payload raises a validation error. This attribute is available only in
- * the GET response of a started application.
- **/
- public Application state(ApplicationState state) {
- this.state = state;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "State of the application. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started application.")
- @JsonProperty("state")
- public ApplicationState getState() {
- return state;
- }
-
- public void setState(ApplicationState state) {
- this.state = state;
- }
-
- /**
- * A blob of key-value pairs of quicklinks to be exported for an application.
- **/
- public Application quicklinks(Map<String, String> quicklinks) {
- this.quicklinks = quicklinks;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A blob of key-value pairs of quicklinks to be exported for an application.")
- @JsonProperty("quicklinks")
- public Map<String, String> getQuicklinks() {
- return quicklinks;
- }
-
- public void setQuicklinks(Map<String, String> quicklinks) {
- this.quicklinks = quicklinks;
- }
-
- /**
- * The YARN queue that this application should be submitted to.
- **/
- public Application queue(String queue) {
- this.queue = queue;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The YARN queue that this application should be submitted to.")
- @JsonProperty("queue")
- public String getQueue() {
- return queue;
- }
-
- public void setQueue(String queue) {
- this.queue = queue;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Application application = (Application) o;
- return Objects.equals(this.name, application.name);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(name);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Application {\n");
-
- sb.append(" name: ").append(toIndentedString(name)).append("\n");
- sb.append(" id: ").append(toIndentedString(id)).append("\n");
- sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
- sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
- sb.append(" launchCommand: ").append(toIndentedString(launchCommand))
- .append("\n");
- sb.append(" launchTime: ").append(toIndentedString(launchTime))
- .append("\n");
- sb.append(" numberOfContainers: ")
- .append(toIndentedString(numberOfContainers)).append("\n");
- sb.append(" numberOfRunningContainers: ")
- .append(toIndentedString(numberOfRunningContainers)).append("\n");
- sb.append(" lifetime: ").append(toIndentedString(lifetime)).append("\n");
- sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy))
- .append("\n");
- sb.append(" components: ").append(toIndentedString(components))
- .append("\n");
- sb.append(" configuration: ").append(toIndentedString(configuration))
- .append("\n");
- sb.append(" containers: ").append(toIndentedString(containers))
- .append("\n");
- sb.append(" state: ").append(toIndentedString(state)).append("\n");
- sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
- .append("\n");
- sb.append(" queue: ").append(toIndentedString(queue)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java
deleted file mode 100644
index acef562..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * The current state of an application.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "The current state of an application.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public enum ApplicationState {
- ACCEPTED, STARTED, READY, STOPPED, FAILED;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
deleted file mode 100644
index b57225a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.util.Objects;
-
-import javax.xml.bind.annotation.XmlRootElement;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * The current status of a submitted application, returned as a response to the
- * GET API.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "The current status of a submitted application, returned as a response to the GET API.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@XmlRootElement
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class ApplicationStatus extends BaseResource {
- private static final long serialVersionUID = -3469885905347851034L;
-
- private String diagnostics = null;
- private ApplicationState state = null;
- private Integer code = null;
-
- /**
- * Diagnostic information (if any) for the reason of the current state of the
- * application. It typically has a non-null value, if the application is in a
- * non-running state.
- **/
- public ApplicationStatus diagnostics(String diagnostics) {
- this.diagnostics = diagnostics;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Diagnostic information (if any) for the reason of the current state of the application. It typically has a non-null value, if the application is in a non-running state.")
- @JsonProperty("diagnostics")
- public String getDiagnostics() {
- return diagnostics;
- }
-
- public void setDiagnostics(String diagnostics) {
- this.diagnostics = diagnostics;
- }
-
- /**
- * Application state.
- **/
- public ApplicationStatus state(ApplicationState state) {
- this.state = state;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Application state.")
- @JsonProperty("state")
- public ApplicationState getState() {
- return state;
- }
-
- public void setState(ApplicationState state) {
- this.state = state;
- }
-
- /**
- * An error code specific to a scenario which app owners should be able to use
- * to understand the failure in addition to the diagnostic information.
- **/
- public ApplicationStatus code(Integer code) {
- this.code = code;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "An error code specific to a scenario which app owners should be able to use to understand the failure in addition to the diagnostic information.")
- @JsonProperty("code")
- public Integer getCode() {
- return code;
- }
-
- public void setCode(Integer code) {
- this.code = code;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- ApplicationStatus applicationStatus = (ApplicationStatus) o;
- return Objects.equals(this.diagnostics, applicationStatus.diagnostics)
- && Objects.equals(this.state, applicationStatus.state)
- && Objects.equals(this.code, applicationStatus.code);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(diagnostics, state, code);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class ApplicationStatus {\n");
-
- sb.append(" diagnostics: ").append(toIndentedString(diagnostics))
- .append("\n");
- sb.append(" state: ").append(toIndentedString(state)).append("\n");
- sb.append(" code: ").append(toIndentedString(code)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
deleted file mode 100644
index 0ddc374..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonValue;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Artifact of an application component.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "Artifact of an application component")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Artifact implements Serializable {
- private static final long serialVersionUID = 3608929500111099035L;
-
- private String id = null;
-
- public enum TypeEnum {
- DOCKER("DOCKER"), TARBALL("TARBALL"), APPLICATION("APPLICATION");
-
- private String value;
-
- TypeEnum(String value) {
- this.value = value;
- }
-
- @Override
- @JsonValue
- public String toString() {
- return value;
- }
- }
-
- private TypeEnum type = TypeEnum.DOCKER;
- private String uri = null;
-
- /**
- * Artifact id. Examples are package location uri for tarball based apps,
- * image name for docker, etc.
- **/
- public Artifact id(String id) {
- this.id = id;
- return this;
- }
-
- @ApiModelProperty(example = "null", required = true, value = "Artifact id. Examples are package location uri for tarball based apps, image name for docker, etc.")
- @JsonProperty("id")
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- /**
- * Artifact type, like docker, tarball, etc. (optional).
- **/
- public Artifact type(TypeEnum type) {
- this.type = type;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact type, like docker, tarball, etc. (optional).")
- @JsonProperty("type")
- public TypeEnum getType() {
- return type;
- }
-
- public void setType(TypeEnum type) {
- this.type = type;
- }
-
- /**
- * Artifact location to support multiple artifact stores (optional).
- **/
- public Artifact uri(String uri) {
- this.uri = uri;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact location to support multiple artifact stores (optional).")
- @JsonProperty("uri")
- public String getUri() {
- return uri;
- }
-
- public void setUri(String uri) {
- this.uri = uri;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Artifact artifact = (Artifact) o;
- return Objects.equals(this.id, artifact.id)
- && Objects.equals(this.type, artifact.type)
- && Objects.equals(this.uri, artifact.uri);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(id, type, uri);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Artifact {\n");
-
- sb.append(" id: ").append(toIndentedString(id)).append("\n");
- sb.append(" type: ").append(toIndentedString(type)).append("\n");
- sb.append(" uri: ").append(toIndentedString(uri)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
deleted file mode 100644
index a87c97f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import java.io.Serializable;
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class BaseResource implements Serializable {
- private static final long serialVersionUID = 1492603053176889431L;
-
- private String uri;
-
- /**
- * Resource location, e.g. \
- * "/applications/helloworld/containers/container_e3751_1458061340047_0008_01_000002\
- * "
- **/
- public String getUri() {
- return uri;
- }
-
- public void setUri(String uri) {
- this.uri = uri;
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append("BaseResource [uri=");
- builder.append(uri);
- builder.append("]");
- return builder.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
deleted file mode 100644
index 633e862..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
+++ /dev/null
@@ -1,412 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * One or more components of the application. If the application is HBase say,
- * then the component can be a simple role like master or regionserver. If the
- * application is a complex business webapp then a component can be other
- * applications say Kafka or Storm. Thereby it opens up the support for complex
- * and nested applications.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "One or more components of the application. If the application is HBase say, then the component can be a simple role like master or regionserver. If the application is a complex business webapp then a component can be other applications say Kafka or Storm. Thereby it opens up the support for complex and nested applications.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@XmlRootElement
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Component implements Serializable {
- private static final long serialVersionUID = -8430058381509087805L;
-
- private String name = null;
- private List<String> dependencies = new ArrayList<String>();
- private ReadinessCheck readinessCheck = null;
- private Artifact artifact = null;
- private String launchCommand = null;
- private Resource resource = null;
- private Long numberOfContainers = null;
- private Boolean runPrivilegedContainer = false;
- private PlacementPolicy placementPolicy = null;
- private Configuration configuration = new Configuration();
- private List<String> quicklinks = new ArrayList<String>();
- private List<Container> containers =
- Collections.synchronizedList(new ArrayList<Container>());
-
- /**
- * Name of the application component (mandatory).
- **/
- public Component name(String name) {
- this.name = name;
- return this;
- }
-
- @ApiModelProperty(example = "null", required = true, value = "Name of the application component (mandatory).")
- @JsonProperty("name")
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- /**
- * An array of application components which should be in READY state (as
- * defined by readiness check), before this component can be started. The
- * dependencies across all components of an application should be represented
- * as a DAG.
- **/
- public Component dependencies(List<String> dependencies) {
- this.dependencies = dependencies;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "An array of application components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an application should be represented as a DAG.")
- @JsonProperty("dependencies")
- public List<String> getDependencies() {
- return dependencies;
- }
-
- public void setDependencies(List<String> dependencies) {
- this.dependencies = dependencies;
- }
-
- /**
- * Readiness check for this app-component.
- **/
- public Component readinessCheck(ReadinessCheck readinessCheck) {
- this.readinessCheck = readinessCheck;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Readiness check for this app-component.")
- @JsonProperty("readiness_check")
- public ReadinessCheck getReadinessCheck() {
- return readinessCheck;
- }
-
- @XmlElement(name = "readiness_check")
- public void setReadinessCheck(ReadinessCheck readinessCheck) {
- this.readinessCheck = readinessCheck;
- }
-
- /**
- * Artifact of the component (optional). If not specified, the application
- * level global artifact takes effect.
- **/
- public Component artifact(Artifact artifact) {
- this.artifact = artifact;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact of the component (optional). If not specified, the application level global artifact takes effect.")
- @JsonProperty("artifact")
- public Artifact getArtifact() {
- return artifact;
- }
-
- public void setArtifact(Artifact artifact) {
- this.artifact = artifact;
- }
-
- /**
- * The custom launch command of this component (optional). When specified at
- * the component level, it overrides the value specified at the global level
- * (if any).
- **/
- public Component launchCommand(String launchCommand) {
- this.launchCommand = launchCommand;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The custom launch command of this component (optional). When specified at the component level, it overrides the value specified at the global level (if any).")
- @JsonProperty("launch_command")
- public String getLaunchCommand() {
- return launchCommand;
- }
-
- @XmlElement(name = "launch_command")
- public void setLaunchCommand(String launchCommand) {
- this.launchCommand = launchCommand;
- }
-
- /**
- * Resource of this component (optional). If not specified, the application
- * level global resource takes effect.
- **/
- public Component resource(Resource resource) {
- this.resource = resource;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Resource of this component (optional). If not specified, the application level global resource takes effect.")
- @JsonProperty("resource")
- public Resource getResource() {
- return resource;
- }
-
- public void setResource(Resource resource) {
- this.resource = resource;
- }
-
- /**
- * Number of containers for this app-component (optional). If not specified,
- * the application level global number_of_containers takes effect.
- **/
- public Component numberOfContainers(Long numberOfContainers) {
- this.numberOfContainers = numberOfContainers;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Number of containers for this app-component (optional). If not specified, the application level global number_of_containers takes effect.")
- @JsonProperty("number_of_containers")
- public Long getNumberOfContainers() {
- return numberOfContainers;
- }
-
- @XmlElement(name = "number_of_containers")
- public void setNumberOfContainers(Long numberOfContainers) {
- this.numberOfContainers = numberOfContainers;
- }
-
- @ApiModelProperty(example = "null", value = "Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.")
- @JsonProperty("containers")
- public List<Container> getContainers() {
- return containers;
- }
-
- public void setContainers(List<Container> containers) {
- this.containers = containers;
- }
-
- public void addContainer(Container container) {
- this.containers.add(container);
- }
-
- public void removeContainer(Container container) {
- containers.remove(container);
- }
- public Container getContainer(String id) {
- for (Container container : containers) {
- if (container.getId().equals(id)) {
- return container;
- }
- }
- return null;
- }
-
- /**
- * Run all containers of this component in privileged mode (YARN-4262).
- **/
- public Component runPrivilegedContainer(Boolean runPrivilegedContainer) {
- this.runPrivilegedContainer = runPrivilegedContainer;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Run all containers of this component in privileged mode (YARN-4262).")
- @JsonProperty("run_privileged_container")
- public Boolean getRunPrivilegedContainer() {
- return runPrivilegedContainer;
- }
-
- @XmlElement(name = "run_privileged_container")
- public void setRunPrivilegedContainer(Boolean runPrivilegedContainer) {
- this.runPrivilegedContainer = runPrivilegedContainer;
- }
-
- /**
- * Advanced scheduling and placement policies for all containers of this
- * component (optional). If not specified, the app level placement_policy
- * takes effect. Refer to the description at the global level for more
- * details.
- **/
- public Component placementPolicy(PlacementPolicy placementPolicy) {
- this.placementPolicy = placementPolicy;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the app level placement_policy takes effect. Refer to the description at the global level for more details.")
- @JsonProperty("placement_policy")
- public PlacementPolicy getPlacementPolicy() {
- return placementPolicy;
- }
-
- @XmlElement(name = "placement_policy")
- public void setPlacementPolicy(PlacementPolicy placementPolicy) {
- this.placementPolicy = placementPolicy;
- }
-
- /**
- * Config properties for this app-component.
- **/
- public Component configuration(Configuration configuration) {
- this.configuration = configuration;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Config properties for this app-component.")
- @JsonProperty("configuration")
- public Configuration getConfiguration() {
- return configuration;
- }
-
- public void setConfiguration(Configuration configuration) {
- this.configuration = configuration;
- }
-
- /**
- * A list of quicklink keys defined at the application level, and to be
- * resolved by this component.
- **/
- public Component quicklinks(List<String> quicklinks) {
- this.quicklinks = quicklinks;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A list of quicklink keys defined at the application level, and to be resolved by this component.")
- @JsonProperty("quicklinks")
- public List<String> getQuicklinks() {
- return quicklinks;
- }
-
- public void setQuicklinks(List<String> quicklinks) {
- this.quicklinks = quicklinks;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Component component = (Component) o;
- return Objects.equals(this.name, component.name)
- && Objects.equals(this.dependencies, component.dependencies)
- && Objects.equals(this.readinessCheck, component.readinessCheck)
- && Objects.equals(this.artifact, component.artifact)
- && Objects.equals(this.launchCommand, component.launchCommand)
- && Objects.equals(this.resource, component.resource)
- && Objects.equals(this.numberOfContainers, component.numberOfContainers)
- && Objects.equals(this.runPrivilegedContainer,
- component.runPrivilegedContainer)
- && Objects.equals(this.placementPolicy, component.placementPolicy)
- && Objects.equals(this.configuration, component.configuration)
- && Objects.equals(this.quicklinks, component.quicklinks);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(name, dependencies, readinessCheck, artifact,
- launchCommand, resource, numberOfContainers,
- runPrivilegedContainer, placementPolicy, configuration, quicklinks);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Component {\n");
-
- sb.append(" name: ").append(toIndentedString(name)).append("\n");
- sb.append(" dependencies: ").append(toIndentedString(dependencies))
- .append("\n");
- sb.append(" readinessCheck: ").append(toIndentedString(readinessCheck))
- .append("\n");
- sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
- sb.append(" launchCommand: ").append(toIndentedString(launchCommand))
- .append("\n");
- sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
- sb.append(" numberOfContainers: ")
- .append(toIndentedString(numberOfContainers)).append("\n");
- sb.append(" containers: ").append(toIndentedString(containers))
- .append("\n");
- sb.append(" runPrivilegedContainer: ")
- .append(toIndentedString(runPrivilegedContainer)).append("\n");
- sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy))
- .append("\n");
- sb.append(" configuration: ").append(toIndentedString(configuration))
- .append("\n");
- sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
- .append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-
- /**
- * Merge from another component into this component without overwriting.
- */
- public void mergeFrom(Component that) {
- if (this.getArtifact() == null) {
- this.setArtifact(that.getArtifact());
- }
- if (this.getResource() == null) {
- this.setResource(that.getResource());
- }
- if (this.getNumberOfContainers() == null) {
- this.setNumberOfContainers(that.getNumberOfContainers());
- }
- if (this.getLaunchCommand() == null) {
- this.setLaunchCommand(that.getLaunchCommand());
- }
- this.getConfiguration().mergeFrom(that.getConfiguration());
- if (this.getQuicklinks() == null) {
- this.setQuicklinks(that.getQuicklinks());
- }
- if (this.getRunPrivilegedContainer() == null) {
- this.setRunPrivilegedContainer(that.getRunPrivilegedContainer());
- }
- if (this.getDependencies() == null) {
- this.setDependencies(that.getDependencies());
- }
- if (this.getPlacementPolicy() == null) {
- this.setPlacementPolicy(that.getPlacementPolicy());
- }
- if (this.getReadinessCheck() == null) {
- this.setReadinessCheck(that.getReadinessCheck());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
deleted file mode 100644
index 2fb494e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonValue;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * A config file that needs to be created and made available as a volume in an
- * application component container.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "A config file that needs to be created and made available as a volume in an application component container.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@XmlRootElement
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class ConfigFile implements Serializable {
- private static final long serialVersionUID = -7009402089417704612L;
-
- public enum TypeEnum {
- XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
- "TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),;
-
- private String value;
-
- TypeEnum(String value) {
- this.value = value;
- }
-
- @Override
- @JsonValue
- public String toString() {
- return value;
- }
- }
-
- private TypeEnum type = null;
- private String destFile = null;
- private String srcFile = null;
- private Map<String, String> props = new HashMap<>();
-
- public ConfigFile copy() {
- ConfigFile copy = new ConfigFile();
- copy.setType(this.getType());
- copy.setSrcFile(this.getSrcFile());
- copy.setDestFile(this.getDestFile());
- if (this.getProps() != null && !this.getProps().isEmpty()) {
- copy.getProps().putAll(this.getProps());
- }
- return copy;
- }
-
- /**
- * Config file in the standard format like xml, properties, json, yaml,
- * template.
- **/
- public ConfigFile type(TypeEnum type) {
- this.type = type;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Config file in the standard format like xml, properties, json, yaml, template.")
- @JsonProperty("type")
- public TypeEnum getType() {
- return type;
- }
-
- public void setType(TypeEnum type) {
- this.type = type;
- }
-
- /**
- * The absolute path that this configuration file should be mounted as, in the
- * application container.
- **/
- public ConfigFile destFile(String destFile) {
- this.destFile = destFile;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The absolute path that this configuration file should be mounted as, in the application container.")
- @JsonProperty("dest_file")
- public String getDestFile() {
- return destFile;
- }
-
- @XmlElement(name = "dest_file")
- public void setDestFile(String destFile) {
- this.destFile = destFile;
- }
-
- /**
- * This provides the source location of the configuration file, the content
- * of which is dumped to dest_file post property substitutions, in the format
- * as specified in type. Typically the src_file would point to a source
- * controlled network accessible file maintained by tools like puppet, chef,
- * or hdfs etc. Currently, only hdfs is supported.
- **/
- public ConfigFile srcFile(String srcFile) {
- this.srcFile = srcFile;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "This provides the source location of the configuration file, "
- + "the content of which is dumped to dest_file post property substitutions, in the format as specified in type. "
- + "Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.")
- @JsonProperty("src_file")
- public String getSrcFile() {
- return srcFile;
- }
-
- @XmlElement(name = "src_file")
- public void setSrcFile(String srcFile) {
- this.srcFile = srcFile;
- }
-
- /**
- A blob of key value pairs that will be dumped in the dest_file in the format
- as specified in type. If src_file is specified, src_file content are dumped
- in the dest_file and these properties will overwrite, if any, existing
- properties in src_file or be added as new properties in src_file.
- **/
- public ConfigFile props(Map<String, String> props) {
- this.props = props;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be dumped in the dest_file in the format as specified in type."
- + " If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any,"
- + " existing properties in src_file or be added as new properties in src_file.")
- @JsonProperty("props")
- public Map<String, String> getProps() {
- return props;
- }
-
- public void setProps(Map<String, String> props) {
- this.props = props;
- }
-
- public long getLong(String name, long defaultValue) {
- if (name == null) {
- return defaultValue;
- }
- String value = props.get(name.trim());
- return Long.parseLong(value);
- }
-
- public boolean getBoolean(String name, boolean defaultValue) {
- if (name == null) {
- return defaultValue;
- }
- return Boolean.valueOf(props.get(name.trim()));
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- ConfigFile configFile = (ConfigFile) o;
- return Objects.equals(this.type, configFile.type)
- && Objects.equals(this.destFile, configFile.destFile)
- && Objects.equals(this.srcFile, configFile.srcFile);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(type, destFile, srcFile, props);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class ConfigFile {\n");
-
- sb.append(" type: ").append(toIndentedString(type)).append("\n");
- sb.append(" destFile: ").append(toIndentedString(destFile)).append("\n");
- sb.append(" srcFile: ").append(toIndentedString(srcFile)).append("\n");
- sb.append(" props: ").append(toIndentedString(props)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
deleted file mode 100644
index 9632265..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
+++ /dev/null
@@ -1,539 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderInternalStateException;
-import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.core.main.LauncherExitCodes;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.state.ContainerOutcome;
-import org.apache.slider.server.appmaster.state.NodeEntry;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.NodeMap;
-import org.apache.slider.server.appmaster.state.ProviderAppState;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.slider.utils.SliderTestBase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map.Entry;
-
-/**
- * Base for app state tests.
- */
-public abstract class BaseMockAppStateTest extends SliderTestBase implements
- MockRoles {
- private static final Logger LOG =
- LoggerFactory.getLogger(BaseMockAppStateTest.class);
- protected static final List<ContainerId> EMPTY_ID_LIST = Collections
- .emptyList();
-
- protected final MockFactory factory = MockFactory.INSTANCE;
- protected MockAppState appState;
- protected MockYarnEngine engine;
- protected FileSystem fs;
- protected SliderFileSystem sliderFileSystem;
- protected File historyWorkDir;
- protected Path historyPath;
- protected MockApplicationId applicationId;
- protected MockApplicationAttemptId applicationAttemptId;
- protected StateAccessForProviders stateAccess;
-
- /**
- * Override point: called in setup() to create the YARN engine; can
- * be changed for different sizes and options.
- * @return
- */
- public MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(8, 8);
- }
-
- @Override
- public void setup() throws Exception {
- super.setup();
- YarnConfiguration conf = SliderUtils.createConfiguration();
- fs = FileSystem.get(new URI("file:///"), conf);
- sliderFileSystem = new SliderFileSystem(fs, conf);
- engine = createYarnEngine();
- initApp();
- }
-
- /**
- * Initialize the application.
- * This uses the binding information supplied by {@link #buildBindingInfo()}.
- */
- protected void initApp()
- throws IOException, BadConfigException, BadClusterStateException {
- String historyDirName = getTestName();
- applicationId = new MockApplicationId(1, 0);
- applicationAttemptId = new MockApplicationAttemptId(applicationId, 1);
-
- historyWorkDir = new File("target/history", historyDirName);
- historyPath = new Path(historyWorkDir.toURI());
- fs.delete(historyPath, true);
- appState = new MockAppState(buildBindingInfo());
- stateAccess = new ProviderAppState(getValidTestName(), appState);
- }
-
- /**
- * Build the binding info from the default constructor values,
- * the roles from {@link #factory}, and an instance definition.
- * from {@link #buildApplication()} ()}
- * @return
- */
- protected AppStateBindingInfo buildBindingInfo() throws IOException {
- AppStateBindingInfo binding = new AppStateBindingInfo();
- binding.application = buildApplication();
- ServiceApiUtil.validateAndResolveApplication(binding.application,
- sliderFileSystem, SliderUtils.createConfiguration());
- //binding.roles = new ArrayList<>(factory.ROLES);
- binding.fs = fs;
- binding.historyPath = historyPath;
- binding.nodeReports = engine.getNodeReports();
- return binding;
- }
-
- /**
- * Override point, define the instance definition.
- * @return the instance definition
- */
- public Application buildApplication() {
- return factory.newApplication(0, 0, 0).name(getValidTestName());
- }
-
- /**
- * Get the test name ... defaults to method name
- * @return the method name
- */
- public String getTestName() {
- return methodName.getMethodName();
- }
-
- public String getValidTestName() {
- return getTestName().toLowerCase(Locale.ENGLISH);
- }
-
- public RoleStatus getRole0Status() {
- return lookupRole(ROLE0);
- }
-
- public RoleStatus lookupRole(String role) {
- return appState.lookupRoleStatus(role);
- }
-
- public RoleStatus getRole1Status() {
- return lookupRole(ROLE1);
- }
-
- public RoleStatus getRole2Status() {
- return lookupRole(ROLE2);
- }
-
- /**
- * Build a role instance from a container assignment.
- * @param assigned
- * @return the instance
- */
- public RoleInstance roleInstance(ContainerAssignment assigned) {
- Container target = assigned.container;
- RoleInstance failedInstance =
- assigned.role.getProviderRole().failedInstances.poll();
- RoleInstance ri;
- if (failedInstance != null) {
- ri = new RoleInstance(target, failedInstance);
- } else {
- ri = new RoleInstance(target, assigned.role.getProviderRole());
- }
- ri.roleId = assigned.role.getPriority();
- ri.role = assigned.role.getName();
- return ri;
- }
-
- public NodeInstance nodeInstance(long age, int live0, int live1, int live2) {
- NodeInstance ni = new NodeInstance(String.format("age%d-[%d,%d,%d]", age,
- live0, live1, live2), MockFactory.ROLE_COUNT);
- ni.getOrCreate(getRole0Status().getKey()).setLastUsed(age);
- ni.getOrCreate(getRole0Status().getKey()).setLive(live0);
- if (live1 > 0) {
- ni.getOrCreate(getRole1Status().getKey()).setLive(live1);
- }
- if (live2 > 0) {
- ni.getOrCreate(getRole2Status().getKey()).setLive(live2);
- }
- return ni;
- }
-
- /**
- * Create a container status event.
- * @param c container
- * @return a status
- */
- ContainerStatus containerStatus(Container c) {
- return containerStatus(c.getId());
- }
-
- /**
- * Create a container status instance for the given ID, declaring
- * that it was shut down by the application itself.
- * @param cid container Id
- * @return the instance
- */
- public ContainerStatus containerStatus(ContainerId cid) {
- ContainerStatus status = containerStatus(cid,
- LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN);
- return status;
- }
-
- public ContainerStatus containerStatus(ContainerId cid, int exitCode) {
- ContainerStatus status = ContainerStatus.newInstance(
- cid,
- ContainerState.COMPLETE,
- "",
- exitCode);
- return status;
- }
-
- /**
- * Create nodes and bring them to the started state.
- * @return a list of roles
- */
- protected List<RoleInstance> createAndStartNodes()
- throws TriggerClusterTeardownException, SliderInternalStateException {
- return createStartAndStopNodes(new ArrayList<>());
- }
-
- /**
- * Create, Start and stop nodes.
- * @param completionResults List filled in with the status on all completed
- * nodes
- * @return the nodes
- */
- public List<RoleInstance> createStartAndStopNodes(
- List<AppState.NodeCompletionResult> completionResults)
- throws TriggerClusterTeardownException, SliderInternalStateException {
- List<ContainerId> released = new ArrayList<>();
- List<RoleInstance> instances = createAndSubmitNodes(released);
- processSubmissionOperations(instances, completionResults, released);
- return instances;
- }
-
- /**
- * Process the start/stop operations.
- * @param instances
- * @param completionResults
- * @param released
- */
- public void processSubmissionOperations(
- List<RoleInstance> instances,
- List<AppState.NodeCompletionResult> completionResults,
- List<ContainerId> released) {
- for (RoleInstance instance : instances) {
- LOG.debug("Started {} on {}", instance.role, instance.id);
- assertNotNull(appState.onNodeManagerContainerStarted(instance
- .getContainerId()));
- }
- releaseContainers(completionResults,
- released,
- ContainerState.COMPLETE,
- "released",
- 0
- );
- }
-
- /**
- * Release a list of containers, updating the completion results.
- * @param completionResults
- * @param containerIds
- * @param containerState
- * @param exitText
- * @param containerExitCode
- * @return
- */
- public void releaseContainers(
- List<AppState.NodeCompletionResult> completionResults,
- List<ContainerId> containerIds,
- ContainerState containerState,
- String exitText,
- int containerExitCode) {
- for (ContainerId id : containerIds) {
- ContainerStatus status = ContainerStatus.newInstance(id,
- containerState,
- exitText,
- containerExitCode);
- completionResults.add(appState.onCompletedContainer(status));
- }
- }
-
- /**
- * Create nodes and submit them.
- * @return a list of roles
- */
- public List<RoleInstance> createAndSubmitNodes()
- throws TriggerClusterTeardownException, SliderInternalStateException {
- return createAndSubmitNodes(new ArrayList<>());
- }
-
- /**
- * Create nodes and submit them.
- * @return a list of roles
- */
- public List<RoleInstance> createAndSubmitNodes(List<ContainerId> containerIds)
- throws TriggerClusterTeardownException, SliderInternalStateException {
- return createAndSubmitNodes(containerIds, new ArrayList<>());
- }
-
- /**
- * Create nodes and submit them.
- * @return a list of roles allocated
- */
- public List<RoleInstance> createAndSubmitNodes(
- List<ContainerId> containerIds,
- List<AbstractRMOperation> operationsOut)
- throws TriggerClusterTeardownException, SliderInternalStateException {
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- return submitOperations(ops, containerIds, operationsOut);
- }
-
- public List<RoleInstance> submitOperations(
- List<AbstractRMOperation> operationsIn,
- List<ContainerId> released) {
- return submitOperations(operationsIn, released, new ArrayList<>());
- }
-
- /**
- * Process the RM operations and send <code>onContainersAllocated</code>
- * events to the app state.
- * @param operationsIn list of incoming ops
- * @param released released containers
- * @return list of outbound operations
- */
- public List<RoleInstance> submitOperations(
- List<AbstractRMOperation> operationsIn,
- List<ContainerId> released,
- List<AbstractRMOperation> operationsOut) {
- List<Container> allocatedContainers = engine.execute(operationsIn,
- released);
- List<ContainerAssignment> assignments = new ArrayList<>();
- appState.onContainersAllocated(allocatedContainers, assignments,
- operationsOut);
-
- List<RoleInstance> roles = new ArrayList<>();
- for (ContainerAssignment assigned : assignments) {
- Container container = assigned.container;
- RoleInstance ri = roleInstance(assigned);
- //tell the app it arrived
- LOG.debug("Start submitted {} on ${}", ri.role, container.getId());
- appState.containerStartSubmitted(container, ri);
- roles.add(ri);
- }
- return roles;
- }
-
- /**
- * Add the AM to the app state.
- */
- protected void addAppMastertoAppState() {
-// appState.buildAppMasterNode(
-// new MockContainerId(applicationAttemptId, 999999L),
-// "appmaster",
-// 0,
-// null);
- }
-
- /**
- * Extract the list of container IDs from the list of role instances.
- * @param instances instance list
- * @param role role to look up
- * @return the list of CIDs
- */
- public List<ContainerId> extractContainerIds(
- List<RoleInstance> instances,
- String role) {
- List<ContainerId> ids = new ArrayList<>();
- for (RoleInstance ri : instances) {
- if (ri.role.equals(role)) {
- ids.add(ri.getContainerId());
- }
- }
- return ids;
- }
-
- /**
- * Record a node as failing.
- * @param node
- * @param id
- * @param count
- * @return the entry
- */
- public NodeEntry recordAsFailed(NodeInstance node, int id, int count) {
- NodeEntry entry = node.getOrCreate(id);
- for (int i = 1; i <= count; i++) {
- entry.containerCompleted(
- false,
- ContainerOutcome.Failed);
- }
- return entry;
- }
-
- protected void recordAllFailed(int id, int count, List<NodeInstance> nodes) {
- for (NodeInstance node : nodes) {
- recordAsFailed(node, id, count);
- }
- }
-
- /**
- * Get the container request of an indexed entry. Includes some assertions
- * for better diagnostics
- * @param ops operation list
- * @param index index in the list
- * @return the request.
- */
- public AMRMClient.ContainerRequest getRequest(List<AbstractRMOperation> ops,
- int index) {
- assertTrue(index < ops.size());
- AbstractRMOperation op = ops.get(index);
- assertTrue(op instanceof ContainerRequestOperation);
- return ((ContainerRequestOperation) op).getRequest();
- }
-
- /**
- * Get the cancel request of an indexed entry. Includes some assertions for
- * better diagnostics
- * @param ops operation list
- * @param index index in the list
- * @return the request.
- */
- public AMRMClient.ContainerRequest getCancel(List<AbstractRMOperation> ops,
- int index) {
- assertTrue(index < ops.size());
- AbstractRMOperation op = ops.get(index);
- assertTrue(op instanceof CancelSingleRequest);
- return ((CancelSingleRequest) op).getRequest();
- }
-
- /**
- * Get the single request of a list of operations; includes the check for
- * the size.
- * @param ops operations list of size 1
- * @return the request within the first ContainerRequestOperation
- */
- public AMRMClient.ContainerRequest getSingleRequest(
- List<AbstractRMOperation> ops) {
- assertEquals(1, ops.size());
- return getRequest(ops, 0);
- }
-
- /**
- * Get the single request of a list of operations; includes the check for
- * the size.
- * @param ops operations list of size 1
- * @return the request within the first operation
- */
- public AMRMClient.ContainerRequest getSingleCancel(
- List<AbstractRMOperation> ops) {
- assertEquals(1, ops.size());
- return getCancel(ops, 0);
- }
-
- /**
- * Get the single release of a list of operations; includes the check for
- * the size.
- * @param ops operations list of size 1
- * @return the request within the first operation
- */
- public ContainerReleaseOperation getSingleRelease(
- List<AbstractRMOperation> ops) {
- assertEquals(1, ops.size());
- AbstractRMOperation op = ops.get(0);
- assertTrue(op instanceof ContainerReleaseOperation);
- return (ContainerReleaseOperation) op;
- }
-
- /**
- * Get the node information as a large JSON String.
- * @return
- */
- protected String nodeInformationSnapshotAsString()
- throws UnsupportedEncodingException, JsonProcessingException {
- return prettyPrintAsJson(stateAccess.getNodeInformationSnapshot());
- }
-
- /**
- * Scan through all containers and assert that the assignment is AA.
- * @param index role index
- */
- protected void assertAllContainersAA(int index) {
- for (Entry<String, NodeInstance> nodeMapEntry : cloneNodemap().entrySet()) {
- String name = nodeMapEntry.getKey();
- NodeInstance ni = nodeMapEntry.getValue();
- NodeEntry nodeEntry = ni.get(index);
- assertTrue("too many instances on node " + name, nodeEntry == null ||
- nodeEntry.isAntiAffinityConstraintHeld());
- }
- }
-
- /**
- * Get a snapshot of the nodemap of the application state.
- * @return a cloned nodemap
- */
- protected NodeMap cloneNodemap() {
- return appState.getRoleHistory().cloneNodemap();
- }
-
- /**
- * Issue a nodes updated event.
- * @param report report to notify
- * @return response of AM
- */
- protected AppState.NodeUpdatedOutcome updateNodes(NodeReport report) {
- return appState.onNodesUpdated(Collections.singletonList(report));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java
deleted file mode 100644
index 66ae0f9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.slider.server.appmaster.SliderAppMaster;
-
-/**
- * Mock AM.
- */
-public class MockAM extends SliderAppMaster {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java
deleted file mode 100644
index 2fcf054..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
-import org.apache.slider.server.appmaster.state.AbstractClusterServices;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * Extended app state that makes more things public.
- */
-public class MockAppState extends AppState {
- public static final int RM_MAX_RAM = 4096;
- public static final int RM_MAX_CORES = 64;
-
- private long time = -1;
-
- public MockAppState(AbstractClusterServices recordFactory) {
- super(recordFactory, new MetricsAndMonitoring());
- setContainerLimits(1, RM_MAX_RAM, 1, RM_MAX_CORES);
- }
-
- /**
- * Instance with a mock record factory.
- */
- public MockAppState() {
- this(new MockClusterServices());
- }
-
- public MockAppState(AppStateBindingInfo bindingInfo)
- throws BadClusterStateException, IOException, BadConfigException {
- this();
- buildInstance(bindingInfo);
- }
-
- public Map<String, ProviderRole> getRoleMap() {
- return super.getRoleMap();
- }
-
- /**
- * Current time. if the <code>time</code> field
- * is set, that value is returned
- * @return the current time.
- */
- protected long now() {
- if (time > 0) {
- return time;
- }
- return System.currentTimeMillis();
- }
-
- public void setTime(long newTime) {
- this.time = newTime;
- }
-
- public void incTime(long inc) {
- this.time = this.time + inc;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java
deleted file mode 100644
index b509625..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-
-class MockApplicationAttemptId extends ApplicationAttemptId {
-
- private ApplicationId applicationId;
- private int attemptId;
-
- public MockApplicationAttemptId() {
- }
-
- public MockApplicationAttemptId(ApplicationId applicationId, int attemptId) {
- this.applicationId = applicationId;
- this.attemptId = attemptId;
- }
-
- @Override
- public ApplicationId getApplicationId() {
- return applicationId;
- }
-
- @Override
- public void setApplicationId(ApplicationId applicationId) {
- this.applicationId = applicationId;
- }
-
- @Override
- public int getAttemptId() {
- return attemptId;
- }
-
- @Override
- public void setAttemptId(int attemptId) {
- this.attemptId = attemptId;
- }
-
- @Override
- protected void build() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationId.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationId.java
deleted file mode 100644
index 01da470..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationId.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-
-/**
- * Mock app id.
- */
-public class MockApplicationId extends ApplicationId {
-
- private int id;
- private long clusterTimestamp;
-
- public MockApplicationId() {
- }
-
- public MockApplicationId(int id) {
- this.id = id;
- }
-
- public MockApplicationId(int id, long clusterTimestamp) {
- this.id = id;
- this.clusterTimestamp = clusterTimestamp;
- }
-
- @Override
- public int getId() {
- return id;
- }
-
- @Override
- public void setId(int id) {
- this.id = id;
- }
-
- @Override
- public long getClusterTimestamp() {
- return clusterTimestamp;
- }
-
- @Override
- public void setClusterTimestamp(long clusterTimestamp) {
- this.clusterTimestamp = clusterTimestamp;
- }
-
- @Override
- public void build() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockClusterServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockClusterServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockClusterServices.java
deleted file mode 100644
index 2578595..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockClusterServices.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.slider.server.appmaster.state.AbstractClusterServices;
-
-/**
- * Mock cluster services.
- */
-public class MockClusterServices extends AbstractClusterServices {
-
- @Override
- public Resource newResource() {
- return new MockResource(0, 0);
- }
-
- @Override
- public Resource newResource(int memory, int cores) {
- return new MockResource(memory, cores);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainer.java
deleted file mode 100644
index 148b7f6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainer.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ExecutionType;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.Token;
-
-/**
- * Mock container.
- */
-public class MockContainer extends Container {
-
- private ContainerId id;
- private NodeId nodeId;
- private String nodeHttpAddress;
- private Resource resource;
- private Priority priority;
- private Token containerToken;
-
- @Override
- public int compareTo(Container other) {
- if (this.getId().compareTo(other.getId()) == 0) {
- if (this.getNodeId().compareTo(other.getNodeId()) == 0) {
- return this.getResource().compareTo(other.getResource());
- } else {
- return this.getNodeId().compareTo(other.getNodeId());
- }
- } else {
- return this.getId().compareTo(other.getId());
- }
- }
-
- @Override
- public String toString() {
- return "MockContainer{ id=" + id +
- ", nodeHttpAddress='" + nodeHttpAddress + "'," +
- " priority=" + priority + " }";
- }
-
- @Override
- public ContainerId getId() {
- return id;
- }
-
- @Override
- public void setId(ContainerId id) {
- this.id = id;
- }
-
- @Override
- public NodeId getNodeId() {
- return nodeId;
- }
-
- @Override
- public void setNodeId(NodeId nodeId) {
- this.nodeId = nodeId;
- }
-
- @Override
- public String getNodeHttpAddress() {
- return nodeHttpAddress;
- }
-
- @Override
- public void setNodeHttpAddress(String nodeHttpAddress) {
- this.nodeHttpAddress = nodeHttpAddress;
- }
-
- @Override
- public Resource getResource() {
- return resource;
- }
-
- @Override
- public void setResource(Resource resource) {
- this.resource = resource;
- }
-
- @Override
- public Priority getPriority() {
- return priority;
- }
-
- @Override
- public void setPriority(Priority priority) {
- this.priority = priority;
- }
-
- @Override
- public Token getContainerToken() {
- return containerToken;
- }
-
- @Override
- public void setContainerToken(Token containerToken) {
- this.containerToken = containerToken;
- }
-
- @Override
- public ExecutionType getExecutionType() {
- return null;
- }
-
- @Override
- public void setExecutionType(ExecutionType executionType) {
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainerId.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainerId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainerId.java
deleted file mode 100644
index 3cbc7e5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainerId.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-
-/**
- * Mock container id.
- */
-public class MockContainerId extends ContainerId implements Cloneable {
-
- private static final MockApplicationAttemptId DEFAULT_APP_ATTEMPT_ID =
- new MockApplicationAttemptId(new MockApplicationId(1), 1);
-
- private long containerId;
- private ApplicationAttemptId applicationAttemptId;
-
- MockContainerId() {
- }
-
- /**
- * Sets up a default app Attempt ID.
- * @param containerId
- */
- MockContainerId(long containerId) {
- this.containerId = containerId;
- this.applicationAttemptId = DEFAULT_APP_ATTEMPT_ID;
- }
-
- public MockContainerId(ApplicationAttemptId applicationAttemptId,
- long containerId) {
- this.containerId = containerId;
- this.applicationAttemptId = applicationAttemptId;
- }
-
- MockContainerId(ContainerId that) {
- containerId = that.getContainerId();
- applicationAttemptId = that.getApplicationAttemptId();
- }
-
- @Deprecated
- @Override
- public int getId() {
- return (int) containerId;
- }
-
- // TODO: Temporarily adding it back
- void setId(int id) {
- containerId = (long) id;
- }
-
- @Override
- public long getContainerId() {
- return this.containerId;
- }
-
- @Override
- public void setContainerId(long id) {
- this.containerId = id;
- }
-
- @Override
- public ApplicationAttemptId getApplicationAttemptId() {
- return applicationAttemptId;
- }
-
- @Override
- public void setApplicationAttemptId(ApplicationAttemptId
- applicationAttemptId) {
- this.applicationAttemptId = applicationAttemptId;
- }
-
- @Override
- public void build() {
-
- }
-
- @Override
- public String toString() {
- return "mockcontainer_" + containerId;
- }
-
- @Override
- protected Object clone() throws CloneNotSupportedException {
- return super.clone();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java
deleted file mode 100644
index 8785b92..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java
+++ /dev/null
@@ -1,273 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.Resource;
-import org.apache.slider.providers.PlacementPolicy;
-import org.apache.slider.providers.ProviderRole;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-
-import static org.apache.slider.api.ResourceKeys.COMPONENT_PLACEMENT_POLICY;
-
-/**
- * Factory for creating things.
- */
-public class MockFactory implements MockRoles {
-
- public static final int NODE_FAILURE_THRESHOLD = 2;
-
- public static final MockFactory INSTANCE = new MockFactory();
-
- /**
- * Basic role.
- */
- public static final ProviderRole PROVIDER_ROLE0 = new ProviderRole(
- MockRoles.ROLE0,
- 0,
- PlacementPolicy.DEFAULT,
- NODE_FAILURE_THRESHOLD,
- 1,
- ResourceKeys.DEF_YARN_LABEL_EXPRESSION);
- /**
- * role 1 is strict. timeout should be irrelevant; same as failures
- */
- public static final ProviderRole PROVIDER_ROLE1 = new ProviderRole(
- MockRoles.ROLE1,
- 1,
- PlacementPolicy.STRICT,
- NODE_FAILURE_THRESHOLD,
- 1,
- ResourceKeys.DEF_YARN_LABEL_EXPRESSION);
-
- /**
- * role 2: longer delay.
- */
- public static final ProviderRole PROVIDER_ROLE2 = new ProviderRole(
- MockRoles.ROLE2,
- 2,
- PlacementPolicy.ANYWHERE,
- NODE_FAILURE_THRESHOLD,
- 2,
- ResourceKeys.DEF_YARN_LABEL_EXPRESSION);
-
- /**
- * Patch up a "role2" role to have anti-affinity set.
- */
- public static final ProviderRole AAROLE_2 = new ProviderRole(
- MockRoles.ROLE2,
- 2,
- PlacementPolicy.ANTI_AFFINITY_REQUIRED,
- NODE_FAILURE_THRESHOLD,
- 2,
- null);
-
- /**
- * Patch up a "role1" role to have anti-affinity set and GPI as the label.
- */
- public static final ProviderRole AAROLE_1_GPU = new ProviderRole(
- MockRoles.ROLE1,
- 1,
- PlacementPolicy.ANTI_AFFINITY_REQUIRED,
- NODE_FAILURE_THRESHOLD,
- 1,
- MockRoles.LABEL_GPU);
-
- private int appIdCount;
- private int attemptIdCount;
- private int containerIdCount;
-
- private ApplicationId appId = newAppId();
- private ApplicationAttemptId attemptId = newApplicationAttemptId(appId);
-
- /**
- * List of roles.
- */
- public static final List<ProviderRole> ROLES = Arrays.asList(
- PROVIDER_ROLE0,
- PROVIDER_ROLE1,
- PROVIDER_ROLE2
- );
-
- public static final int ROLE_COUNT = ROLES.size();
-
- MockContainerId newContainerId() {
- return newContainerId(attemptId);
- }
-
- MockContainerId newContainerId(ApplicationAttemptId attemptId0) {
- MockContainerId cid = new MockContainerId(attemptId0, containerIdCount++);
- return cid;
- }
-
- MockApplicationAttemptId newApplicationAttemptId(ApplicationId appId0) {
- MockApplicationAttemptId id = new MockApplicationAttemptId(appId0,
- attemptIdCount++);
- return id;
- }
-
- MockApplicationId newAppId() {
- MockApplicationId id = new MockApplicationId();
- id.setId(appIdCount++);
- return id;
- }
-
- public MockNodeId newNodeId(String host) {
- return new MockNodeId(host);
- }
-
- MockContainer newContainer(ContainerId cid) {
- MockContainer c = new MockContainer();
- c.setId(cid);
- return c;
- }
-
- public MockContainer newContainer() {
- return newContainer(newContainerId());
- }
-
- public MockContainer newContainer(NodeId nodeId, Priority priority) {
- MockContainer container = newContainer(newContainerId());
- container.setNodeId(nodeId);
- container.setPriority(priority);
- return container;
- }
-
- /**
- * Build a new container using the request to supply priority and resource.
- * @param req request
- * @param host hostname to assign to
- * @return the container
- */
- public MockContainer newContainer(AMRMClient.ContainerRequest req, String
- host) {
- MockContainer container = newContainer(newContainerId());
- container.setResource(req.getCapability());
- container.setPriority(req.getPriority());
- container.setNodeId(new MockNodeId(host));
- return container;
- }
-
- /**
- * Create a new instance with the given components definined in the
- * resources section.
- * @param r1
- * @param r2
- * @param r3
- * @return
- */
- public Application newApplication(long r1, long r2, long r3) {
- Application application = new Application();
- application.setLaunchCommand("sleep 60");
- application.setResource(new Resource().memory("256"));
- application.getConfiguration().setProperty(ResourceKeys
- .NODE_FAILURE_THRESHOLD, Integer.toString(NODE_FAILURE_THRESHOLD));
- List<Component> components = application.getComponents();
- Component c1 = new Component().name(ROLE0).numberOfContainers(r1);
- c1.getConfiguration().setProperty(COMPONENT_PLACEMENT_POLICY,
- Integer.toString(PlacementPolicy.DEFAULT));
- Component c2 = new Component().name(ROLE1).numberOfContainers(r2);
- c2.getConfiguration().setProperty(COMPONENT_PLACEMENT_POLICY,
- Integer.toString(PlacementPolicy.STRICT));
- Component c3 = new Component().name(ROLE2).numberOfContainers(r3);
- c3.getConfiguration().setProperty(COMPONENT_PLACEMENT_POLICY,
- Integer.toString(PlacementPolicy.ANYWHERE));
- components.add(c1);
- components.add(c2);
- components.add(c3);
- return application;
- }
-
- public MockResource newResource(int memory, int vcores) {
- return new MockResource(memory, vcores);
- }
-
- ContainerStatus newContainerStatus() {
- return newContainerStatus(null, null, "", 0);
- }
-
- ContainerStatus newContainerStatus(ContainerId containerId,
- ContainerState containerState, String diagnostics, int exitStatus) {
- return ContainerStatus.newInstance(containerId, containerState,
- diagnostics, exitStatus);
- }
-
- /**
- * Create a single instance.
- * @param hostname
- * @param nodeState
- * @param label
- */
- public NodeReport newNodeReport(String hostname, NodeState nodeState,
- String label) {
- NodeId nodeId = NodeId.newInstance(hostname, 80);
- Integer.valueOf(hostname, 16);
- return newNodeReport(hostname, nodeId, nodeState, label);
- }
-
- NodeReport newNodeReport(
- String hostname,
- NodeId nodeId,
- NodeState nodeState,
- String label) {
- NodeReport report = new NodeReportPBImpl();
- HashSet<String> nodeLabels = new HashSet<>();
- nodeLabels.add(label);
- report.setNodeId(nodeId);
- report.setNodeLabels(nodeLabels);
- report.setNodeState(nodeState);
- report.setHttpAddress("http$hostname:80");
- return report;
- }
-
- /**
- * Create a list of instances -one for each hostname.
- * @param hostnames hosts
- * @return
- */
- public List<NodeReport> createNodeReports(
- List<String> hostnames, NodeState nodeState, String label) {
- if (nodeState == null) {
- nodeState = NodeState.RUNNING;
- }
- List<NodeReport> reports = new ArrayList<>();
- for (String name : hostnames) {
- reports.add(newNodeReport(name, nodeState, label));
- }
- return reports;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFileSystem.java
deleted file mode 100644
index 72d1665..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFileSystem.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.fs.FilterFileSystem;
-import org.apache.hadoop.fs.Path;
-
-import java.io.IOException;
-
-/**
- *
- */
-class MockFileSystem extends FilterFileSystem{
- @Override
- public Path resolvePath(Path p) throws IOException {
- return new Path("hdfs://localhost/", p);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockNodeId.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockNodeId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockNodeId.java
deleted file mode 100644
index 9d2379a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockNodeId.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.yarn.api.records.NodeId;
-
-/**
- * Mock node id.
- */
-public class MockNodeId extends NodeId {
- private String host;
- private int port;
-
- public MockNodeId() {
- }
-
- MockNodeId(String host) {
- this.host = host;
- }
-
- public MockNodeId(String host, int port) {
- this.host = host;
- this.port = port;
- }
-
- public String getHost() {
- return host;
- }
-
- public void setHost(String host) {
- this.host = host;
- }
-
- public int getPort() {
- return port;
- }
-
- public void setPort(int port) {
- this.port = port;
- }
-
- @Override
- protected void build() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockPriority.java
deleted file mode 100644
index 36f97cc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockPriority.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.yarn.api.records.Priority;
-
-/**
- * Mock priority.
- */
-public class MockPriority extends Priority {
-
- private int priority;
-
- public MockPriority(int priority) {
- this.priority = priority;
- }
-
- MockPriority() {
- }
-
- @Override
- public int getPriority() {
- return priority;
- }
-
- @Override
- public void setPriority(int priority) {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.java
deleted file mode 100644
index 3dd764a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.apache.slider.server.appmaster.operations.RMOperationHandler;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Mock RM operation handler.
- */
-public class MockRMOperationHandler extends RMOperationHandler {
- protected static final Logger LOG =
- LoggerFactory.getLogger(MockRMOperationHandler.class);
-
- private List<AbstractRMOperation> operations = new ArrayList<>();
- private int requests;
- private int releases;
- // number available to cancel
- private int availableToCancel = 0;
- // count of cancelled values. This must be explicitly set
- private int cancelled;
- // number blacklisted
- private int blacklisted = 0;
-
- @Override
- public void releaseAssignedContainer(ContainerId containerId) {
- operations.add(new ContainerReleaseOperation(containerId));
- LOG.info("Releasing container ID " + containerId.getContainerId());
- releases++;
- }
-
- @Override
- public void addContainerRequest(AMRMClient.ContainerRequest req) {
- operations.add(new ContainerRequestOperation(req));
- LOG.info("Requesting container role #" + req.getPriority());
- requests++;
- }
-
- @Override
- public int cancelContainerRequests(
- Priority priority1,
- Priority priority2,
- int count) {
- int releaseable = Math.min(count, availableToCancel);
- availableToCancel -= releaseable;
- cancelled += releaseable;
- return releaseable;
- }
-
- @Override
- public void cancelSingleRequest(AMRMClient.ContainerRequest request) {
- // here assume that there is a copy of this request in the list
- if (availableToCancel > 0) {
- availableToCancel--;
- cancelled++;
- }
- }
-
- @Override
- public void updateBlacklist(List<String> blacklistAdditions, List<String>
- blacklistRemovals) {
- blacklisted += blacklistAdditions.size();
- blacklisted -= blacklistRemovals.size();
- }
-
- /**
- * Clear the history.
- */
- public void clear() {
- operations.clear();
- releases = 0;
- requests = 0;
- }
-
- public AbstractRMOperation getFirstOp() {
- return operations.get(0);
- }
-
- public int getNumReleases() {
- return releases;
- }
-
- public void setAvailableToCancel(int num) {
- this.availableToCancel = num;
- }
-
- public int getAvailableToCancel() {
- return availableToCancel;
- }
-
- public int getBlacklisted() {
- return blacklisted;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.java
deleted file mode 100644
index eb34586..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-/**
- * Node report for testing.
- */
-class MockRecordFactory {
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRegistryOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRegistryOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRegistryOperations.java
deleted file mode 100644
index 4917f1b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRegistryOperations.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.types.RegistryPathStatus;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.service.AbstractService;
-
-import java.util.List;
-
-/**
- * Simple stub registry for when one is needed for its API, but the operations
- * are not actually required.
- */
-class MockRegistryOperations extends AbstractService implements
- RegistryOperations {
-
- MockRegistryOperations() {
- super("mock");
- }
-
- @Override
- public boolean mknode(String path, boolean createParents) {
- return true;
- }
-
- @Override
- public void bind(String path, ServiceRecord record, int flags) {
- }
-
- @Override
- public ServiceRecord resolve(String path) throws PathNotFoundException {
- throw new PathNotFoundException(path);
- }
-
- @Override
- public RegistryPathStatus stat(String path) throws PathNotFoundException {
- throw new PathNotFoundException(path);
- }
-
- @Override
- public boolean exists(String path) {
- return false;
- }
-
- @Override
- public List<String> list(String path) throws PathNotFoundException {
- throw new PathNotFoundException(path);
- }
-
- @Override
- public void delete(String path, boolean recursive) {
-
- }
-
- @Override
- public boolean addWriteAccessor(String id, String pass) {
- return true;
- }
-
- @Override
- public void clearWriteAccessors() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockResource.java
deleted file mode 100644
index 3a2ccd7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockResource.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-
-/**
- * Mock resource.
- */
-public class MockResource extends Resource {
- private int memory;
- private int virtualCores;
-
- public MockResource(int memory, int vcores) {
- this.memory = memory;
- this.virtualCores = vcores;
- }
-
- @Override
- public int compareTo(Resource other) {
- long diff = this.getMemorySize() - other.getMemorySize();
- if (diff == 0) {
- diff = this.getVirtualCores() - other.getVirtualCores();
- }
- return diff == 0 ? 0 : (diff > 0 ? 1 : -1);
- }
-
- @Override
- public long getMemorySize() {
- return memory;
- }
-
- @Override
- public void setMemorySize(long memorySize) {
- memory = (int) memorySize;
- }
-
- @Override
- public int getVirtualCores() {
- return virtualCores;
- }
-
- @Override
- public void setVirtualCores(int vCores) {
- this.virtualCores = vCores;
- }
-
- @Deprecated
- @Override
- public int getMemory() {
- return memory;
- }
-
- @Deprecated
- @Override
- public void setMemory(int memory) {
- this.memory = memory;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.java
deleted file mode 100644
index 8e88b0d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.server.appmaster.state.RoleHistory;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Subclass to enable access to some of the protected methods.
- */
-public class MockRoleHistory extends RoleHistory {
-
- /**
- * Take a list of provider roles and build the history from them,
- * dynamically creating the role status entries on the way.
- * @param providerRoles provider role list
- * @throws BadConfigException configuration problem with the role list
- */
- public MockRoleHistory(List<ProviderRole> providerRoles) throws
- BadConfigException {
- super(convertRoles(providerRoles), new MockClusterServices());
- }
-
- static List<RoleStatus> convertRoles(List<ProviderRole> providerRoles) {
- List<RoleStatus> statuses = new ArrayList<>();
- for (ProviderRole role : providerRoles) {
- statuses.add(new RoleStatus(role));
- }
- return statuses;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoles.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoles.java
deleted file mode 100644
index bad82bd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoles.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.model.mock;
-
-/**
- * Mock role constants.
- */
-public interface MockRoles {
-
- String ROLE0 = "role0";
- String ROLE1 = "role1";
- String ROLE2 = "role2";
- String LABEL_GPU = "gpu";
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.java
deleted file mode 100644
index 6b685a0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.java
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-
-/**
- * Models the cluster itself: a set of mock cluster nodes.
- *
- * nodes retain the slot model with a limit of 2^8 slots/host -this
- * lets us use 24 bits of the container ID for hosts, and so simulate
- * larger hosts.
- *
- * upper 32: index into nodes in the cluster
- * NodeID hostname is the index in hex format; this is parsed down to the index
- * to resolve the host
- *
- * Important: container IDs will be reused as containers get recycled. This
- * is not an attempt to realistically mimic a real YARN cluster, just
- * simulate it enough for Slider to explore node re-use and its handling
- * of successful and unsuccessful allocations.
- *
- * There is little or no checking of valid parameters in here -this is for
- * test use, not production.
- */
-public class MockYarnCluster {
- protected static final Logger LOG =
- LoggerFactory.getLogger(MockYarnCluster.class);
-
- private final int clusterSize;
- private final int containersPerNode;
- private MockYarnClusterNode[] nodes;
-
- MockYarnCluster(int clusterSize, int containersPerNode) {
- this.clusterSize = clusterSize;
- this.containersPerNode = containersPerNode;
- build();
- }
-
- public int getClusterSize() {
- return clusterSize;
- }
-
- @Override
- public String toString() {
- return "MockYarnCluster size=" + clusterSize + ", capacity=" +
- totalClusterCapacity()+ ", in use=" + containersInUse();
- }
-
- /**
- * Build the cluster.
- */
- private void build() {
- nodes = new MockYarnClusterNode[clusterSize];
- for (int i = 0; i < clusterSize; i++) {
- nodes[i] = new MockYarnClusterNode(i, containersPerNode);
- }
- }
-
- public MockYarnClusterNode nodeAt(int index) {
- return nodes[index];
- }
-
- MockYarnClusterNode lookup(String hostname) {
- int index = Integer.valueOf(hostname, 16);
- return nodeAt(index);
- }
-
- MockYarnClusterNode lookup(NodeId nodeId) {
- return lookup(nodeId.getHost());
- }
-
- MockYarnClusterNode lookupOwner(ContainerId cid) {
- return nodeAt(extractHost(cid.getContainerId()));
- }
-
- /**
- * Release a container: return true if it was actually in use.
- * @param cid container ID
- * @return the container released
- */
- MockYarnClusterContainer release(ContainerId cid) {
- int host = extractHost(cid.getContainerId());
- MockYarnClusterContainer inUse = nodeAt(host).release(cid.getContainerId());
- LOG.debug("Released {} inuse={}", cid, inUse);
- return inUse;
- }
-
- int containersInUse() {
- int count = 0;
- for (MockYarnClusterNode it : nodes) {
- count += it.containersInUse();
- }
- return count;
- }
-
- /**
- * Containers free.
- * @return
- */
- int containersFree() {
- return totalClusterCapacity() - containersInUse();
- }
-
- int totalClusterCapacity() {
- return clusterSize * containersPerNode;
- }
-
- /**
- * Reset all the containers.
- */
- public void reset() {
- for (MockYarnClusterNode node : nodes) {
- node.reset();
- }
- }
-
- /**
- * Bulk allocate the specific number of containers on a range of the cluster.
- * @param startNode start of the range
- * @param endNode end of the range
- * @param count count
- * @return the number actually allocated -it will be less the count supplied
- * if the node was full
- */
- public int bulkAllocate(int startNode, int endNode, int count) {
- int total = 0;
- for (int i = startNode; i <= endNode; i++) {
- total += nodeAt(i).bulkAllocate(count).size();
- }
- return total;
- }
-
- /**
- * Get the list of node reports. These are not cloned; updates will persist
- * in the nodemap
- * @return current node report list
- */
- List<NodeReport> getNodeReports() {
- List<NodeReport> reports = new ArrayList<>();
-
- for (MockYarnClusterNode n : nodes) {
- reports.add(n.nodeReport);
- }
- return reports;
- }
-
- /**
- * Model cluster nodes on the simpler "slot" model than the YARN-era
- * resource allocation model. Why? Easier to implement scheduling.
- * Of course, if someone does want to implement the full process...
- *
- */
- public static class MockYarnClusterNode {
-
- private final int nodeIndex;
- private final String hostname;
- private List<String> labels = new ArrayList<>();
- private final MockNodeId nodeId;
- private final MockYarnClusterContainer[] containers;
- private boolean offline;
- private NodeReport nodeReport;
-
- public MockYarnClusterNode(int index, int size) {
- nodeIndex = index;
- hostname = String.format(Locale.ENGLISH, "%08x", index);
- nodeId = new MockNodeId(hostname, 0);
-
- containers = new MockYarnClusterContainer[size];
- for (int i = 0; i < size; i++) {
- int cid = makeCid(index, i);
- MockContainerId mci = new MockContainerId(cid);
- containers[i] = new MockYarnClusterContainer(mci);
- }
-
- nodeReport = MockFactory.INSTANCE.newNodeReport(hostname, nodeId,
- NodeState.RUNNING, "");
- }
-
- public String getHostname() {
- return hostname;
- }
-
- public NodeId getNodeId() {
- return nodeId;
- }
-
- /**
- * Look up a container.
- * @param containerId
- * @return
- */
- public MockYarnClusterContainer lookup(int containerId) {
- return containers[extractContainer(containerId)];
- }
-
- /**
- * Go offline; release all containers.
- */
- public void goOffline() {
- if (!offline) {
- offline = true;
- reset();
- }
- }
-
- public void goOnline() {
- offline = false;
- }
-
- /**
- * Allocate a container -if one is available.
- * @return the container or null for none free
- * -or the cluster node is offline
- */
- public MockYarnClusterContainer allocate() {
- if (!offline) {
- for (int i = 0; i < containers.length; i++) {
- MockYarnClusterContainer c = containers[i];
- if (!c.busy) {
- c.busy = true;
- return c;
- }
- }
- }
- return null;
- }
-
- /**
- * Bulk allocate the specific number of containers.
- * @param count count
- * @return the list actually allocated -it will be less the count supplied
- * if the node was full
- */
- public List<MockYarnClusterContainer> bulkAllocate(int count) {
- List<MockYarnClusterContainer> result = new ArrayList<>();
- for (int i = 0; i < count; i++) {
- MockYarnClusterContainer allocation = allocate();
- if (allocation == null) {
- break;
- }
- result.add(allocation);
- }
- return result;
- }
-
- /**
- * Release a container.
- * @param cid container ID
- * @return the container if the container was busy before the release
- */
- public MockYarnClusterContainer release(long cid) {
- MockYarnClusterContainer container = containers[extractContainer(cid)];
- boolean b = container.busy;
- container.busy = false;
- return b? container: null;
- }
-
- public String httpAddress() {
- return "http://$hostname/";
- }
-
- /**
- * Reset all the containers.
- */
- public void reset() {
- for (MockYarnClusterContainer cont : containers) {
- cont.reset();
- }
- }
-
- public int containersInUse() {
- int c = 0;
- for (MockYarnClusterContainer cont : containers) {
- c += cont.busy ? 1 : 0;
- }
- return c;
- }
-
- public int containersFree() {
- return containers.length - containersInUse();
- }
- }
-
- /**
- * Cluster container.
- */
- public static class MockYarnClusterContainer {
- private MockContainerId cid;
- private boolean busy;
-
- MockYarnClusterContainer(MockContainerId cid) {
- this.cid = cid;
- }
-
- public MockContainerId getCid() {
- return cid;
- }
-
- void reset() {
- busy = false;
- }
- }
-
- public static int makeCid(int hostIndex, int containerIndex) {
- return (hostIndex << 8) | containerIndex & 0xff;
- }
-
- public static final int extractHost(long cid) {
- return (int)((cid >>> 8) & 0xffff);
- }
-
- public static final int extractContainer(long cid) {
- return (int)(cid & 0xff);
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
deleted file mode 100644
index 64fd8ae..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
+++ /dev/null
@@ -1,611 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.xml.sax.SAXException;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.StringWriter;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-/**
- * Methods to aid in config, both in the Configuration class and
- * with other parts of setting up Slider-initated processes.
- *
- * Some of the methods take an argument of a map iterable for their sources; this allows
- * the same method
- */
-public class ConfigHelper {
- private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);
-
- /**
- * Dump the (sorted) configuration
- * @param conf config
- * @return the sorted keyset
- */
- public static Set<String> dumpConf(Configuration conf) {
- Set<String> keys = sortedConfigKeys(conf);
- for (String key : keys) {
- log.info("{}={}", key, conf.get(key));
- }
- return keys;
- }
-
- /**
- * Take a configuration and return a sorted set
- * @param conf config
- * @return the sorted keyset
-
- */
- public static Set<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) {
- TreeSet<String> sorted = new TreeSet<String>();
- for (Map.Entry<String, String> entry : conf) {
- sorted.add(entry.getKey());
- }
- return sorted;
- }
-
- /**
- * Set an entire map full of values
- *
- * @param config config to patch
- * @param map map of data
- * @param origin origin data
- */
- public static void addConfigMap(Configuration config,
- Map<String, String> map,
- String origin) throws BadConfigException {
- addConfigMap(config, map.entrySet(), origin);
- }
-
- /**
- * Set an entire map full of values
- *
- * @param config config to patch
- * @param map map of data
- * @param origin origin data
- */
- public static void addConfigMap(Configuration config,
- Iterable<Map.Entry<String, String>> map,
- String origin) throws BadConfigException {
- for (Map.Entry<String, String> mapEntry : map) {
- String key = mapEntry.getKey();
- String value = mapEntry.getValue();
- if (value == null) {
- throw new BadConfigException("Null value for property " + key);
- }
- config.set(key, value, origin);
- }
- }
-
-
- /**
- * Save a config file in a destination directory on a given filesystem
- * @param systemConf system conf used for creating filesystems
- * @param confToSave config to save
- * @param confdir the directory path where the file is to go
- * @param filename the filename
- * @return the destination path where the file was saved
- * @throws IOException IO problems
- */
- public static Path saveConfig(Configuration systemConf,
- Configuration confToSave,
- Path confdir,
- String filename) throws IOException {
- FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);
- Path destPath = new Path(confdir, filename);
- saveConfig(fs, destPath, confToSave);
- return destPath;
- }
-
- /**
- * Save a config
- * @param fs filesystem
- * @param destPath dest to save
- * @param confToSave config to save
- * @throws IOException IO problems
- */
- public static void saveConfig(FileSystem fs,
- Path destPath,
- Configuration confToSave) throws
- IOException {
- FSDataOutputStream fos = fs.create(destPath);
- try {
- confToSave.writeXml(fos);
- } finally {
- IOUtils.closeStream(fos);
- }
- }
-
- /**
- * Convert to an XML string
- * @param conf configuration
- * @return conf
- * @throws IOException
- */
- public static String toXml(Configuration conf) throws IOException {
- StringWriter writer = new StringWriter();
- conf.writeXml(writer);
- return writer.toString();
- }
-
- /**
- * This will load and parse a configuration to an XML document
- * @param fs filesystem
- * @param path path
- * @return an XML document
- * @throws IOException IO failure
- */
- public Document parseConfiguration(FileSystem fs,
- Path path) throws
- IOException {
-
-
- byte[] data = loadBytes(fs, path);
- //this is here to track down a parse issue
- //related to configurations
- String s = new String(data, 0, data.length, "UTF-8");
- log.debug("XML resource {} is \"{}\"", path, s);
-/* JDK7
- try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
- Document document = parseConfigXML(in);
- return document;
- } catch (ParserConfigurationException | SAXException e) {
- throw new IOException(e);
- }
-*/
- ByteArrayInputStream in= null;
- try {
- in = new ByteArrayInputStream(data);
- Document document = parseConfigXML(in);
- return document;
- } catch (ParserConfigurationException e) {
- throw new IOException(e);
- } catch (SAXException e) {
- throw new IOException(e);
- } finally {
- IOUtils.closeStream(in);
- }
- }
-
- public static byte[] loadBytes(FileSystem fs, Path path) throws IOException {
- int len = (int) fs.getLength(path);
- byte[] data = new byte[len];
- /* JDK7
- try(FSDataInputStream in = fs.open(path)) {
- in.readFully(0, data);
- }
-*/
- FSDataInputStream in = null;
- in = fs.open(path);
- try {
- in.readFully(0, data);
- } finally {
- IOUtils.closeStream(in);
- }
- return data;
- }
-
- /**
- * Load a configuration from ANY FS path. The normal Configuration
- * loader only works with file:// URIs
- * @param fs filesystem
- * @param path path
- * @return a loaded resource
- * @throws IOException
- */
- public static Configuration loadConfiguration(FileSystem fs,
- Path path) throws IOException {
- byte[] data = loadBytes(fs, path);
-
- ByteArrayInputStream in2;
-
- in2 = new ByteArrayInputStream(data);
- Configuration conf1 = new Configuration(false);
- conf1.addResource(in2);
- //now clone it while dropping all its sources
- Configuration conf2 = new Configuration(false);
- String src = path.toString();
- for (Map.Entry<String, String> entry : conf1) {
- String key = entry.getKey();
- String value = entry.getValue();
- conf2.set(key, value, src);
- }
- return conf2;
- }
-
-
- /**
- * Generate a config file in a destination directory on the local filesystem
- * @param confdir the directory path where the file is to go
- * @param filename the filename
- * @return the destination path
- */
- public static File saveConfig(Configuration generatingConf,
- File confdir,
- String filename) throws IOException {
-
-
- File destPath = new File(confdir, filename);
- OutputStream fos = new FileOutputStream(destPath);
- try {
- generatingConf.writeXml(fos);
- } finally {
- IOUtils.closeStream(fos);
- }
- return destPath;
- }
-
- /**
- * Parse an XML Hadoop configuration into an XML document. x-include
- * is supported, but as the location isn't passed in, relative
- * URIs are out.
- * @param in instream
- * @return a document
- * @throws ParserConfigurationException parser feature problems
- * @throws IOException IO problems
- * @throws SAXException XML is invalid
- */
- public static Document parseConfigXML(InputStream in) throws
- ParserConfigurationException,
- IOException,
- SAXException {
- DocumentBuilderFactory docBuilderFactory
- = DocumentBuilderFactory.newInstance();
- //ignore all comments inside the xml file
- docBuilderFactory.setIgnoringComments(true);
-
- //allow includes in the xml file
- docBuilderFactory.setNamespaceAware(true);
- docBuilderFactory.setXIncludeAware(true);
- DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
- return builder.parse(in);
- }
-
- /**
- * Load a Hadoop configuration from a local file.
- * @param file file to load
- * @return a configuration which hasn't actually had the load triggered
- * yet.
- * @throws FileNotFoundException file is not there
- * @throws IOException any other IO problem
- */
- public static Configuration loadConfFromFile(File file) throws
- IOException {
- return loadConfFromFile(file, false);
- }
-
- /**
- *
- * Load a Hadoop configuration from a local file.
- * @param file file to load
- * @param loadDefaults flag to indicate if the defaults should be loaded yet
- * @return a configuration which hasn't actually had the load triggered
- * yet.
- * @throws FileNotFoundException file is not there
- * @throws IOException any other IO problem
- */
- public static Configuration loadConfFromFile(File file,
- boolean loadDefaults) throws IOException {
- if (!file.exists()) {
- throw new FileNotFoundException("File not found :"
- + file.getAbsoluteFile());
- }
- Configuration conf = new Configuration(loadDefaults);
- try {
- conf.addResource(file.toURI().toURL());
- } catch (MalformedURLException e) {
- // should never happen...
- throw new IOException(
- "File " + file.toURI() + " doesn't have a valid URL");
- }
- return conf;
- }
-
- /**
- * Add a configuration from a file to an existing configuration
- * @param conf existing configuration
- * @param file file to load
- * @param overwrite flag to indicate new values should overwrite the predecessor
- * @return the merged configuration
- * @throws IOException
- */
- public static Configuration addConfigurationFile(Configuration conf,
- File file, boolean overwrite)
- throws IOException {
- Configuration c2 = loadConfFromFile(file, false);
- mergeConfigurations(conf, c2, file.getAbsolutePath(), overwrite);
- return conf;
- }
-
- /**
- * Add the system env variables with the given prefix (by convention, env.)
- * @param conf existing configuration
- * @param prefix prefix
- */
- public static void addEnvironmentVariables(Configuration conf, String prefix) {
- Map<String, String> env = System.getenv();
- for (Map.Entry<String, String> entry : env.entrySet()) {
- conf.set(prefix + entry.getKey(),entry.getValue(), "env");
- }
- }
-
- /**
- * looks for the config under $confdir/$templateFilename; if not there
- * loads it from /conf/templateFile.
- * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
- * origin to help debug what's happening
- * @param systemConf system conf
- * @param confdir conf dir in FS
- * @param templateFilename filename in the confdir
- * @param fallbackResource resource to fall back on
- * @return loaded conf
- * @throws IOException IO problems
- */
- public static Configuration loadTemplateConfiguration(Configuration systemConf,
- Path confdir,
- String templateFilename,
- String fallbackResource) throws
- IOException {
- FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);
-
- Path templatePath = new Path(confdir, templateFilename);
- return loadTemplateConfiguration(fs, templatePath, fallbackResource);
- }
-
- /**
- * looks for the config under $confdir/$templateFilename; if not there
- * loads it from /conf/templateFile.
- * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
- * origin to help debug what's happening.
- * @param fs Filesystem
- * @param templatePath HDFS path for template
- * @param fallbackResource resource to fall back on, or "" for no fallback
- * @return loaded conf
- * @throws IOException IO problems
- * @throws FileNotFoundException if the path doesn't have a file and there
- * was no fallback.
- */
- public static Configuration loadTemplateConfiguration(FileSystem fs,
- Path templatePath,
- String fallbackResource)
- throws IOException {
- Configuration conf;
- String origin;
- if (fs.exists(templatePath)) {
- log.debug("Loading template configuration {}", templatePath);
- conf = loadConfiguration(fs, templatePath);
- origin = templatePath.toString();
- } else {
- if (fallbackResource.isEmpty()) {
- throw new FileNotFoundException("No config file found at " + templatePath);
- }
- log.debug("Template {} not found" +
- " -reverting to classpath resource {}", templatePath, fallbackResource);
- conf = new Configuration(false);
- conf.addResource(fallbackResource);
- origin = "Resource " + fallbackResource;
- }
- //force a get
- conf.get(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN);
- //now set the origin
- conf.set(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN, origin);
- return conf;
- }
-
-
- /**
- * For testing: dump a configuration
- * @param conf configuration
- * @return listing in key=value style
- */
- public static String dumpConfigToString(Configuration conf) {
- Set<String> sorted = sortedConfigKeys(conf);
-
- StringBuilder builder = new StringBuilder();
- for (String key : sorted) {
-
- builder.append(key)
- .append("=")
- .append(conf.get(key))
- .append("\n");
- }
- return builder.toString();
- }
-
- /**
- * Merge in one configuration above another
- * @param base base config
- * @param merge one to merge. This MUST be a non-default-load config to avoid
- * merge origin confusion
- * @param origin description of the origin for the put operation
- * @param overwrite flag to indicate new values should overwrite the predecessor
- * @return the base with the merged values
- */
- public static Configuration mergeConfigurations(Configuration base,
- Iterable<Map.Entry<String, String>> merge,
- String origin,
- boolean overwrite) {
- for (Map.Entry<String, String> entry : merge) {
- String key = entry.getKey();
- if (overwrite || base.get(key) == null) {
- base.set(key, entry.getValue(), origin);
- }
- }
- return base;
- }
-
- /**
- * Register a resource as a default resource.
- * Do not attempt to use this unless you understand that the
- * order in which default resources are loaded affects the outcome,
- * and that subclasses of Configuration often register new default
- * resources
- * @param resource the resource name
- * @return the URL or null
- */
- public static URL registerDefaultResource(String resource) {
- URL resURL = getResourceUrl(resource);
- if (resURL != null) {
- Configuration.addDefaultResource(resource);
- }
- return resURL;
- }
-
- /**
- * Load a configuration from a resource on this classpath.
- * If the resource is not found, an empty configuration is returned
- * @param resource the resource name
- * @return the loaded configuration.
- */
- public static Configuration loadFromResource(String resource) {
- Configuration conf = new Configuration(false);
- URL resURL = getResourceUrl(resource);
- if (resURL != null) {
- log.debug("loaded resources from {}", resURL);
- conf.addResource(resource);
- } else{
- log.debug("failed to find {} on the classpath", resource);
- }
- return conf;
-
- }
-
- /**
- * Get the URL to a resource, null if not on the CP
- * @param resource resource to look for
- * @return the URL or null
- */
- public static URL getResourceUrl(String resource) {
- return ConfigHelper.class.getClassLoader()
- .getResource(resource);
- }
-
- /**
- * Load a resource that must be on the classpath
- * @param resource the resource name
- * @return the loaded configuration
- * @throws FileNotFoundException if the resource is missing
- */
- public static Configuration loadMandatoryResource(String resource)
- throws FileNotFoundException {
- Configuration conf = new Configuration(false);
- URL resURL = getResourceUrl(resource);
- if (resURL != null) {
- log.debug("loaded resources from {}", resURL);
- conf.addResource(resource);
- } else {
- throw new FileNotFoundException(resource);
- }
- return conf;
- }
-
- /**
- * Propagate a property from a source to a dest config, with a best-effort
- * attempt at propagating the origin.
- * If the
- * @param dest destination
- * @param src source
- * @param key key to try to copy
- * @return true if the key was found and propagated
- */
- public static boolean propagate(Configuration dest,
- Configuration src,
- String key) {
- String val = src.get(key);
- if (val != null) {
- String[] origin = src.getPropertySources(key);
- if (origin != null && origin.length > 0) {
- dest.set(key, val, origin[0]);
- } else {
- dest.set(key, val);
- return true;
- }
- }
- return false;
- }
-
-
- /**
- * Take a configuration, return a hash map
- * @param conf conf
- * @return hash map
- */
- public static Map<String, String> buildMapFromConfiguration(Configuration conf) {
- Map<String, String> map = new HashMap<String, String>();
- return SliderUtils.mergeEntries(map, conf);
- }
-
- /**
- * This goes through the keyset of one configuration and retrieves each value
- * from a value source -a different or the same configuration. This triggers
- * the property resolution process of the value, resolving any variables against
- * in-config or inherited configurations
- * @param keysource source of keys
- * @param valuesource the source of values
- * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code>
- */
- public static Configuration resolveConfiguration(
- Iterable<Map.Entry<String, String>> keysource,
- Configuration valuesource) {
- Configuration result = new Configuration(false);
- for (Map.Entry<String, String> entry : keysource) {
- String key = entry.getKey();
- String value = valuesource.get(key);
- Preconditions.checkState(value != null,
- "no reference for \"%s\" in values", key);
- result.set(key, value);
- }
- return result;
- }
-
- /**
- * Register anything we consider deprecated
- */
- public static void registerDeprecatedConfigItems() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
deleted file mode 100644
index 43eb270..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ /dev/null
@@ -1,700 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
-import org.apache.slider.core.persist.Filenames;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS;
-import static org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS;
-
-public class CoreFileSystem {
- private static final Logger
- log = LoggerFactory.getLogger(CoreFileSystem.class);
-
- private static final String UTF_8 = "UTF-8";
-
- protected final FileSystem fileSystem;
- protected final Configuration configuration;
-
- public CoreFileSystem(FileSystem fileSystem, Configuration configuration) {
- Preconditions.checkNotNull(fileSystem,
- "Cannot create a CoreFileSystem with a null FileSystem");
- Preconditions.checkNotNull(configuration,
- "Cannot create a CoreFileSystem with a null Configuration");
- this.fileSystem = fileSystem;
- this.configuration = configuration;
- }
-
- public CoreFileSystem(Configuration configuration) throws IOException {
- Preconditions.checkNotNull(configuration,
- "Cannot create a CoreFileSystem with a null Configuration");
- this.fileSystem = FileSystem.get(configuration);
- this.configuration = configuration;
- }
-
- /**
- * Get the temp path for this cluster
- * @param clustername name of the cluster
- * @return path for temp files (is not purged)
- */
- public Path getTempPathForCluster(String clustername) {
- Path clusterDir = buildClusterDirPath(clustername);
- return new Path(clusterDir, SliderKeys.TMP_DIR_PREFIX);
- }
-
- /**
- * Returns the underlying FileSystem for this object.
- *
- * @return filesystem
- */
- public FileSystem getFileSystem() {
- return fileSystem;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("CoreFileSystem{");
- sb.append("fileSystem=").append(fileSystem.getUri());
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * Build up the path string for a cluster instance -no attempt to
- * create the directory is made
- *
- * @param clustername name of the cluster
- * @return the path for persistent data
- */
- public Path buildClusterDirPath(String clustername) {
- Preconditions.checkNotNull(clustername);
- Path path = getBaseApplicationPath();
- return new Path(path, SliderKeys.CLUSTER_DIRECTORY + "/" + clustername);
- }
-
- /**
- * Build up the path string for app def folder -no attempt to
- * create the directory is made
- *
- * @param clustername name of the cluster
- * @return the path for persistent data
- */
- public Path buildAppDefDirPath(String clustername) {
- Path path = buildClusterDirPath(clustername);
- return new Path(path, SliderKeys.APP_DEF_DIR);
- }
-
- /**
- * Build up the path string for addon folder -no attempt to
- * create the directory is made
- *
- * @param clustername name of the cluster
- * @return the path for persistent data
- */
- public Path buildAddonDirPath(String clustername, String addonId) {
- Preconditions.checkNotNull(addonId);
- Path path = buildClusterDirPath(clustername);
- return new Path(path, SliderKeys.ADDONS_DIR + "/" + addonId);
- }
-
- /**
- * Build up the path string for keytab install location -no attempt to
- * create the directory is made
- *
- * @return the path for keytab
- */
- public Path buildKeytabInstallationDirPath(String keytabFolder) {
- Preconditions.checkNotNull(keytabFolder);
- Path path = getBaseApplicationPath();
- return new Path(path, SliderKeys.KEYTAB_DIR + "/" + keytabFolder);
- }
-
- /**
- * Build up the path string for keytab install location -no attempt to
- * create the directory is made
- *
- * @return the path for keytab installation location
- */
- public Path buildKeytabPath(String keytabDir, String keytabName, String clusterName) {
- Path homePath = getHomeDirectory();
- Path baseKeytabDir;
- if (keytabDir != null) {
- baseKeytabDir = new Path(homePath, keytabDir);
- } else {
- baseKeytabDir = new Path(buildClusterDirPath(clusterName),
- SliderKeys.KEYTAB_DIR);
- }
- return keytabName == null ? baseKeytabDir :
- new Path(baseKeytabDir, keytabName);
- }
-
- /**
- * Build up the path string for resource install location -no attempt to
- * create the directory is made
- *
- * @return the path for resource
- */
- public Path buildResourcePath(String resourceFolder) {
- Preconditions.checkNotNull(resourceFolder);
- Path path = getBaseApplicationPath();
- return new Path(path, SliderKeys.RESOURCE_DIR + "/" + resourceFolder);
- }
-
- /**
- * Build up the path string for resource install location -no attempt to
- * create the directory is made
- *
- * @return the path for resource
- */
- public Path buildResourcePath(String dirName, String fileName) {
- Preconditions.checkNotNull(dirName);
- Preconditions.checkNotNull(fileName);
- Path path = getBaseApplicationPath();
- return new Path(path, SliderKeys.RESOURCE_DIR + "/" + dirName + "/" + fileName);
- }
-
- /**
- * Create a directory with the given permissions.
- *
- * @param dir directory
- * @param clusterPerms cluster permissions
- * @throws IOException IO problem
- * @throws BadClusterStateException any cluster state problem
- */
- public void createWithPermissions(Path dir, FsPermission clusterPerms) throws
- IOException,
- BadClusterStateException {
- if (fileSystem.isFile(dir)) {
- // HADOOP-9361 shows some filesystems don't correctly fail here
- throw new BadClusterStateException(
- "Cannot create a directory over a file %s", dir);
- }
- log.debug("mkdir {} with perms {}", dir, clusterPerms);
- //no mask whatoever
- fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");
- fileSystem.mkdirs(dir, clusterPerms);
- //and force set it anyway just to make sure
- fileSystem.setPermission(dir, clusterPerms);
- }
-
- /**
- * Get the permissions of a path
- *
- * @param path path to check
- * @return the permissions
- * @throws IOException any IO problem (including file not found)
- */
- public FsPermission getPathPermissions(Path path) throws IOException {
- FileStatus status = fileSystem.getFileStatus(path);
- return status.getPermission();
- }
-
- public FsPermission getInstanceDirectoryPermissions() {
- String clusterDirPermsOct =
- configuration.get(CLUSTER_DIRECTORY_PERMISSIONS,
- DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
- return new FsPermission(clusterDirPermsOct);
- }
-
- /**
- * Verify that the cluster directory is not present
- *
- * @param clustername name of the cluster
- * @param clusterDirectory actual directory to look for
- * @throws IOException trouble with FS
- * @throws SliderException If the directory exists
- */
- public void verifyClusterDirectoryNonexistent(String clustername,
- Path clusterDirectory)
- throws IOException, SliderException {
- if (fileSystem.exists(clusterDirectory)) {
- throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
- ErrorStrings.PRINTF_E_INSTANCE_ALREADY_EXISTS, clustername,
- clusterDirectory);
- }
- }
- /**
- * Verify that the given directory is not present
- *
- * @param clusterDirectory actual directory to look for
- * @throws IOException trouble with FS
- * @throws SliderException If the directory exists
- */
- public void verifyDirectoryNonexistent(Path clusterDirectory) throws
- IOException,
- SliderException {
- if (fileSystem.exists(clusterDirectory)) {
-
- log.error("Dir {} exists: {}",
- clusterDirectory,
- listFSDir(clusterDirectory));
- throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
- ErrorStrings.PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS,
- clusterDirectory);
- }
- }
-
- /**
- * Verify that a user has write access to a directory.
- * It does this by creating then deleting a temp file
- *
- * @param dirPath actual directory to look for
- * @throws FileNotFoundException file not found
- * @throws IOException trouble with FS
- * @throws BadClusterStateException if the directory is not writeable
- */
- public void verifyDirectoryWriteAccess(Path dirPath) throws IOException,
- SliderException {
- verifyPathExists(dirPath);
- Path tempFile = new Path(dirPath, "tmp-file-for-checks");
- try {
- FSDataOutputStream out ;
- out = fileSystem.create(tempFile, true);
- IOUtils.closeStream(out);
- fileSystem.delete(tempFile, false);
- } catch (IOException e) {
- log.warn("Failed to create file {}: {}", tempFile, e);
- throw new BadClusterStateException(e,
- "Unable to write to directory %s : %s", dirPath, e.toString());
- }
- }
-
- /**
- * Verify that a path exists
- * @param path path to check
- * @throws FileNotFoundException file not found
- * @throws IOException trouble with FS
- */
- public void verifyPathExists(Path path) throws IOException {
- if (!fileSystem.exists(path)) {
- throw new FileNotFoundException(path.toString());
- }
- }
-
- /**
- * Verify that a path exists
- * @param path path to check
- * @throws FileNotFoundException file not found or is not a file
- * @throws IOException trouble with FS
- */
- public void verifyFileExists(Path path) throws IOException {
- FileStatus status = fileSystem.getFileStatus(path);
-
- if (!status.isFile()) {
- throw new FileNotFoundException("Not a file: " + path.toString());
- }
- }
-
- /**
- * Given a path, check if it exists and is a file
- *
- * @param path
- * absolute path to the file to check
- * @returns true if and only if path exists and is a file, false for all other
- * reasons including if file check throws IOException
- */
- public boolean isFile(Path path) {
- boolean isFile = false;
- try {
- FileStatus status = fileSystem.getFileStatus(path);
- if (status.isFile()) {
- isFile = true;
- }
- } catch (IOException e) {
- // ignore, isFile is already set to false
- }
- return isFile;
- }
-
- /**
- * Get the base path
- *
- * @return the base path optionally configured by
- * {@link SliderXmlConfKeys#KEY_SLIDER_BASE_PATH}
- */
- public Path getBaseApplicationPath() {
- String configuredBasePath = configuration.get(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH);
- return configuredBasePath != null ? new Path(configuredBasePath) :
- new Path(getHomeDirectory(), SliderKeys.SLIDER_BASE_DIRECTORY);
- }
-
- /**
- * Get slider dependency parent dir in HDFS
- *
- * @return the parent dir path of slider.tar.gz in HDFS
- */
- public Path getDependencyPath() {
- // FIXME: 3/20/17 HDP ???????????
- String parentDir = (SliderUtils.isHdp()) ? SliderKeys.SLIDER_DEPENDENCY_HDP_PARENT_DIR
- + SliderKeys.SLIDER_DEPENDENCY_DIR
- : SliderKeys.SLIDER_DEPENDENCY_DIR;
- Path dependencyPath = new Path(String.format(parentDir,
- SliderUtils.getSliderVersion()));
- return dependencyPath;
- }
-
- /**
- * Get slider.tar.gz absolute filepath in HDFS
- *
- * @return the absolute path to slider.tar.gz in HDFS
- */
- public Path getDependencyTarGzip() {
- Path dependencyLibAmPath = getDependencyPath();
- Path dependencyLibTarGzip = new Path(
- dependencyLibAmPath.toUri().toString(),
- SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME
- + SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT);
- return dependencyLibTarGzip;
- }
-
- public Path getHomeDirectory() {
- return fileSystem.getHomeDirectory();
- }
-
- public boolean maybeAddImagePath(Map<String, LocalResource> localResources,
- Path imagePath) throws IOException {
- if (imagePath != null) {
- LocalResource resource = createAmResource(imagePath,
- LocalResourceType.ARCHIVE);
- localResources.put(SliderKeys.LOCAL_TARBALL_INSTALL_SUBDIR, resource);
- return true;
- } else {
- return false;
- }
- }
-
- public boolean maybeAddImagePath(Map<String, LocalResource> localResources,
- String imagePath) throws IOException {
-
- return imagePath != null &&
- maybeAddImagePath(localResources, new Path(imagePath));
- }
-
-
-
-
- /**
- * Create an AM resource from the
- *
- * @param destPath dest path in filesystem
- * @param resourceType resource type
- * @return the resource set up wih application-level visibility and the
- * timestamp & size set from the file stats.
- */
- public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException {
- FileStatus destStatus = fileSystem.getFileStatus(destPath);
- LocalResource amResource = Records.newRecord(LocalResource.class);
- amResource.setType(resourceType);
- // Set visibility of the resource
- // Setting to most private option
- amResource.setVisibility(LocalResourceVisibility.APPLICATION);
- // Set the resource to be copied over
- amResource.setResource(ConverterUtils.getYarnUrlFromPath(fileSystem
- .resolvePath(destStatus.getPath())));
- // Set timestamp and length of file so that the framework
- // can do basic sanity checks for the local resource
- // after it has been copied over to ensure it is the same
- // resource the client intended to use with the application
- amResource.setTimestamp(destStatus.getModificationTime());
- amResource.setSize(destStatus.getLen());
- return amResource;
- }
-
- /**
- * Register all files under a fs path as a directory to push out
- *
- * @param srcDir src dir
- * @param destRelativeDir dest dir (no trailing /)
- * @return the map of entries
- */
- public Map<String, LocalResource> submitDirectory(Path srcDir, String destRelativeDir) throws IOException {
- //now register each of the files in the directory to be
- //copied to the destination
- FileStatus[] fileset = fileSystem.listStatus(srcDir);
- Map<String, LocalResource> localResources =
- new HashMap<String, LocalResource>(fileset.length);
- for (FileStatus entry : fileset) {
-
- LocalResource resource = createAmResource(entry.getPath(),
- LocalResourceType.FILE);
- String relativePath = destRelativeDir + "/" + entry.getPath().getName();
- localResources.put(relativePath, resource);
- }
- return localResources;
- }
-
- /**
- * Submit a JAR containing a specific class, returning
- * the resource to be mapped in
- *
- * @param clazz class to look for
- * @param subdir subdirectory (expected to end in a "/")
- * @param jarName <i>At the destination</i>
- * @return the local resource ref
- * @throws IOException trouble copying to HDFS
- */
- public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName)
- throws IOException, SliderException {
- File localFile = SliderUtils.findContainingJarOrFail(clazz);
- return submitFile(localFile, tempPath, subdir, jarName);
- }
-
- /**
- * Submit a local file to the filesystem references by the instance's cluster
- * filesystem
- *
- * @param localFile filename
- * @param subdir subdirectory (expected to end in a "/")
- * @param destFileName destination filename
- * @return the local resource ref
- * @throws IOException trouble copying to HDFS
- */
- public LocalResource submitFile(File localFile, Path tempPath, String subdir, String destFileName)
- throws IOException {
- Path src = new Path(localFile.toString());
- Path subdirPath = new Path(tempPath, subdir);
- fileSystem.mkdirs(subdirPath);
- Path destPath = new Path(subdirPath, destFileName);
- log.debug("Copying {} (size={} bytes) to {}", localFile, localFile.length(), destPath);
-
- fileSystem.copyFromLocalFile(false, true, src, destPath);
-
- // Set the type of resource - file or archive
- // archives are untarred at destination
- // we don't need the jar file to be untarred for now
- return createAmResource(destPath, LocalResourceType.FILE);
- }
-
- /**
- * Submit the AM tar.gz resource referenced by the instance's cluster
- * filesystem. Also, update the providerResources object with the new
- * resource.
- *
- * @param providerResources
- * the provider resource map to be updated
- * @throws IOException
- * trouble copying to HDFS
- */
- public void submitTarGzipAndUpdate(
- Map<String, LocalResource> providerResources) throws IOException,
- BadClusterStateException {
- Path dependencyLibTarGzip = getDependencyTarGzip();
- LocalResource lc = createAmResource(dependencyLibTarGzip,
- LocalResourceType.ARCHIVE);
- providerResources.put(SliderKeys.SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK, lc);
- }
-
- public void copyLocalFileToHdfs(File localPath,
- Path destPath, FsPermission fp)
- throws IOException {
- if (localPath == null || destPath == null) {
- throw new IOException("Either localPath or destPath is null");
- }
- fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
- "000");
- fileSystem.mkdirs(destPath.getParent(), fp);
- log.info("Copying file {} to {}", localPath.toURI(), destPath.toUri());
-
- fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()),
- destPath);
- // set file permissions of the destPath
- fileSystem.setPermission(destPath, fp);
- }
-
- public void copyHdfsFileToLocal(Path hdfsPath, File destFile)
- throws IOException {
- if (hdfsPath == null || destFile == null) {
- throw new IOException("Either hdfsPath or destPath is null");
- }
- log.info("Copying file {} to {}", hdfsPath.toUri(), destFile.toURI());
-
- Path destPath = new Path(destFile.getPath());
- fileSystem.copyToLocalFile(hdfsPath, destPath);
- }
-
- /**
- * list entries in a filesystem directory
- *
- * @param path directory
- * @return a listing, one to a line
- * @throws IOException
- */
- public String listFSDir(Path path) throws IOException {
- FileStatus[] stats = fileSystem.listStatus(path);
- StringBuilder builder = new StringBuilder();
- for (FileStatus stat : stats) {
- builder.append(stat.getPath().toString())
- .append("\t")
- .append(stat.getLen())
- .append("\n");
- }
- return builder.toString();
- }
-
- /**
- * List all application instances persisted for this user, giving the
- * path. The instance name is the last element in the path
- * @return a possibly empty map of application instance names to paths
- */
- public Map<String, Path> listPersistentInstances() throws IOException {
- FileSystem fs = getFileSystem();
- Path path = new Path(getBaseApplicationPath(), SliderKeys.CLUSTER_DIRECTORY);
- log.debug("Looking for all persisted application at {}", path.toString());
- if (!fs.exists(path)) {
- // special case: no instances have ever been created
- return new HashMap<String, Path>(0);
- }
- FileStatus[] statuses = fs.listStatus(path);
- Map<String, Path> instances = new HashMap<String, Path>(statuses.length);
-
- // enum the child entries
- for (FileStatus status : statuses) {
- if (status.isDirectory()) {
- // for directories, look for an internal.json underneath
- Path child = status.getPath();
- Path internalJson = new Path(child, Filenames.INTERNAL);
- if (fs.exists(internalJson)) {
- // success => this is an instance
- instances.put(child.getName(), child);
- } else {
- log.info("Malformed cluster found at {}. It does not appear to be a valid persisted instance.",
- child.toString());
- }
- }
- }
- return instances;
- }
-
- public void touch(Path path, boolean overwrite) throws IOException {
- FSDataOutputStream out = null;
- try {
- out = fileSystem.create(path, overwrite);
- } finally {
- IOUtils.closeStream(out);
- }
- }
-
- public void cat(Path path, boolean overwrite, String data) throws IOException {
- FSDataOutputStream out = null;
- try {
- out = fileSystem.create(path, overwrite);
- byte[] bytes = data.getBytes(Charset.forName("UTF-8"));
- out.write(bytes);
- } finally {
- IOUtils.closeStream(out);
- }
- }
-
- public String cat(Path path) throws IOException {
- FileStatus status = fileSystem.getFileStatus(path);
- byte[] b = new byte[(int) status.getLen()];
- FSDataInputStream in = null;
- try {
- in = fileSystem.open(path);
- int count = in.read(b);
- return new String(b, 0, count, UTF_8);
- } finally {
- IOUtils.closeStream(in);
- }
- }
-
- /**
- * Create a path that must exist in the cluster fs
- * @param uri uri to create
- * @return the path
- * @throws SliderException if the path does not exist
- */
- public Path createPathThatMustExist(String uri) throws
- SliderException, IOException {
- Preconditions.checkNotNull(uri);
- Path path = new Path(uri);
- verifyPathExists(path);
- return path;
- }
-
- /**
- * Locate an application conf json in the FS. This includes a check to verify
- * that the file is there.
- *
- * @param clustername name of the cluster
- * @return the path to the spec.
- * @throws IOException IO problems
- * @throws SliderException if the path isn't there
- */
- public Path locateInstanceDefinition(String clustername) throws IOException,
- SliderException {
- Path clusterDirectory = buildClusterDirPath(clustername);
- Path appConfPath =
- new Path(clusterDirectory, Filenames.APPCONF);
- verifyClusterSpecExists(clustername, appConfPath);
- return appConfPath;
- }
-
- /**
- * Verify that a cluster specification exists
- * @param clustername name of the cluster (For errors only)
- * @param clusterSpecPath cluster specification path
- * @throws IOException IO problems
- * @throws SliderException if the cluster specification is not present
- */
- public void verifyClusterSpecExists(String clustername, Path clusterSpecPath)
- throws IOException,
- SliderException {
- if (!fileSystem.isFile(clusterSpecPath)) {
- log.debug("Missing specification file {}", clusterSpecPath);
- throw UnknownApplicationInstanceException.unknownInstance(
- clustername + "\n (definition not found at " + clusterSpecPath);
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Duration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Duration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Duration.java
deleted file mode 100644
index e5fa424..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Duration.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import java.io.Closeable;
-
-/**
- * A duration in milliseconds. This class can be used
- * to count time, and to be polled to see if a time limit has
- * passed.
- */
-public class Duration implements Closeable {
- public long start, finish;
- public final long limit;
-
- /**
- * Create a duration instance with a limit of 0
- */
- public Duration() {
- this(0);
- }
-
- /**
- * Create a duration with a limit specified in millis
- * @param limit duration in milliseconds
- */
- public Duration(long limit) {
- this.limit = limit;
- }
-
- /**
- * Start
- * @return self
- */
- public Duration start() {
- start = now();
- return this;
- }
-
- /**
- * The close operation relays to {@link #finish()}.
- * Implementing it allows Duration instances to be automatically
- * finish()'d in Java7 try blocks for when used in measuring durations.
- */
- @Override
- public final void close() {
- finish();
- }
-
- public void finish() {
- finish = now();
- }
-
- protected long now() {
- return System.nanoTime()/1000000;
- }
-
- public long getInterval() {
- return finish - start;
- }
-
- /**
- * return true if the limit has been exceeded
- * @return true if a limit was set and the current time
- * exceeds it.
- */
- public boolean getLimitExceeded() {
- return limit >= 0 && ((now() - start) > limit);
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append("Duration");
- if (finish >= start) {
- builder.append(" finished at ").append(getInterval()).append(" millis;");
- } else {
- if (start > 0) {
- builder.append(" started but not yet finished;");
- } else {
- builder.append(" unstarted;");
- }
- }
- if (limit > 0) {
- builder.append(" limit: ").append(limit).append(" millis");
- if (getLimitExceeded()) {
- builder.append(" - exceeded");
- }
- }
- return builder.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
deleted file mode 100644
index 235d3da..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.common.tools;
-
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * a scanner which can take an input string for a range or scan the lot.
- */
-public class PortScanner {
- private static Pattern NUMBER_RANGE = Pattern.compile("^(\\d+)\\s*-\\s*(\\d+)$");
- private static Pattern SINGLE_NUMBER = Pattern.compile("^\\d+$");
-
- private List<Integer> remainingPortsToCheck;
-
- public PortScanner() {
- }
-
- public void setPortRange(String input) throws BadConfigException {
- // first split based on commas
- Set<Integer> inputPorts= new TreeSet<Integer>();
- String[] ranges = input.split(",");
- for ( String range : ranges ) {
- if (range.trim().isEmpty()) {
- continue;
- }
- Matcher m = SINGLE_NUMBER.matcher(range.trim());
- if (m.find()) {
- inputPorts.add(Integer.parseInt(m.group()));
- continue;
- }
- m = NUMBER_RANGE.matcher(range.trim());
- if (m.find()) {
- String[] boundaryValues = m.group(0).split("-");
- int start = Integer.parseInt(boundaryValues[0].trim());
- int end = Integer.parseInt(boundaryValues[1].trim());
- if (end < start) {
- throw new BadConfigException("End of port range is before start: "
- + range + " in input: " + input);
- }
- for (int i = start; i < end + 1; i++) {
- inputPorts.add(i);
- }
- continue;
- }
- throw new BadConfigException("Bad port range: " + range + " in input: "
- + input);
- }
- if (inputPorts.size() == 0) {
- throw new BadConfigException("No ports found in range: " + input);
- }
- this.remainingPortsToCheck = new ArrayList<Integer>(inputPorts);
- }
-
- public List<Integer> getRemainingPortsToCheck() {
- return remainingPortsToCheck;
- }
-
- public int getAvailablePort() throws SliderException, IOException {
- if (remainingPortsToCheck != null) {
- return getAvailablePortViaPortArray();
- } else {
- return SliderUtils.getOpenPort();
- }
- }
-
- private int getAvailablePortViaPortArray() throws SliderException {
- boolean found = false;
- int availablePort = -1;
- Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator();
- while (portsToCheck.hasNext() && !found) {
- int portToCheck = portsToCheck.next();
- found = SliderUtils.isPortAvailable(portToCheck);
- if (found) {
- availablePort = portToCheck;
- portsToCheck.remove();
- }
- }
-
- if (availablePort < 0) {
- throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
- "No available ports found in configured range {}",
- remainingPortsToCheck);
- }
-
- return availablePort;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java
deleted file mode 100644
index 40b07bd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import java.io.IOException;
-
-/**
- * Extends Core Filesystem with operations to manipulate ClusterDescription
- * persistent state
- */
-public class SliderFileSystem extends CoreFileSystem {
-
- Path appDir = null;
-
- public SliderFileSystem(FileSystem fileSystem,
- Configuration configuration) {
- super(fileSystem, configuration);
- }
-
- public SliderFileSystem(Configuration configuration) throws IOException {
- super(configuration);
- }
-
- public void setAppDir(Path appDir) {
- this.appDir = appDir;
- }
-
- public Path getAppDir() {
- return this.appDir;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
deleted file mode 100644
index 1fa07ce..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.net.HostAndPort;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class ZookeeperUtils {
- public static final int DEFAULT_PORT = 2181;
-
- public static String buildConnectionString(String zkHosts, int port) {
- String zkPort = Integer.toString(port);
- //parse the hosts
- String[] hostlist = zkHosts.split(",", 0);
- String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
- return quorum;
- }
-
- /**
- * Take a quorum list and split it to (trimmed) pairs
- * @param hostPortQuorumList list of form h1:port, h2:port2,...
- * @return a possibly empty list of values between commas. They may not be
- * valid hostname:port pairs
- */
- public static List<String> splitToPairs(String hostPortQuorumList) {
- // split an address hot
- String[] strings = StringUtils.getStrings(hostPortQuorumList);
- int len = 0;
- if (strings != null) {
- len = strings.length;
- }
- List<String> tuples = new ArrayList<String>(len);
- if (strings != null) {
- for (String s : strings) {
- tuples.add(s.trim());
- }
- }
- return tuples;
- }
-
- /**
- * Split a quorum list into a list of hostnames and ports
- * @param hostPortQuorumList split to a list of hosts and ports
- * @return a list of values
- */
- public static List<HostAndPort> splitToHostsAndPorts(String hostPortQuorumList) {
- // split an address hot
- String[] strings = StringUtils.getStrings(hostPortQuorumList);
- int len = 0;
- if (strings != null) {
- len = strings.length;
- }
- List<HostAndPort> list = new ArrayList<HostAndPort>(len);
- if (strings != null) {
- for (String s : strings) {
- list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
- }
- }
- return list;
- }
-
- /**
- * Build up to a hosts only list
- * @param hostAndPorts
- * @return a list of the hosts only
- */
- public static String buildHostsOnlyList(List<HostAndPort> hostAndPorts) {
- StringBuilder sb = new StringBuilder();
- for (HostAndPort hostAndPort : hostAndPorts) {
- sb.append(hostAndPort.getHostText()).append(",");
- }
- if (sb.length() > 0) {
- sb.delete(sb.length() - 1, sb.length());
- }
- return sb.toString();
- }
-
- public static String buildQuorumEntry(HostAndPort hostAndPort,
- int defaultPort) {
- String s = hostAndPort.toString();
- if (hostAndPort.hasPort()) {
- return s;
- } else {
- return s + ":" + defaultPort;
- }
- }
-
- /**
- * Build a quorum list, injecting a ":defaultPort" ref if needed on
- * any entry without one
- * @param hostAndPorts
- * @param defaultPort
- * @return
- */
- public static String buildQuorum(List<HostAndPort> hostAndPorts, int defaultPort) {
- List<String> entries = new ArrayList<String>(hostAndPorts.size());
- for (HostAndPort hostAndPort : hostAndPorts) {
- entries.add(buildQuorumEntry(hostAndPort, defaultPort));
- }
- return SliderUtils.join(entries, ",", false);
- }
-
- public static String convertToHostsOnlyList(String quorum) throws
- BadConfigException {
- List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
- return ZookeeperUtils.buildHostsOnlyList(hostAndPorts);
- }
-
- public static List<HostAndPort> splitToHostsAndPortsStrictly(String quorum) throws
- BadConfigException {
- List<HostAndPort> hostAndPorts =
- ZookeeperUtils.splitToHostsAndPorts(quorum);
- if (hostAndPorts.isEmpty()) {
- throw new BadConfigException("empty zookeeper quorum");
- }
- return hostAndPorts;
- }
-
- public static int getFirstPort(String quorum, int defVal) throws
- BadConfigException {
- List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
- int port = hostAndPorts.get(0).getPortOrDefault(defVal);
- return port;
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto
deleted file mode 100644
index 0a21c24..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.hadoop.yarn.proto";
-option java_outer_classname = "ClientAMProtocol";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.yarn;
-
-service ClientAMProtocolService {
- rpc flexComponents(FlexComponentsRequestProto) returns (FlexComponentsResponseProto);
- rpc getStatus(GetStatusRequestProto) returns (GetStatusResponseProto);
- rpc stop(StopRequestProto) returns (StopResponseProto);
-}
-
-message FlexComponentsRequestProto {
- repeated ComponentCountProto components = 1;
-}
-
-message ComponentCountProto {
- optional string name = 1;
- optional int64 numberOfContainers = 2;
-}
-
-message FlexComponentsResponseProto{
-}
-
-message GetStatusRequestProto {
-
-}
-message GetStatusResponseProto {
- optional string status = 1;
-}
-
-message StopRequestProto {
-
-}
-
-message StopResponseProto {
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
deleted file mode 100644
index 4fa81ee..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import com.google.common.base.Supplier;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.NMClient;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
-import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.service.component.ComponentState;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.TimeoutException;
-
-import static org.mockito.Mockito.mock;
-
-public class MockServiceAM extends ServiceMaster {
-
- Application application;
- // The list of containers fed by tests to be returned on
- // AMRMClientCallBackHandler#onContainersAllocated
- final List<Container> feedContainers =
- Collections.synchronizedList(new LinkedList<>());
-
- public MockServiceAM(Application application) {
- super(application.getName());
- this.application = application;
- }
-
-
- @Override
- protected ContainerId getAMContainerId()
- throws BadClusterStateException {
- return ContainerId.newContainerId(ApplicationAttemptId
- .newInstance(ApplicationId.fromString(application.getId()), 1), 1);
- }
-
- @Override
- protected Path getAppDir() {
- Path path = new Path(new Path("target", "apps"), application.getName());
- System.out.println("Application path: " + path);
- return path;
- }
-
- @Override
- protected ServiceScheduler createServiceScheduler(ServiceContext context)
- throws IOException, YarnException {
- return new ServiceScheduler(context) {
-
- @Override
- protected YarnRegistryViewForProviders createYarnRegistryOperations(
- ServiceContext context, RegistryOperations registryClient) {
- return mock(YarnRegistryViewForProviders.class);
- }
-
- @Override
- protected AMRMClientAsync<AMRMClient.ContainerRequest> createAMRMClient() {
- AMRMClientImpl client1 = new AMRMClientImpl() {
- @Override public AllocateResponse allocate(float progressIndicator)
- throws YarnException, IOException {
-
- AllocateResponse.AllocateResponseBuilder builder =
- AllocateResponse.newBuilder();
- synchronized (feedContainers) {
- if (feedContainers.isEmpty()) {
- System.out.println("Allocating........ no containers");
- return builder.build();
- } else {
- // The AMRMClient will return containers for compoenent that are
- // at FLEXING state
- List<Container> allocatedContainers = new LinkedList<>();
- Iterator<Container> itor = feedContainers.iterator();
- while (itor.hasNext()) {
- Container c = itor.next();
- org.apache.hadoop.yarn.service.component.Component component =
- componentsById.get(c.getAllocationRequestId());
- if (component.getState() == ComponentState.FLEXING) {
- System.out.println("Allocated container " + c.getId());
- allocatedContainers.add(c);
- itor.remove();
- }
- }
- return builder.allocatedContainers(allocatedContainers).build();
- }
- }
- }
-
- @Override
- public RegisterApplicationMasterResponse registerApplicationMaster(
- String appHostName, int appHostPort, String appTrackingUrl) {
- return mock(RegisterApplicationMasterResponse.class);
- }
-
- @Override public void unregisterApplicationMaster(
- FinalApplicationStatus appStatus, String appMessage,
- String appTrackingUrl) {
- // DO nothing
- }
- };
-
- return AMRMClientAsync
- .createAMRMClientAsync(client1, 1000,
- this.new AMRMClientCallback());
- }
-
- @Override
- public NMClientAsync createNMClient() {
- NMClientAsync nmClientAsync = super.createNMClient();
- nmClientAsync.setClient(mock(NMClient.class));
- return nmClientAsync;
- }
- };
- }
-
- @Override protected void loadApplicationJson(ServiceContext context,
- SliderFileSystem fs) throws IOException {
- context.application = application;
- }
-
- /**
- *
- * @param application The application for the component
- * @param id The id for the container
- * @param compName The component to which the container is fed
- * @return
- */
- public Container feedContainerToComp(Application application, int id,
- String compName) {
- ApplicationId applicationId = ApplicationId.fromString(application.getId());
- ContainerId containerId = ContainerId
- .newContainerId(ApplicationAttemptId.newInstance(applicationId, 1), id);
- NodeId nodeId = NodeId.newInstance("localhost", 1234);
- Container container = Container
- .newInstance(containerId, nodeId, "localhost",
- Resource.newInstance(100, 1), Priority.newInstance(0), null);
-
- long allocateId =
- context.scheduler.getAllComponents().get(compName).getAllocateId();
- container.setAllocationRequestId(allocateId);
- synchronized (feedContainers) {
- feedContainers.add(container);
- }
- return container;
- }
-
- public void flexComponent(String compName, long numberOfContainers)
- throws IOException {
- ClientAMProtocol.ComponentCountProto componentCountProto =
- ClientAMProtocol.ComponentCountProto.newBuilder().setName(compName)
- .setNumberOfContainers(numberOfContainers).build();
- ClientAMProtocol.FlexComponentsRequestProto requestProto =
- ClientAMProtocol.FlexComponentsRequestProto.newBuilder()
- .addComponents(componentCountProto).build();
- context.clientAMService.flexComponents(requestProto);
- }
-
- public Component getComponent(String compName) {
- return context.scheduler.getAllComponents().get(compName);
- }
-
- public void waitForDependenciesSatisfied(String compName)
- throws TimeoutException, InterruptedException {
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override public Boolean get() {
- return context.scheduler.getAllComponents().get(compName)
- .areDependenciesReady();
- }
- }, 1000, 20000);
- }
-
- public void waitForNumDesiredContainers(String compName,
- int numDesiredContainers) throws TimeoutException, InterruptedException {
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override public Boolean get() {
- return context.scheduler.getAllComponents().get(compName)
- .getNumDesiredInstances() == numDesiredContainers;
- }
- }, 1000, 20000);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
deleted file mode 100644
index 73172bf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
-import org.codehaus.jackson.map.PropertyNamingStrategy;
-
-public class ServiceTestUtils {
-
- public static final JsonSerDeser<Application> JSON_SER_DESER =
- new JsonSerDeser<>(Application.class,
- PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
-
- // Example service definition
- // 2 components, each of which has 2 containers.
- protected Application createExampleApplication() {
- Application exampleApp = new Application();
- exampleApp.setName("example-app");
- exampleApp.addComponent(createComponent("compa"));
- exampleApp.addComponent(createComponent("compb"));
- return exampleApp;
- }
-
- protected Component createComponent(String name) {
- return createComponent(name, 2L, "sleep 1000");
- }
-
- protected Component createComponent(String name, long numContainers,
- String command) {
- Component comp1 = new Component();
- comp1.setNumberOfContainers(numContainers);
- comp1.setLaunchCommand(command);
- comp1.setName(name);
- Resource resource = new Resource();
- comp1.setResource(resource);
- resource.setMemory("128");
- resource.setCpus(1);
- return comp1;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
deleted file mode 100644
index 1a22875..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
+++ /dev/null
@@ -1,529 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME;
-import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
-import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*;
-import static org.easymock.EasyMock.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-/**
- * Test for ServiceApiUtil helper methods.
- */
-public class TestServiceApiUtil {
- private static final Logger LOG = LoggerFactory
- .getLogger(TestServiceApiUtil.class);
- private static final String EXCEPTION_PREFIX = "Should have thrown " +
- "exception: ";
- private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
- "exception: ";
-
- private static final String LEN_64_STR =
- "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01";
-
- private static final YarnConfiguration CONF_DEFAULT_DNS = new
- YarnConfiguration();
- private static final YarnConfiguration CONF_DNS_ENABLED = new
- YarnConfiguration();
-
- @BeforeClass
- public static void init() {
- CONF_DNS_ENABLED.setBoolean(RegistryConstants.KEY_DNS_ENABLED, true);
- }
-
- @Test(timeout = 90000)
- public void testResourceValidation() throws Exception {
- assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR
- .length());
-
- SliderFileSystem sfs = initMock(null);
-
- Application app = new Application();
-
- // no name
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no name");
- } catch (IllegalArgumentException e) {
- assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
- }
-
- // bad format name
- String[] badNames = {"4finance", "Finance", "finance@home", LEN_64_STR};
- for (String badName : badNames) {
- app.setName(badName);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName);
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(
- ERROR_APPLICATION_NAME_INVALID_FORMAT, badName), e.getMessage());
- }
- }
-
- // launch command not specified
- app.setName(LEN_64_STR);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DEFAULT_DNS);
- Assert.fail(EXCEPTION_PREFIX + "application with no launch command");
- } catch (IllegalArgumentException e) {
- assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
- e.getMessage());
- }
-
- // launch command not specified
- app.setName(LEN_64_STR.substring(0, RegistryConstants
- .MAX_FQDN_LABEL_LENGTH));
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no launch command");
- } catch (IllegalArgumentException e) {
- assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
- e.getMessage());
- }
-
- // resource not specified
- app.setLaunchCommand("sleep 3600");
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no resource");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID,
- DEFAULT_COMPONENT_NAME), e.getMessage());
- }
-
- // memory not specified
- Resource res = new Resource();
- app.setResource(res);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no memory");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID,
- DEFAULT_COMPONENT_NAME), e.getMessage());
- }
-
- // invalid no of cpus
- res.setMemory("100mb");
- res.setCpus(-2);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(
- EXCEPTION_PREFIX + "application with invalid no of cpus");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE,
- DEFAULT_COMPONENT_NAME), e.getMessage());
- }
-
- // number of containers not specified
- res.setCpus(2);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no container count");
- } catch (IllegalArgumentException e) {
- Assert.assertTrue(e.getMessage()
- .contains(ERROR_CONTAINERS_COUNT_INVALID));
- }
-
- // specifying profile along with cpus/memory raises exception
- res.setProfile("hbase_finance_large");
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX
- + "application with resource profile along with cpus/memory");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(RestApiErrorMessages
- .ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
- DEFAULT_COMPONENT_NAME),
- e.getMessage());
- }
-
- // currently resource profile alone is not supported.
- // TODO: remove the next test once resource profile alone is supported.
- res.setCpus(null);
- res.setMemory(null);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with resource profile only");
- } catch (IllegalArgumentException e) {
- assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET,
- e.getMessage());
- }
-
- // unset profile here and add cpus/memory back
- res.setProfile(null);
- res.setCpus(2);
- res.setMemory("2gb");
-
- // null number of containers
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "null number of containers");
- } catch (IllegalArgumentException e) {
- Assert.assertTrue(e.getMessage()
- .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
- }
-
- // negative number of containers
- app.setNumberOfContainers(-1L);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "negative number of containers");
- } catch (IllegalArgumentException e) {
- Assert.assertTrue(e.getMessage()
- .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
- }
-
- // everything valid here
- app.setNumberOfContainers(5L);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- LOG.error("application attributes specified should be valid here", e);
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
- }
-
- @Test
- public void testArtifacts() throws IOException {
- SliderFileSystem sfs = initMock(null);
-
- Application app = new Application();
- app.setName("name");
- Resource res = new Resource();
- app.setResource(res);
- res.setMemory("512M");
- app.setNumberOfContainers(3L);
-
- // no artifact id fails with default type
- Artifact artifact = new Artifact();
- app.setArtifact(artifact);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
- } catch (IllegalArgumentException e) {
- assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
- }
-
- // no artifact id fails with APPLICATION type
- artifact.setType(Artifact.TypeEnum.APPLICATION);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
- } catch (IllegalArgumentException e) {
- assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
- }
-
- // no artifact id fails with TARBALL type
- artifact.setType(Artifact.TypeEnum.TARBALL);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
- } catch (IllegalArgumentException e) {
- assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
- }
-
- // everything valid here
- artifact.setType(Artifact.TypeEnum.DOCKER);
- artifact.setId("docker.io/centos:centos7");
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- LOG.error("application attributes specified should be valid here", e);
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- // defaults assigned
- assertEquals(app.getComponents().get(0).getName(),
- DEFAULT_COMPONENT_NAME);
- assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME);
- }
-
- private static Resource createValidResource() {
- Resource res = new Resource();
- res.setMemory("512M");
- return res;
- }
-
- private static Component createValidComponent(String compName) {
- Component comp = new Component();
- comp.setName(compName);
- comp.setResource(createValidResource());
- comp.setNumberOfContainers(1L);
- return comp;
- }
-
- private static Application createValidApplication(String compName) {
- Application app = new Application();
- app.setLaunchCommand("sleep 3600");
- app.setName("name");
- app.setResource(createValidResource());
- app.setNumberOfContainers(1L);
- if (compName != null) {
- app.addComponent(createValidComponent(compName));
- }
- return app;
- }
-
- private static SliderFileSystem initMock(Application ext) throws IOException {
- SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
- FileSystem mockFs = createNiceMock(FileSystem.class);
- JsonSerDeser<Application> jsonSerDeser = createNiceMock(JsonSerDeser
- .class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- if (ext != null) {
- expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
- .anyTimes();
- }
- replay(sfs, mockFs, jsonSerDeser);
- ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
- return sfs;
- }
-
- @Test
- public void testExternalApplication() throws IOException {
- Application ext = createValidApplication("comp1");
- SliderFileSystem sfs = initMock(ext);
-
- Application app = createValidApplication(null);
-
- Artifact artifact = new Artifact();
- artifact.setType(Artifact.TypeEnum.APPLICATION);
- artifact.setId("id");
- app.setArtifact(artifact);
-
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- assertEquals(1, app.getComponents().size());
- assertNotNull(app.getComponent("comp1"));
- }
-
- @Test
- public void testDuplicateComponents() throws IOException {
- SliderFileSystem sfs = initMock(null);
-
- String compName = "comp1";
- Application app = createValidApplication(compName);
- app.addComponent(createValidComponent(compName));
-
- // duplicate component name fails
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with component collision");
- } catch (IllegalArgumentException e) {
- assertEquals("Component name collision: " + compName, e.getMessage());
- }
- }
-
- @Test
- public void testExternalDuplicateComponent() throws IOException {
- Application ext = createValidApplication("comp1");
- SliderFileSystem sfs = initMock(ext);
-
- Application app = createValidApplication("comp1");
- Artifact artifact = new Artifact();
- artifact.setType(Artifact.TypeEnum.APPLICATION);
- artifact.setId("id");
- app.getComponent("comp1").setArtifact(artifact);
-
- // duplicate component name okay in the case of APPLICATION component
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
- }
-
- @Test
- public void testExternalComponent() throws IOException {
- Application ext = createValidApplication("comp1");
- SliderFileSystem sfs = initMock(ext);
-
- Application app = createValidApplication("comp2");
- Artifact artifact = new Artifact();
- artifact.setType(Artifact.TypeEnum.APPLICATION);
- artifact.setId("id");
- app.setArtifact(artifact);
-
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- assertEquals(1, app.getComponents().size());
- // artifact ID not inherited from global
- assertNotNull(app.getComponent("comp2"));
-
- // set APPLICATION artifact id on component
- app.getComponent("comp2").setArtifact(artifact);
-
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- assertEquals(1, app.getComponents().size());
- // original component replaced by external component
- assertNotNull(app.getComponent("comp1"));
- }
-
- public static void verifyDependencySorting(List<Component> components,
- Component... expectedSorting) {
- Collection<Component> actualSorting = ServiceApiUtil.sortByDependencies(
- components);
- assertEquals(expectedSorting.length, actualSorting.size());
- int i = 0;
- for (Component component : actualSorting) {
- assertEquals(expectedSorting[i++], component);
- }
- }
-
- @Test
- public void testDependencySorting() throws IOException {
- Component a = new Component().name("a");
- Component b = new Component().name("b");
- Component c = new Component().name("c");
- Component d = new Component().name("d").dependencies(Arrays.asList("c"));
- Component e = new Component().name("e").dependencies(Arrays.asList("b",
- "d"));
-
- verifyDependencySorting(Arrays.asList(a, b, c), a, b, c);
- verifyDependencySorting(Arrays.asList(c, a, b), c, a, b);
- verifyDependencySorting(Arrays.asList(a, b, c, d, e), a, b, c, d, e);
- verifyDependencySorting(Arrays.asList(e, d, c, b, a), c, b, a, d, e);
-
- c.setDependencies(Arrays.asList("e"));
- try {
- verifyDependencySorting(Arrays.asList(a, b, c, d, e));
- Assert.fail(EXCEPTION_PREFIX + "components with dependency cycle");
- } catch (IllegalArgumentException ex) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_DEPENDENCY_CYCLE, Arrays.asList(c, d,
- e)), ex.getMessage());
- }
-
- SliderFileSystem sfs = initMock(null);
- Application application = createValidApplication(null);
- application.setComponents(Arrays.asList(c, d, e));
- try {
- ServiceApiUtil.validateAndResolveApplication(application, sfs,
- CONF_DEFAULT_DNS);
- Assert.fail(EXCEPTION_PREFIX + "components with bad dependencies");
- } catch (IllegalArgumentException ex) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, "b", "e"), ex
- .getMessage());
- }
- }
-
- @Test
- public void testInvalidComponent() throws IOException {
- SliderFileSystem sfs = initMock(null);
- testComponent(sfs);
- }
-
- @Test
- public void testValidateCompName() {
- String[] invalidNames = {
- "EXAMPLE", // UPPER case not allowed
- "example_app" // underscore not allowed.
- };
- for (String name : invalidNames) {
- try {
- ServiceApiUtil.validateCompName(name);
- Assert.fail();
- } catch (IllegalArgumentException ex) {
- ex.printStackTrace();
- }
- }
- }
-
- private static void testComponent(SliderFileSystem sfs)
- throws IOException {
- int maxLen = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
- assertEquals(19, Long.toString(Long.MAX_VALUE).length());
- maxLen = maxLen - Long.toString(Long.MAX_VALUE).length();
-
- String compName = LEN_64_STR.substring(0, maxLen + 1);
- Application app = createValidApplication(null);
- app.addComponent(createValidComponent(compName));
-
- // invalid component name fails if dns is enabled
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with invalid component name");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(RestApiErrorMessages
- .ERROR_COMPONENT_NAME_INVALID, maxLen, compName), e.getMessage());
- }
-
- // does not fail if dns is disabled
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DEFAULT_DNS);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- compName = LEN_64_STR.substring(0, maxLen);
- app = createValidApplication(null);
- app.addComponent(createValidComponent(compName));
-
- // does not fail
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
deleted file mode 100644
index a36e0b4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ /dev/null
@@ -1,472 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.curator.test.TestingCluster;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.ContainerState;
-import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
-import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM;
-import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.AM_RESOURCE_MEM;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-
-/**
- * End to end tests to test deploying services with MiniYarnCluster and a in-JVM
- * ZK testing cluster.
- */
-public class TestYarnNativeServices extends ServiceTestUtils{
-
- private static final Log LOG =
- LogFactory.getLog(TestYarnNativeServices.class);
-
- private MiniYARNCluster yarnCluster = null;
- private MiniDFSCluster hdfsCluster = null;
- private FileSystem fs = null;
- protected Configuration conf = null;
- private static final int NUM_NMS = 1;
- private File basedir;
-
- @Rule
- public TemporaryFolder tmpFolder = new TemporaryFolder();
-
- @Before
- public void setup() throws Exception {
- setupInternal(NUM_NMS);
- }
-
- private void setupInternal(int numNodeManager)
- throws Exception {
- LOG.info("Starting up YARN cluster");
-// Logger rootLogger = LogManager.getRootLogger();
-// rootLogger.setLevel(Level.DEBUG);
- conf = new YarnConfiguration();
- conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
- // reduce the teardown waiting time
- conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000);
- conf.set("yarn.log.dir", "target");
- // mark if we need to launch the v1 timeline server
- // disable aux-service based timeline aggregators
- conf.set(YarnConfiguration.NM_AUX_SERVICES, "");
- conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8");
- // Enable ContainersMonitorImpl
- conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR,
- LinuxResourceCalculatorPlugin.class.getName());
- conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE,
- ProcfsBasedProcessTree.class.getName());
- conf.setBoolean(
- YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, true);
- conf.setBoolean(TIMELINE_SERVICE_ENABLED, false);
- conf.setInt(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100);
- conf.setLong(DEBUG_NM_DELETE_DELAY_SEC, 60000);
- conf.setLong(AM_RESOURCE_MEM, 526);
- conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 5);
- // Disable vmem check to disallow NM killing the container
- conf.setBoolean(NM_VMEM_CHECK_ENABLED, false);
- conf.setBoolean(NM_PMEM_CHECK_ENABLED, false);
- // setup zk cluster
- TestingCluster zkCluster;
- zkCluster = new TestingCluster(1);
- zkCluster.start();
- conf.set(YarnConfiguration.RM_ZK_ADDRESS, zkCluster.getConnectString());
- conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString());
- LOG.info("ZK cluster: " + zkCluster.getConnectString());
-
- fs = FileSystem.get(conf);
- basedir = new File("target", "apps");
- if (basedir.exists()) {
- FileUtils.deleteDirectory(basedir);
- } else {
- basedir.mkdirs();
- }
-
- conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
-
- if (yarnCluster == null) {
- yarnCluster =
- new MiniYARNCluster(TestYarnNativeServices.class.getSimpleName(), 1,
- numNodeManager, 1, 1);
- yarnCluster.init(conf);
- yarnCluster.start();
-
- waitForNMsToRegister();
-
- URL url = Thread.currentThread().getContextClassLoader()
- .getResource("yarn-site.xml");
- if (url == null) {
- throw new RuntimeException(
- "Could not find 'yarn-site.xml' dummy file in classpath");
- }
- Configuration yarnClusterConfig = yarnCluster.getConfig();
- yarnClusterConfig.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
- new File(url.getPath()).getParent());
- //write the document to a buffer (not directly to the file, as that
- //can cause the file being written to get read -which will then fail.
- ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
- yarnClusterConfig.writeXml(bytesOut);
- bytesOut.close();
- //write the bytes to the file in the classpath
- OutputStream os = new FileOutputStream(new File(url.getPath()));
- os.write(bytesOut.toByteArray());
- os.close();
- LOG.info("Write yarn-site.xml configs to: " + url);
- }
- if (hdfsCluster == null) {
- HdfsConfiguration hdfsConfig = new HdfsConfiguration();
- hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig)
- .numDataNodes(1).build();
- }
-
- try {
- Thread.sleep(2000);
- } catch (InterruptedException e) {
- LOG.info("setup thread sleep interrupted. message=" + e.getMessage());
- }
-
-
- }
-
- private void waitForNMsToRegister() throws Exception {
- int sec = 60;
- while (sec >= 0) {
- if (yarnCluster.getResourceManager().getRMContext().getRMNodes().size()
- >= NUM_NMS) {
- break;
- }
- Thread.sleep(1000);
- sec--;
- }
- }
-
- @After
- public void tearDown() throws IOException {
- if (yarnCluster != null) {
- try {
- yarnCluster.stop();
- } finally {
- yarnCluster = null;
- }
- }
- if (hdfsCluster != null) {
- try {
- hdfsCluster.shutdown();
- } finally {
- hdfsCluster = null;
- }
- }
- if (basedir != null) {
- FileUtils.deleteDirectory(basedir);
- }
- SliderFileSystem sfs = new SliderFileSystem(conf);
- Path appDir = sfs.getBaseApplicationPath();
- sfs.getFileSystem().delete(appDir, true);
- }
-
-
-
- // End-to-end test to use ServiceClient to deploy a service.
- // 1. Create a service with 2 components, each of which has 2 containers
- // 2. Flex up each component to 3 containers and check the component instance names
- // 3. Flex down each component to 1 container and check the component instance names
- // 4. Flex up each component to 2 containers and check the component instance names
- // 5. Stop the service
- // 6. Destroy the service
- @Test (timeout = 200000)
- public void testCreateFlexStopDestroyService() throws Exception {
- ServiceClient client = createClient();
- Application exampleApp = createExampleApplication();
- client.actionCreate(exampleApp);
- SliderFileSystem fileSystem = new SliderFileSystem(conf);
- Path appDir = fileSystem.buildClusterDirPath(exampleApp.getName());
- // check app.json is persisted.
- Assert.assertTrue(
- fs.exists(new Path(appDir, exampleApp.getName() + ".json")));
- waitForAllCompToBeReady(client, exampleApp);
-
- // Flex two components, each from 2 container to 3 containers.
- flexComponents(client, exampleApp, 3L);
- // wait for flex to be completed, increase from 2 to 3 containers.
- waitForAllCompToBeReady(client, exampleApp);
- // check all instances name for each component are in sequential order.
- checkCompInstancesInOrder(client, exampleApp);
-
- // flex down to 1
- flexComponents(client, exampleApp, 1L);
- waitForAllCompToBeReady(client, exampleApp);
- checkCompInstancesInOrder(client, exampleApp);
-
- // check component dir and registry are cleaned up.
-
- // flex up again to 2
- flexComponents(client, exampleApp, 2L);
- waitForAllCompToBeReady(client, exampleApp);
- checkCompInstancesInOrder(client, exampleApp);
-
- // stop the service
- LOG.info("Stop the service");
- client.actionStop(exampleApp.getName(), true);
- ApplicationReport report = client.getYarnClient()
- .getApplicationReport(ApplicationId.fromString(exampleApp.getId()));
- // AM unregisters with RM successfully
- Assert.assertEquals(FINISHED, report.getYarnApplicationState());
- Assert.assertEquals(FinalApplicationStatus.ENDED,
- report.getFinalApplicationStatus());
-
- LOG.info("Destroy the service");
- //destroy the service and check the app dir is deleted from fs.
- client.actionDestroy(exampleApp.getName());
- // check the application dir on hdfs (in this case, local fs) are deleted.
- Assert.assertFalse(fs.exists(appDir));
- }
-
- // Create compa with 2 containers
- // Create compb with 2 containers which depends on compa
- // Check containers for compa started before containers for compb
- @Test (timeout = 200000)
- public void testComponentStartOrder() throws Exception {
- ServiceClient client = createClient();
- Application exampleApp = new Application();
- exampleApp.setName("teststartorder");
- exampleApp.addComponent(createComponent("compa", 2, "sleep 1000"));
- Component compb = createComponent("compb", 2, "sleep 1000");
-
- // Let compb depedends on compa;
- compb.setDependencies(Collections.singletonList("compa"));
- exampleApp.addComponent(compb);
-
- client.actionCreate(exampleApp);
- waitForAllCompToBeReady(client, exampleApp);
-
- // check that containers for compa are launched before containers for compb
- checkContainerLaunchDependencies(client, exampleApp, "compa", "compb");
-
- client.actionStop(exampleApp.getName(), true);
- client.actionDestroy(exampleApp.getName());
- }
-
- // Check containers launched are in dependency order
- // Get all containers into a list and sort based on container launch time e.g.
- // compa-c1, compa-c2, compb-c1, compb-c2;
- // check that the container's launch time are align with the dependencies.
- private void checkContainerLaunchDependencies(ServiceClient client,
- Application exampleApp, String... compOrder)
- throws IOException, YarnException {
- Application retrievedApp = client.getStatus(exampleApp.getName());
- List<Container> containerList = new ArrayList<>();
- for (Component component : retrievedApp.getComponents()) {
- containerList.addAll(component.getContainers());
- }
- // sort based on launchTime
- containerList
- .sort((o1, o2) -> o1.getLaunchTime().compareTo(o2.getLaunchTime()));
- LOG.info("containerList: " + containerList);
- // check the containers are in the dependency order.
- int index = 0;
- for (String comp : compOrder) {
- long num = retrievedApp.getComponent(comp).getNumberOfContainers();
- for (int i = 0; i < num; i++) {
- String compInstanceName = containerList.get(index).getComponentName();
- String compName =
- compInstanceName.substring(0, compInstanceName.lastIndexOf('-'));
- Assert.assertEquals(comp, compName);
- index++;
- }
- }
- }
-
-
- private Map<String, Long> flexComponents(ServiceClient client,
- Application exampleApp, long count) throws YarnException, IOException {
- Map<String, Long> compCounts = new HashMap<>();
- compCounts.put("compa", count);
- compCounts.put("compb", count);
- // flex will update the persisted conf to reflect latest number of containers.
- exampleApp.getComponent("compa").setNumberOfContainers(count);
- exampleApp.getComponent("compb").setNumberOfContainers(count);
- client.flexByRestService(exampleApp.getName(), compCounts);
- return compCounts;
- }
-
- // Check each component's comp instances name are in sequential order.
- // E.g. If there are two instances compA-1 and compA-2
- // When flex up to 4 instances, it should be compA-1 , compA-2, compA-3, compA-4
- // When flex down to 3 instances, it should be compA-1 , compA-2, compA-3.
- private void checkCompInstancesInOrder(ServiceClient client,
- Application exampleApp) throws IOException, YarnException {
- Application application = client.getStatus(exampleApp.getName());
- for (Component comp : application.getComponents()) {
- checkEachCompInstancesInOrder(comp);
- }
- }
-
- private void checkRegistryAndCompDirDeleted() {
-
- }
-
- private void checkEachCompInstancesInOrder(Component component) {
- long expectedNumInstances = component.getNumberOfContainers();
- Assert.assertEquals(expectedNumInstances, component.getContainers().size());
- TreeSet<String> instances = new TreeSet<>();
- for (Container container : component.getContainers()) {
- instances.add(container.getComponentName());
- }
-
- int i = 0;
- for (String s : instances) {
- Assert.assertEquals(component.getName() + "-" + i, s);
- i++;
- }
- }
-
- private void waitForOneCompToBeReady(ServiceClient client,
- Application exampleApp, String readyComp)
- throws TimeoutException, InterruptedException {
- long numExpectedContainers =
- exampleApp.getComponent(readyComp).getNumberOfContainers();
- GenericTestUtils.waitFor(() -> {
- try {
- Application retrievedApp = client.getStatus(exampleApp.getName());
- Component retrievedComp = retrievedApp.getComponent(readyComp);
-
- if (retrievedComp.getContainers() != null
- && retrievedComp.getContainers().size() == numExpectedContainers) {
- LOG.info(readyComp + " found " + numExpectedContainers
- + " containers running");
- return true;
- } else {
- LOG.info(" Waiting for " + readyComp + "'s containers to be running");
- return false;
- }
- } catch (Exception e) {
- e.printStackTrace();
- return false;
- }
- }, 2000, 200000);
- }
-
- // wait until all the containers for all components become ready state
- private void waitForAllCompToBeReady(ServiceClient client,
- Application exampleApp) throws TimeoutException, InterruptedException {
- int expectedTotalContainers = countTotalContainers(exampleApp);
- GenericTestUtils.waitFor(() -> {
- try {
- Application retrievedApp = client.getStatus(exampleApp.getName());
- int totalReadyContainers = 0;
- LOG.info("Num Components " + retrievedApp.getComponents().size());
- for (Component component : retrievedApp.getComponents()) {
- LOG.info("looking for " + component.getName());
- LOG.info(component);
- if (component.getContainers() != null) {
- if (component.getContainers().size() == exampleApp
- .getComponent(component.getName()).getNumberOfContainers()) {
- for (Container container : component.getContainers()) {
- LOG.info(
- "Container state " + container.getState() + ", component "
- + component.getName());
- if (container.getState() == ContainerState.READY) {
- totalReadyContainers++;
- LOG.info("Found 1 ready container " + container.getId());
- }
- }
- } else {
- LOG.info(component.getName() + " Expected number of containers "
- + exampleApp.getComponent(component.getName())
- .getNumberOfContainers() + ", current = " + component
- .getContainers());
- }
- }
- }
- LOG.info("Exit loop, totalReadyContainers= " + totalReadyContainers
- + " expected = " + expectedTotalContainers);
- return totalReadyContainers == expectedTotalContainers;
- } catch (Exception e) {
- e.printStackTrace();
- return false;
- }
- }, 2000, 200000);
- }
-
- private ServiceClient createClient() throws Exception {
- ServiceClient client = new ServiceClient() {
- @Override protected Path addJarResource(String appName,
- Map<String, LocalResource> localResources)
- throws IOException, SliderException {
- // do nothing, the Unit test will use local jars
- return null;
- }
- };
- client.init(conf);
- client.start();
- return client;
- }
-
-
- private int countTotalContainers(Application application) {
- int totalContainers = 0;
- for (Component component : application.getComponents()) {
- totalContainers += component.getNumberOfContainers();
- }
- return totalContainers;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
deleted file mode 100644
index a22c000..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-
-/**
- * Test for building / resolving components of type APPLICATION.
- */
-public class TestBuildExternalComponents {
-
- protected Configuration conf = new YarnConfiguration();
- private File basedir;
-
- // Check component names match with expected
- private static void checkComponentNames(List<Component> components,
- Set<String> expectedComponents) {
- Assert.assertEquals(expectedComponents.size(), components.size());
- for (Component comp : components) {
- Assert.assertTrue(expectedComponents.contains(comp.getName()));
- }
- }
-
- // 1. Build the appDef and store on fs
- // 2. check component names
- private void buildAndCheckComponents(String appName, String appDef,
- SliderFileSystem sfs, Set<String> names) throws Throwable {
- String[] args =
- { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
- ClientArgs clientArgs = new ClientArgs(args);
- clientArgs.parse();
- ServiceCLI cli = new ServiceCLI() {
- @Override protected void createServiceClient() {
- client = new ServiceClient();
- client.init(conf);
- client.start();
- }
- };
- cli.exec(clientArgs);
-
- // verify generated conf
- List<Component> components =
- ServiceApiUtil.getApplicationComponents(sfs, appName);
- checkComponentNames(components, names);
- }
-
- @Before
- public void setup() throws IOException {
- basedir = new File("target", "apps");
- if (basedir.exists()) {
- FileUtils.deleteDirectory(basedir);
- } else {
- basedir.mkdirs();
- }
- conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
- }
-
- @After
- public void tearDown() throws IOException {
- if (basedir != null) {
- FileUtils.deleteDirectory(basedir);
- }
- }
-
- // Test applications defining external components(APPLICATION type)
- // can be resolved correctly
- @Test
- public void testExternalComponentBuild() throws Throwable {
- SliderFileSystem sfs = new SliderFileSystem(conf);
-
- Set<String> nameSet = new HashSet<>();
- nameSet.add("simple");
- nameSet.add("master");
- nameSet.add("worker");
-
- // app-1 has 3 components: simple, master, worker
- buildAndCheckComponents("app-1", ExampleAppJson.APP_JSON, sfs, nameSet);
- buildAndCheckComponents("external-0", ExampleAppJson.EXTERNAL_JSON_0, sfs,
- nameSet);
-
- nameSet.add("other");
-
- // external1 has 3 components: simple(APPLICATION - app1), master and other
- buildAndCheckComponents("external-1", ExampleAppJson.EXTERNAL_JSON_1, sfs,
- nameSet);
-
- nameSet.add("another");
-
- // external2 has 2 components: ext(APPLICATION - external1), another
- buildAndCheckComponents("external-2", ExampleAppJson.EXTERNAL_JSON_2, sfs,
- nameSet);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
deleted file mode 100644
index 20c06ab..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-import static org.mockito.Mockito.mock;
-
-public class TestServiceCLI {
-
- protected Configuration conf = new YarnConfiguration();
- private File basedir;
- private ServiceCLI cli;
- private SliderFileSystem fs;
-
- private void buildApp(String appName, String appDef) throws Throwable {
- String[] args =
- { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
- ClientArgs clientArgs = new ClientArgs(args);
- clientArgs.parse();
- cli.exec(clientArgs);
- }
-
- @Before
- public void setup() throws Throwable {
- basedir = new File("target", "apps");
- conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
- fs = new SliderFileSystem(conf);
- if (basedir.exists()) {
- FileUtils.deleteDirectory(basedir);
- } else {
- basedir.mkdirs();
- }
-
- // create a CLI and skip connection to AM
- cli = new ServiceCLI() {
- @Override protected void createServiceClient() {
- client = new ServiceClient() {
- @Override protected ClientAMProtocol getAMProxy(String appName,
- ApplicationReport report) throws IOException {
- return mock(ClientAMProtocol.class);
- }
- @Override protected ClientAMProtocol getAMProxy(String appName)
- throws IOException, YarnException {
- return mock(ClientAMProtocol.class);
- }
- };
- client.init(conf);
- client.start();
- }
- };
- }
-
- @After
- public void tearDown() throws IOException {
- if (basedir != null) {
- FileUtils.deleteDirectory(basedir);
- }
- }
-
- // Test flex components count are persisted.
- @Test
- public void testFlexComponents() throws Throwable {
- buildApp("service-1", ExampleAppJson.APP_JSON);
-
- checkCompCount("master", 1L);
-
- // increase by 2
- String[] flexUpArgs = {"flex", "service-1", "--component", "master" , "+2"};
- ClientArgs clientArgs = new ClientArgs(flexUpArgs);
- clientArgs.parse();
- cli.exec(clientArgs);
-
- checkCompCount("master", 3L);
-
- // decrease by 1
- String[] flexDownArgs = {"flex", "service-1", "--component", "master", "-1"};
- clientArgs = new ClientArgs(flexDownArgs);
- clientArgs.parse();
- cli.exec(clientArgs);
-
- checkCompCount("master", 2L);
-
- String[] flexAbsoluteArgs = {"flex", "service-1", "--component", "master", "10"};
- clientArgs = new ClientArgs(flexAbsoluteArgs);
- clientArgs.parse();
- cli.exec(clientArgs);
-
- checkCompCount("master", 10L);
- }
-
- private void checkCompCount(String compName, long count) throws IOException {
- List<Component> components =
- ServiceApiUtil.getApplicationComponents(fs, "service-1");
- for (Component component : components) {
- if (component.getName().equals(compName)) {
- Assert.assertEquals(count, component.getNumberOfContainers().longValue());
- return;
- }
- }
- Assert.fail();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
deleted file mode 100644
index 9e13200..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-
-import org.apache.hadoop.yarn.service.api.records.Application;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
-
-/**
- * Names of the example configs.
- */
-public final class ExampleAppJson {
-
- public static final String APP_JSON = "app.json";
- public static final String OVERRIDE_JSON = "app-override.json";
- public static final String DEFAULT_JSON = "default.json";
- public static final String EXTERNAL_JSON_0 = "external0.json";
- public static final String EXTERNAL_JSON_1 = "external1.json";
- public static final String EXTERNAL_JSON_2 = "external2.json";
-
- public static final String PACKAGE = "/org/apache/hadoop/yarn/service/conf/examples/";
-
-
- private static final String[] ALL_EXAMPLES = {APP_JSON, OVERRIDE_JSON,
- DEFAULT_JSON};
-
- public static final List<String> ALL_EXAMPLE_RESOURCES = new ArrayList<>();
- static {
- for (String example : ALL_EXAMPLES) {
- ALL_EXAMPLE_RESOURCES.add(PACKAGE + example);
- }
- }
-
- private ExampleAppJson() {
- }
-
- public static Application loadResource(String name) throws IOException {
- return JSON_SER_DESER.fromResource(PACKAGE + name);
- }
-
- public static String resourceName(String name) {
- return "target/test-classes" + PACKAGE + name;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
deleted file mode 100644
index 954d117..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.api.records.Configuration;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.hadoop.yarn.service.conf.ExampleAppJson.*;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
-import static org.easymock.EasyMock.*;
-
-/**
- * Test global configuration resolution.
- */
-public class TestAppJsonResolve extends Assert {
- protected static final Logger LOG =
- LoggerFactory.getLogger(TestAppJsonResolve.class);
-
- @Test
- public void testOverride() throws Throwable {
- Application orig = ExampleAppJson.loadResource(OVERRIDE_JSON);
-
- Configuration global = orig.getConfiguration();
- assertEquals("a", global.getProperty("g1"));
- assertEquals("b", global.getProperty("g2"));
- assertEquals(2, global.getFiles().size());
-
- Configuration simple = orig.getComponent("simple").getConfiguration();
- assertEquals(0, simple.getProperties().size());
- assertEquals(1, simple.getFiles().size());
-
- Configuration master = orig.getComponent("master").getConfiguration();
- assertEquals("m", master.getProperty("name"));
- assertEquals("overridden", master.getProperty("g1"));
- assertEquals(0, master.getFiles().size());
-
- Configuration worker = orig.getComponent("worker").getConfiguration();
- LOG.info("worker = {}", worker);
- assertEquals(3, worker.getProperties().size());
- assertEquals(0, worker.getFiles().size());
-
- assertEquals("worker", worker.getProperty("name"));
- assertEquals("overridden-by-worker", worker.getProperty("g1"));
- assertNull(worker.getProperty("g2"));
- assertEquals("1000", worker.getProperty("timeout"));
-
- // here is the resolution
- SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
- FileSystem mockFs = createNiceMock(FileSystem.class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- replay(sfs, mockFs);
- ServiceApiUtil.validateAndResolveApplication(orig, sfs, new
- YarnConfiguration());
-
- global = orig.getConfiguration();
- LOG.info("global = {}", global);
- assertEquals("a", global.getProperty("g1"));
- assertEquals("b", global.getProperty("g2"));
- assertEquals(2, global.getFiles().size());
-
- simple = orig.getComponent("simple").getConfiguration();
- assertEquals(2, simple.getProperties().size());
- assertEquals("a", simple.getProperty("g1"));
- assertEquals("b", simple.getProperty("g2"));
- assertEquals(2, simple.getFiles().size());
-
- Set<ConfigFile> files = new HashSet<>();
- Map<String, String> props = new HashMap<>();
- props.put("k1", "overridden");
- props.put("k2", "v2");
- files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
- .PROPERTIES).props(props));
- files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
- .XML).props(Collections.singletonMap("k3", "v3")));
- assertTrue(files.contains(simple.getFiles().get(0)));
- assertTrue(files.contains(simple.getFiles().get(1)));
-
- master = orig.getComponent("master").getConfiguration();
- LOG.info("master = {}", master);
- assertEquals(3, master.getProperties().size());
- assertEquals("m", master.getProperty("name"));
- assertEquals("overridden", master.getProperty("g1"));
- assertEquals("b", master.getProperty("g2"));
- assertEquals(2, master.getFiles().size());
-
- props.put("k1", "v1");
- files.clear();
- files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
- .PROPERTIES).props(props));
- files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
- .XML).props(Collections.singletonMap("k3", "v3")));
-
- assertTrue(files.contains(master.getFiles().get(0)));
- assertTrue(files.contains(master.getFiles().get(1)));
-
- worker = orig.getComponent("worker").getConfiguration();
- LOG.info("worker = {}", worker);
- assertEquals(4, worker.getProperties().size());
-
- assertEquals("worker", worker.getProperty("name"));
- assertEquals("overridden-by-worker", worker.getProperty("g1"));
- assertEquals("b", worker.getProperty("g2"));
- assertEquals("1000", worker.getProperty("timeout"));
- assertEquals(2, worker.getFiles().size());
-
- assertTrue(files.contains(worker.getFiles().get(0)));
- assertTrue(files.contains(worker.getFiles().get(1)));
- }
-
- @Test
- public void testOverrideExternalConfiguration() throws IOException {
- Application orig = ExampleAppJson.loadResource(EXTERNAL_JSON_1);
-
- Configuration global = orig.getConfiguration();
- assertEquals(0, global.getProperties().size());
-
- assertEquals(3, orig.getComponents().size());
-
- Configuration simple = orig.getComponent("simple").getConfiguration();
- assertEquals(0, simple.getProperties().size());
-
- Configuration master = orig.getComponent("master").getConfiguration();
- assertEquals(1, master.getProperties().size());
- assertEquals("is-overridden", master.getProperty("g3"));
-
- Configuration other = orig.getComponent("other").getConfiguration();
- assertEquals(0, other.getProperties().size());
-
- // load the external application
- SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
- FileSystem mockFs = createNiceMock(FileSystem.class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- replay(sfs, mockFs);
- Application ext = ExampleAppJson.loadResource(APP_JSON);
- ServiceApiUtil.validateAndResolveApplication(ext, sfs, new
- YarnConfiguration());
- reset(sfs, mockFs);
-
- // perform the resolution on original application
- JsonSerDeser<Application> jsonSerDeser = createNiceMock(JsonSerDeser
- .class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
- .anyTimes();
- replay(sfs, mockFs, jsonSerDeser);
- ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
- ServiceApiUtil.validateAndResolveApplication(orig, sfs, new
- YarnConfiguration());
-
- global = orig.getConfiguration();
- assertEquals(0, global.getProperties().size());
-
- assertEquals(4, orig.getComponents().size());
-
- simple = orig.getComponent("simple").getConfiguration();
- assertEquals(3, simple.getProperties().size());
- assertEquals("a", simple.getProperty("g1"));
- assertEquals("b", simple.getProperty("g2"));
- assertEquals("60",
- simple.getProperty("yarn.service.failure-count-reset.window"));
-
- master = orig.getComponent("master").getConfiguration();
- assertEquals(5, master.getProperties().size());
- assertEquals("512M", master.getProperty("jvm.heapsize"));
- assertEquals("overridden", master.getProperty("g1"));
- assertEquals("b", master.getProperty("g2"));
- assertEquals("is-overridden", master.getProperty("g3"));
- assertEquals("60",
- simple.getProperty("yarn.service.failure-count-reset.window"));
-
- Configuration worker = orig.getComponent("worker").getConfiguration();
- LOG.info("worker = {}", worker);
- assertEquals(4, worker.getProperties().size());
- assertEquals("512M", worker.getProperty("jvm.heapsize"));
- assertEquals("overridden-by-worker", worker.getProperty("g1"));
- assertEquals("b", worker.getProperty("g2"));
- assertEquals("60",
- worker.getProperty("yarn.service.failure-count-reset.window"));
-
- other = orig.getComponent("other").getConfiguration();
- assertEquals(0, other.getProperties().size());
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[80/86] [abbrv] hadoop git commit: YARN-7186. Add examples in
yarn-service. Contributed by Jian He
Posted by ji...@apache.org.
YARN-7186. Add examples in yarn-service. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c329bba2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c329bba2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c329bba2
Branch: refs/heads/yarn-native-services
Commit: c329bba24cd2d55db476347aee451eacf2574444
Parents: b69afb7
Author: Billie Rinaldi <bi...@apache.org>
Authored: Thu Sep 14 15:43:00 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 25 16:37:25 2017 -0700
----------------------------------------------------------------------
.../resources/assemblies/hadoop-yarn-dist.xml | 7 ++++
.../examples/sleeper/sleeper.json | 15 +++++++
.../yarn/service/client/ServiceClient.java | 26 ++++++++++--
.../client/params/AbstractActionArgs.java | 2 +-
.../AbstractClusterBuildingActionArgs.java | 2 +-
.../service/client/params/ActionBuildArgs.java | 7 +++-
.../service/client/params/ActionCreateArgs.java | 12 +++++-
.../service/client/params/ActionExamples.java | 26 ------------
.../yarn/service/client/params/Arguments.java | 4 +-
.../yarn/service/client/params/ClientArgs.java | 44 --------------------
.../yarn/service/client/params/CommonArgs.java | 15 +------
.../service/client/params/SliderActions.java | 2 -
.../src/site/markdown/YarnCommands.md | 34 ++++++++-------
.../site/markdown/yarn-service/QuickStart.md | 9 +++-
14 files changed, 91 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 8b3d292..83633ac 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -98,6 +98,13 @@
<outputDirectory>etc/hadoop</outputDirectory>
</fileSet>
<fileSet>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples</directory>
+ <outputDirectory>/share/hadoop/${hadoop.component}/yarn-service-examples</outputDirectory>
+ <includes>
+ <include>**/*</include>
+ </includes>
+ </fileSet>
+ <fileSet>
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
<includes>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json
new file mode 100644
index 0000000..89ce527
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json
@@ -0,0 +1,15 @@
+{
+ "name": "sleeper-service",
+ "components" :
+ [
+ {
+ "name": "sleeper",
+ "number_of_containers": 2,
+ "launch_command": "sleep 900000",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index bfddc44..6890aef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
+import org.apache.hadoop.yarn.service.client.params.ActionCreateArgs;
import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
import org.apache.hadoop.yarn.service.client.params.Arguments;
@@ -160,12 +161,15 @@ public class ServiceClient extends CompositeService
AbstractClusterBuildingActionArgs args) throws IOException {
File file = args.getFile();
Path filePath = new Path(file.getAbsolutePath());
- LOG.info("Loading app json from: " + filePath);
+ LOG.info("Loading service definition from: " + filePath);
Service service = jsonSerDeser
.load(FileSystem.getLocal(getConfig()), filePath);
if (args.lifetime > 0) {
service.setLifetime(args.lifetime);
}
+ if (!StringUtils.isEmpty(args.getServiceName())) {
+ service.setName(args.getServiceName());
+ }
return service;
}
@@ -182,9 +186,23 @@ public class ServiceClient extends CompositeService
return EXIT_SUCCESS;
}
- public int actionCreate(AbstractClusterBuildingActionArgs args)
+ public int actionCreate(ActionCreateArgs args)
throws IOException, YarnException {
- actionCreate(loadAppJsonFromLocalFS(args));
+ Service serviceDef;
+ if (args.file != null) {
+ serviceDef = loadAppJsonFromLocalFS(args);
+ } else if (!StringUtils.isEmpty(args.example)) {
+ // create an example service
+ String yarnHome = System
+ .getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
+ args.file = new File(MessageFormat
+ .format("{0}/share/hadoop/yarn/yarn-service-examples/{1}/{2}.json",
+ yarnHome, args.example, args.example));
+ serviceDef = loadAppJsonFromLocalFS(args);
+ } else {
+ throw new YarnException("No service definition provided!");
+ }
+ actionCreate(serviceDef);
return EXIT_SUCCESS;
}
@@ -213,7 +231,7 @@ public class ServiceClient extends CompositeService
Map<String, Long> componentCounts =
new HashMap<>(flexArgs.getComponentMap().size());
Service persistedService =
- ServiceApiUtil.loadService(fs, flexArgs.getClusterName());
+ ServiceApiUtil.loadService(fs, flexArgs.getServiceName());
if (!StringUtils.isEmpty(persistedService.getId())) {
cachedAppIds.put(persistedService.getName(),
ApplicationId.fromString(persistedService.getId()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
index 8b2d93e..9b7e2a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
@@ -50,7 +50,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
* get the name: relies on arg 1 being the cluster name in all operations
* @return the name argument, null if there is none
*/
- public String getClusterName() {
+ public String getServiceName() {
return (parameters.isEmpty()) ? null : parameters.get(0);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
index d215436..434c6d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
@@ -28,7 +28,7 @@ import java.io.File;
*/
public abstract class AbstractClusterBuildingActionArgs
extends AbstractActionArgs {
- @Parameter(names = { ARG_FILE, ARG_FILE_SHORT }, required = true,
+ @Parameter(names = { ARG_FILE, ARG_FILE_SHORT },
description = "The path to the service definition file in JSON format.")
public File file;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
index cf61525..0228125 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
@Parameters(commandNames = { SliderActions.ACTION_BUILD},
commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
@@ -30,7 +31,9 @@ public class ActionBuildArgs extends AbstractClusterBuildingActionArgs {
}
@Override
- public int getMinParams() {
- return 0;
+ public void validate() throws BadCommandArgumentsException {
+ if (file == null) {
+ throw new BadCommandArgumentsException("No service definition provided.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
index c43f4fc..039f528 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
@@ -18,21 +18,29 @@
package org.apache.hadoop.yarn.service.client.params;
+import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
@Parameters(commandNames = { SliderActions.ACTION_CREATE},
commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
public class ActionCreateArgs extends AbstractClusterBuildingActionArgs {
+ @Parameter(names = { ARG_EXAMPLE, ARG_EXAMPLE_SHORT },
+ description = "The name of the example service such as sleeper")
+ public String example;
+
@Override
public String getActionName() {
return SliderActions.ACTION_CREATE;
}
@Override
- public int getMinParams() {
- return 0;
+ public void validate() throws BadCommandArgumentsException {
+ if (file == null && example == null) {
+ throw new BadCommandArgumentsException("No service definition provided.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java
deleted file mode 100644
index e489e17..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_EXAMPLES},
- commandDescription = SliderActions.DESCRIBE_ACTION_EXAMPLES)
-public class ActionExamples {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
index 911440d..9672621 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
@@ -42,8 +42,8 @@ public interface Arguments {
String ARG_DELETE = "--delete";
String ARG_DEST = "--dest";
String ARG_DESTDIR = "--destdir";
- String ARG_FILESYSTEM = "--fs";
- String ARG_FILESYSTEM_LONG = "--filesystem";
+ String ARG_EXAMPLE = "--example";
+ String ARG_EXAMPLE_SHORT = "-e";
String ARG_FOLDER = "--folder";
String ARG_FORCE = "--force";
String ARG_FORMAT = "--format";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
index f479cd2..84d3bbd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.service.client.params;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
@@ -104,51 +102,10 @@ public class ClientArgs extends CommonArgs {
return actionDependencyArgs;
}
- public ActionDestroyArgs getActionDestroyArgs() {
- return actionDestroyArgs;
- }
-
- public ActionExistsArgs getActionExistsArgs() {
- return actionExistsArgs;
- }
-
public ActionFlexArgs getActionFlexArgs() {
return actionFlexArgs;
}
- public ActionFreezeArgs getActionFreezeArgs() {
- return actionFreezeArgs;
- }
-
- public ActionListArgs getActionListArgs() {
- return actionListArgs;
- }
-
-
- public ActionRegistryArgs getActionRegistryArgs() {
- return actionRegistryArgs;
- }
-
- public ActionResolveArgs getActionResolveArgs() {
- return actionResolveArgs;
- }
-
- public ActionResourceArgs getActionResourceArgs() {
- return actionResourceArgs;
- }
-
- public ActionStatusArgs getActionStatusArgs() {
- return actionStatusArgs;
- }
-
- public ActionThawArgs getActionThawArgs() {
- return actionThawArgs;
- }
-
- public ActionTokensArgs getActionTokenArgs() {
- return actionTokenArgs;
- }
-
/**
* Look at the chosen action and bind it as the core action for the operation.
* @throws SliderException bad argument or similar
@@ -227,7 +184,6 @@ public class ClientArgs extends CommonArgs {
case ACTION_UPDATE:
bindCoreAction(actionUpdateArgs);
break;
-
default:
throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
+ " " + action);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
index a5638a4..145b44a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
@@ -24,7 +24,6 @@ import com.beust.jcommander.ParameterDescription;
import com.beust.jcommander.ParameterException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
@@ -88,7 +87,7 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
* @return the name argument, null if there is none
*/
public String getClusterName() {
- return coreAction.getClusterName();
+ return coreAction.getServiceName();
}
protected CommonArgs(String[] args) {
@@ -142,10 +141,6 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
return result;
}
- public static String usage(CommonArgs serviceArgs) {
- return usage(serviceArgs, null);
- }
-
/**
* Parse routine -includes registering the action-specific argument classes
* and postprocess it
@@ -164,14 +159,6 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
postProcess();
}
- /**
- * Add a command
- * @param name action
- * @param arg value
- */
- protected void addAction(String name, Object arg) {
- commander.addCommand(name, arg);
- }
protected void addActions(Object... actions) {
for (Object action : actions) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
index fa05f2e..680bc0f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
@@ -30,7 +30,6 @@ public interface SliderActions {
String ACTION_UPGRADE = "upgrade";
String ACTION_DESTROY = "destroy";
String ACTION_EXISTS = "exists";
- String ACTION_EXAMPLES = "examples";
String ACTION_FLEX = "flex";
String ACTION_STOP = "stop";
String ACTION_HELP = "help";
@@ -59,7 +58,6 @@ public interface SliderActions {
"Destroy a stopped service, service must be stopped first before destroying.";
String DESCRIBE_ACTION_EXISTS =
"Probe for a service running";
- String DESCRIBE_ACTION_EXAMPLES = "Run an example service on YARN";
String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
String DESCRIBE_ACTION_FREEZE =
"Stop a running service";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 8968f13..55d007b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -76,28 +76,32 @@ Usage `yarn service [sub-command] [service-name] [options]`
* `build`: Build a service with its specifications, but do not start it.
```
- Usage: yarn service build --file [file]
- ```
+ Usage: yarn service build [service-name] --file [file]
+ Fields:
+ service-name Optional. If specified, it will override the name in the service definition.
- | COMMAND\_OPTIONS | Description |
- |:---- |:---- |
- | --file or -f | The local path to the service definition file |
+ Options:
+ --file,-f The local path to the service definition file
+ ```
* `create`: create a service, it's equivalent to first invoke build and then start.
```
- Usage: yarn service create --file [file]
+ Usage: yarn service create [service-name] --file [file]
+ Fields:
+ service-name Optional. If specified, it will override the name in the service definition.
+
+ Options:
+ --file,-f The local path to the service definition file.
+ --example,-e The name of the example service such as:
+ Sleeper A simple service that launches a few non-docker sleep containers on YARN.
```
- | COMMAND\_OPTIONS | Description |
- |:---- |:---- |
- | --file or -f | The local path to the service definition file |
* `dependency`: Yarn service framework dependency (libraries) management.
```
Usage: yarn service dependency [options]
+ Option:
+ --upload Pre-upload the dependency jars onto HDFS to expediate service launch process.
```
- | COMMAND\_OPTIONS | Description |
- |:---- |:---- |
- | --upload | Pre-upload the dependency jars onto HDFS to expediate service launch process. |
* `destroy`: Destroy a stopped service, service must be stopped first before destroying.
```
@@ -106,10 +110,10 @@ Usage `yarn service [sub-command] [service-name] [options]`
* `flex`: Flex a service's component by increasing or decreasing the number of containers.
```
Usage: yarn service flex [service-name] --component [component-name] [count]
+ Options:
+ --component [component-name] [count]
+ Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.
```
- | COMMAND\_OPTIONS | Description |
- |:---- |:---- |
- | --component [component-name] [count] | Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.|
* `status`: Get the status of a service.
```
Usage: yarn service status [service-name]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index 327566b..ab415de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -43,7 +43,14 @@ Below is a simple service definition that launches sleep containers on YARN by w
]
}
```
-
+User can simply run a pre-built example service on YARN using below command:
+```
+yarn service create [service-name] --example [example-name]
+```
+e.g. Below command launches a `sleeper` service named as `my-sleeper` on YARN.
+```
+yarn service create my-sleeper --example sleeper
+```
For launching docker based services using YARN Service framework, please refer to [API doc](YarnServiceAPI.md).
## Manage services on YARN via CLI
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
new file mode 100644
index 0000000..b851fb7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.servicemonitor.probe;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Base class of all probes.
+ */
+public abstract class Probe implements MonitorKeys {
+
+ protected final Configuration conf;
+ private String name;
+
+ /**
+ * Create a probe of a specific name
+ *
+ * @param name probe name
+ * @param conf configuration being stored.
+ */
+ public Probe(String name, Configuration conf) {
+ this.name = name;
+ this.conf = conf;
+ }
+
+
+ protected void setName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+
+ @Override
+ public String toString() {
+ return getName();
+ }
+
+ public static String getProperty(Map<String, String> props, String name,
+ String defaultValue) throws IOException {
+ String value = props.get(name);
+ if (StringUtils.isEmpty(value)) {
+ if (defaultValue == null) {
+ throw new IOException(name + " not specified");
+ }
+ return defaultValue;
+ }
+ return value;
+ }
+
+ public static int getPropertyInt(Map<String, String> props, String name,
+ Integer defaultValue) throws IOException {
+ String value = props.get(name);
+ if (StringUtils.isEmpty(value)) {
+ if (defaultValue == null) {
+ throw new IOException(name + " not specified");
+ }
+ return defaultValue;
+ }
+ return Integer.parseInt(value);
+ }
+
+ /**
+ * perform any prelaunch initialization
+ */
+ public void init() throws IOException {
+
+ }
+
+ /**
+ * Ping the endpoint. All exceptions must be caught and included in the
+ * (failure) status.
+ *
+ * @param instance instance to ping
+ * @return the status
+ */
+ public abstract ProbeStatus ping(ComponentInstance instance);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
new file mode 100644
index 0000000..7cd761c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.servicemonitor.probe;
+
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * Status message of a probe. This is designed to be sent over the wire, though the exception
+ * Had better be unserializable at the far end if that is to work.
+ */
+public final class ProbeStatus implements Serializable {
+ private static final long serialVersionUID = 165468L;
+
+ private long timestamp;
+ private String timestampText;
+ private boolean success;
+ private boolean realOutcome;
+ private String message;
+ private Throwable thrown;
+ private transient Probe originator;
+
+ public ProbeStatus() {
+ }
+
+ public ProbeStatus(long timestamp, String message, Throwable thrown) {
+ this.success = false;
+ this.message = message;
+ this.thrown = thrown;
+ setTimestamp(timestamp);
+ }
+
+ public ProbeStatus(long timestamp, String message) {
+ this.success = true;
+ setTimestamp(timestamp);
+ this.message = message;
+ this.thrown = null;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ timestampText = new Date(timestamp).toString();
+ }
+
+ public boolean isSuccess() {
+ return success;
+ }
+
+ /**
+ * Set both the success and the real outcome bits to the same value
+ * @param success the new value
+ */
+ public void setSuccess(boolean success) {
+ this.success = success;
+ realOutcome = success;
+ }
+
+ public String getTimestampText() {
+ return timestampText;
+ }
+
+ public boolean getRealOutcome() {
+ return realOutcome;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ public Throwable getThrown() {
+ return thrown;
+ }
+
+ public void setThrown(Throwable thrown) {
+ this.thrown = thrown;
+ }
+
+ /**
+ * Get the probe that generated this result. May be null
+ * @return a possibly null reference to a probe
+ */
+ public Probe getOriginator() {
+ return originator;
+ }
+
+ /**
+ * The probe has succeeded -capture the current timestamp, set
+ * success to true, and record any other data needed.
+ * @param probe probe
+ */
+ public void succeed(Probe probe) {
+ finish(probe, true, probe.getName(), null);
+ }
+
+ /**
+ * A probe has failed either because the test returned false, or an exception
+ * was thrown. The {@link #success} field is set to false, any exception
+ * thrown is recorded.
+ * @param probe probe that failed
+ * @param thrown an exception that was thrown.
+ */
+ public void fail(Probe probe, Throwable thrown) {
+ finish(probe, false, "Failure in " + probe, thrown);
+ }
+
+ public void finish(Probe probe, boolean succeeded, String text, Throwable thrown) {
+ setTimestamp(System.currentTimeMillis());
+ setSuccess(succeeded);
+ originator = probe;
+ message = text;
+ this.thrown = thrown;
+ }
+
+ @Override
+ public String toString() {
+ LogEntryBuilder builder = new LogEntryBuilder("Probe Status");
+ builder.elt("time", timestampText)
+ .elt("outcome", (success ? "success" : "failure"));
+
+ if (success != realOutcome) {
+ builder.elt("originaloutcome", (realOutcome ? "success" : "failure"));
+ }
+ builder.elt("message", message);
+ if (thrown != null) {
+ builder.elt("exception", thrown);
+ }
+
+ return builder.toString();
+ }
+
+ /**
+ * Flip the success bit on while the real outcome bit is kept false
+ */
+ public void markAsSuccessful() {
+ success = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
index 4f39921..78a7171 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
@@ -84,9 +84,6 @@ public final class ServiceTimelineMetricsConstants {
public static final String DESCRIPTION = "DESCRIPTION";
- public static final String UNIQUE_COMPONENT_SUPPORT =
- "UNIQUE_COMPONENT_SUPPORT";
-
public static final String RUN_PRIVILEGED_CONTAINER =
"RUN_PRIVILEGED_CONTAINER";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
index f115063..243baea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
@@ -21,24 +21,19 @@ package org.apache.hadoop.yarn.service.timelineservice;
import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.api.resource.Configuration;
-import org.apache.slider.api.resource.Container;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.server.appmaster.actions.ActionStopSlider;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,6 +45,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import static org.apache.hadoop.yarn.service.timelineservice.ServiceTimelineMetricsConstants.DIAGNOSTICS_INFO;
+
/**
* A single service that publishes all the Timeline Entities.
*/
@@ -87,7 +84,8 @@ public class ServiceTimelinePublisher extends CompositeService {
timelineClient = client;
}
- public void serviceAttemptRegistered(Application application) {
+ public void serviceAttemptRegistered(Application application,
+ org.apache.hadoop.conf.Configuration systemConf) {
long currentTimeMillis = application.getLaunchTime() == null
? System.currentTimeMillis() : application.getLaunchTime().getTime();
@@ -114,9 +112,12 @@ public class ServiceTimelinePublisher extends CompositeService {
// publish before configurations published
putEntity(entity);
- // publish application specific configurations
- publishConfigurations(application.getConfiguration(), application.getId(),
- ServiceTimelineEntityType.SERVICE_ATTEMPT.toString(), true);
+ // publish system config - YarnConfiguration
+ populateTimelineEntity(systemConf.iterator(), application.getId(),
+ ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
+ // publish user conf
+ publishUserConf(application.getConfiguration(), application.getId(),
+ ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
// publish component as separate entity.
publishComponents(application.getComponents());
@@ -129,12 +130,14 @@ public class ServiceTimelinePublisher extends CompositeService {
putEntity(entity);
}
- public void serviceAttemptUnregistered(ServiceContext context) {
+ public void serviceAttemptUnregistered(ServiceContext context,
+ String diagnostics) {
TimelineEntity entity = createServiceAttemptEntity(
context.attemptId.getApplicationId().toString());
Map<String, Object> entityInfos = new HashMap<String, Object>();
entityInfos.put(ServiceTimelineMetricsConstants.STATE,
- FinalApplicationStatus.FAILED);
+ FinalApplicationStatus.ENDED);
+ entityInfos.put(DIAGNOSTICS_INFO, diagnostics);
entity.addInfo(entityInfos);
// add an event
@@ -147,39 +150,6 @@ public class ServiceTimelinePublisher extends CompositeService {
putEntity(entity);
}
- public void serviceAttemptUnregistered(AppState appState,
- ActionStopSlider stopAction) {
- long currentTimeMillis = System.currentTimeMillis();
-
- TimelineEntity entity =
- createServiceAttemptEntity(appState.getClusterStatus().getId());
-
- // add info
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.EXIT_STATUS_CODE,
- stopAction.getExitCode());
- entityInfos.put(ServiceTimelineMetricsConstants.STATE,
- stopAction.getFinalApplicationStatus().toString());
- if (stopAction.getMessage() != null) {
- entityInfos.put(ServiceTimelineMetricsConstants.EXIT_REASON,
- stopAction.getMessage());
- }
- if (stopAction.getEx() != null) {
- entityInfos.put(ServiceTimelineMetricsConstants.DIAGNOSTICS_INFO,
- stopAction.getEx().toString());
- }
- entity.addInfo(entityInfos);
-
- // add an event
- TimelineEvent startEvent = new TimelineEvent();
- startEvent
- .setId(ServiceTimelineEvent.SERVICE_ATTEMPT_UNREGISTERED.toString());
- startEvent.setTimestamp(currentTimeMillis);
- entity.addEvent(startEvent);
-
- putEntity(entity);
- }
-
public void componentInstanceStarted(Container container,
ComponentInstance instance) {
@@ -210,29 +180,6 @@ public class ServiceTimelinePublisher extends CompositeService {
putEntity(entity);
}
- public void componentInstanceFinished(RoleInstance instance) {
- TimelineEntity entity = createComponentInstanceEntity(instance.id);
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.EXIT_STATUS_CODE,
- instance.exitCode);
- entityInfos.put(ServiceTimelineMetricsConstants.DIAGNOSTICS_INFO,
- instance.diagnostics);
- // TODO need to change the state based on enum value.
- entityInfos.put(ServiceTimelineMetricsConstants.STATE, "FINISHED");
- entity.addInfo(entityInfos);
-
- // add an event
- TimelineEvent startEvent = new TimelineEvent();
- startEvent
- .setId(ServiceTimelineEvent.COMPONENT_INSTANCE_UNREGISTERED.toString());
- startEvent.setTimestamp(System.currentTimeMillis());
- entity.addEvent(startEvent);
-
- putEntity(entity);
- }
-
public void componentInstanceFinished(ComponentInstance instance,
int exitCode, ContainerState state, String diagnostics) {
TimelineEntity entity = createComponentInstanceEntity(
@@ -242,7 +189,7 @@ public class ServiceTimelinePublisher extends CompositeService {
Map<String, Object> entityInfos = new HashMap<String, Object>();
entityInfos.put(ServiceTimelineMetricsConstants.EXIT_STATUS_CODE,
exitCode);
- entityInfos.put(ServiceTimelineMetricsConstants.DIAGNOSTICS_INFO, diagnostics);
+ entityInfos.put(DIAGNOSTICS_INFO, diagnostics);
entityInfos.put(ServiceTimelineMetricsConstants.STATE, state);
entity.addInfo(entityInfos);
@@ -302,8 +249,6 @@ public class ServiceTimelinePublisher extends CompositeService {
entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_COMMAND,
component.getLaunchCommand());
}
- entityInfos.put(ServiceTimelineMetricsConstants.UNIQUE_COMPONENT_SUPPORT,
- component.getUniqueComponentSupport().toString());
entityInfos.put(ServiceTimelineMetricsConstants.RUN_PRIVILEGED_CONTAINER,
component.getRunPrivilegedContainer().toString());
if (component.getPlacementPolicy() != null) {
@@ -315,31 +260,26 @@ public class ServiceTimelinePublisher extends CompositeService {
putEntity(entity);
// publish component specific configurations
- publishConfigurations(component.getConfiguration(), component.getName(),
- ServiceTimelineEntityType.COMPONENT.toString(), false);
+ publishUserConf(component.getConfiguration(), component.getName(),
+ ServiceTimelineEntityType.COMPONENT.toString());
}
}
- private void publishConfigurations(Configuration configuration,
- String entityId, String entityType, boolean isServiceAttemptEntity) {
- if (isServiceAttemptEntity) {
- // publish slider-client.xml properties at service level
- publishConfigurations(SliderUtils.loadSliderClientXML().iterator(),
- entityId, entityType);
- }
- publishConfigurations(configuration.getProperties().entrySet().iterator(),
+ private void publishUserConf(Configuration configuration,
+ String entityId, String entityType) {
+ populateTimelineEntity(configuration.getProperties().entrySet().iterator(),
entityId, entityType);
- publishConfigurations(configuration.getEnv().entrySet().iterator(),
+ populateTimelineEntity(configuration.getEnv().entrySet().iterator(),
entityId, entityType);
for (ConfigFile configFile : configuration.getFiles()) {
- publishConfigurations(configFile.getProps().entrySet().iterator(),
+ populateTimelineEntity(configFile.getProps().entrySet().iterator(),
entityId, entityType);
}
}
- private void publishConfigurations(Iterator<Entry<String, String>> iterator,
+ private void populateTimelineEntity(Iterator<Entry<String, String>> iterator,
String entityId, String entityType) {
int configSize = 0;
TimelineEntity entity = createTimelineEntity(entityId, entityType);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java
new file mode 100644
index 0000000..2607c08
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.JsonMappingException;
+
+import java.io.IOException;
+
+/**
+ * Persistence of {@link SerializedApplicationReport}
+ *
+ */
+public class ApplicationReportSerDeser
+ extends JsonSerDeser<SerializedApplicationReport> {
+ public ApplicationReportSerDeser() {
+ super(SerializedApplicationReport.class);
+ }
+
+
+ private static final ApplicationReportSerDeser
+ staticinstance = new ApplicationReportSerDeser();
+
+ /**
+ * Convert an instance to a JSON string -sync access to a shared ser/deser
+ * object instance
+ * @param instance object to convert
+ * @return a JSON string description
+ * @throws JsonParseException parse problems
+ * @throws JsonMappingException O/J mapping problems
+ */
+ public static String toString(SerializedApplicationReport instance)
+ throws IOException, JsonGenerationException, JsonMappingException {
+ synchronized (staticinstance) {
+ return staticinstance.toJson(instance);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
new file mode 100644
index 0000000..86896b2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.registry.client.impl.zk.RegistryInternalConstants;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.encodeForRegistry;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.convertUsername;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.getCurrentUsernameUnencoded;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
+
+/**
+ * Generic code to get the URLs for clients via the registry
+ */
+public class ClientRegistryBinder {
+ private static final Logger log =
+ LoggerFactory.getLogger(ClientRegistryBinder.class);
+
+ private final RegistryOperations operations;
+
+ public ClientRegistryBinder(RegistryOperations operations) {
+ this.operations = operations;
+ }
+
+ /**
+ * Buld the user path -switches to the system path if the user is "".
+ * It also cross-converts the username to ascii via punycode
+ * @param username username or ""
+ * @return the path to the user
+ */
+ public static String homePathForUser(String username) {
+ Preconditions.checkArgument(username != null, "null user");
+
+ // catch recursion
+ if (username.startsWith(RegistryConstants.PATH_USERS)) {
+ return username;
+ }
+
+ if (username.isEmpty()) {
+ return RegistryConstants.PATH_SYSTEM_SERVICES;
+ }
+
+ // convert username to registry name
+ String convertedName = convertUsername(username);
+
+ return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
+ encodeForRegistry(convertedName));
+ }
+
+ /**
+ * Get the current username, before any encoding has been applied.
+ * @return the current user from the kerberos identity, falling back
+ * to the user and/or env variables.
+ */
+ public static String currentUsernameUnencoded() {
+ String env_hadoop_username = System.getenv(
+ RegistryInternalConstants.HADOOP_USER_NAME);
+ return getCurrentUsernameUnencoded(env_hadoop_username);
+ }
+
+ /**
+ * Qualify a user.
+ * <ol>
+ * <li> <code>"~"</code> maps to user home path home</li>
+ * <li> <code>"~user"</code> maps to <code>/users/$user</code></li>
+ * <li> <code>"/"</code> maps to <code>/services/</code></li>
+ * </ol>
+ * @param user the username
+ * @return the base path
+ */
+ public static String qualifyUser(String user) {
+ // qualify the user
+ String t = user.trim();
+ if (t.startsWith("/")) {
+ // already resolved
+ return t;
+ } else if (t.equals("~")) {
+ // self
+ return currentUsernameUnencoded();
+ } else if (t.startsWith("~")) {
+ // another user
+ // convert username to registry name
+ String convertedName = convertUsername(t.substring(1));
+
+ return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
+ encodeForRegistry(convertedName));
+ } else {
+ return "/" + t;
+ }
+ }
+
+ /**
+ * Look up an external REST API
+ * @param user user which will be qualified as per {@link #qualifyUser(String)}
+ * @param serviceClass service class
+ * @param instance instance name
+ * @param api API
+ * @return the API, or an exception is raised.
+ * @throws IOException
+ */
+ public String lookupExternalRestAPI(String user,
+ String serviceClass,
+ String instance,
+ String api)
+ throws IOException {
+ String qualified = qualifyUser(user);
+ String path = servicePath(qualified, serviceClass, instance);
+ String restAPI = resolveExternalRestAPI(api, path);
+ if (restAPI == null) {
+ throw new PathNotFoundException(path + " API " + api);
+ }
+ return restAPI;
+ }
+
+ /**
+ * Resolve a service record then return an external REST API exported it.
+ *
+ * @param api API to resolve
+ * @param path path of the service record
+ * @return null if the record exists but the API is absent or it has no
+ * REST endpoints.
+ * @throws IOException resolution problems, as covered in
+ * {@link RegistryOperations#resolve(String)}
+ */
+ protected String resolveExternalRestAPI(String api, String path) throws
+ IOException {
+ ServiceRecord record = operations.resolve(path);
+ return lookupRestAPI(record, api, true);
+ }
+
+ /**
+ * Look up an external REST API endpoint
+ * @param record service record
+ * @param api URI of api
+ * @param external flag to indicate this is an external record
+ * @return the first endpoint of the implementation, or null if there
+ * is no entry for the API, implementation or it's the wrong type.
+ */
+ public static String lookupRestAPI(ServiceRecord record,
+ String api, boolean external) throws InvalidRecordException {
+ try {
+ String url = null;
+ Endpoint endpoint = getEndpoint(record, api, external);
+ List<String> addresses =
+ RegistryTypeUtils.retrieveAddressesUriType(endpoint);
+ if (addresses != null && !addresses.isEmpty()) {
+ url = addresses.get(0);
+ }
+ return url;
+ } catch (InvalidRecordException e) {
+ log.debug("looking for API {}", api, e);
+ return null;
+ }
+ }
+
+ /**
+ * Get an endpont by API
+ * @param record service record
+ * @param api API
+ * @param external flag to indicate this is an external record
+ * @return the endpoint or null
+ */
+ public static Endpoint getEndpoint(ServiceRecord record,
+ String api,
+ boolean external) {
+ return external ? record.getExternalEndpoint(api)
+ : record.getInternalEndpoint(api);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java
new file mode 100644
index 0000000..9f0e5d4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+/**
+ * Some general comparators
+ */
+public class Comparators {
+
+ public static class LongComparator implements Comparator<Long>, Serializable {
+ @Override
+ public int compare(Long o1, Long o2) {
+ return o1.compareTo(o2);
+ }
+ }
+
+ public static class InvertedLongComparator
+ implements Comparator<Long>, Serializable {
+ @Override
+ public int compare(Long o1, Long o2) {
+ return o2.compareTo(o1);
+ }
+ }
+
+ /**
+ * Little template class to reverse any comparitor
+ * @param <CompareType> the type that is being compared
+ */
+ public static class ComparatorReverser<CompareType> implements Comparator<CompareType>,
+ Serializable {
+
+ final Comparator<CompareType> instance;
+
+ public ComparatorReverser(Comparator<CompareType> instance) {
+ this.instance = instance;
+ }
+
+ @Override
+ public int compare(CompareType first, CompareType second) {
+ return instance.compare(second, first);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
new file mode 100644
index 0000000..fe8cce8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.net.URL;
+import java.util.Map;
+
+/**
+ * Methods to aid in config, both in the Configuration class and
+ * with other parts of setting up Slider-initated processes.
+ *
+ * Some of the methods take an argument of a map iterable for their sources; this allows
+ * the same method
+ */
+public class ConfigHelper {
+ private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);
+
+ /**
+ * Set an entire map full of values
+ *
+ * @param config config to patch
+ * @param map map of data
+ * @param origin origin data
+ */
+ public static void addConfigMap(Configuration config,
+ Map<String, String> map,
+ String origin) throws BadConfigException {
+ addConfigMap(config, map.entrySet(), origin);
+ }
+
+ /**
+ * Set an entire map full of values
+ *
+ * @param config config to patch
+ * @param map map of data
+ * @param origin origin data
+ */
+ public static void addConfigMap(Configuration config,
+ Iterable<Map.Entry<String, String>> map,
+ String origin) throws BadConfigException {
+ for (Map.Entry<String, String> mapEntry : map) {
+ String key = mapEntry.getKey();
+ String value = mapEntry.getValue();
+ if (value == null) {
+ throw new BadConfigException("Null value for property " + key);
+ }
+ config.set(key, value, origin);
+ }
+ }
+
+ /**
+ * Convert to an XML string
+ * @param conf configuration
+ * @return conf
+ * @throws IOException
+ */
+ public static String toXml(Configuration conf) throws IOException {
+ StringWriter writer = new StringWriter();
+ conf.writeXml(writer);
+ return writer.toString();
+ }
+
+
+ /**
+ * Register a resource as a default resource.
+ * Do not attempt to use this unless you understand that the
+ * order in which default resources are loaded affects the outcome,
+ * and that subclasses of Configuration often register new default
+ * resources
+ * @param resource the resource name
+ * @return the URL or null
+ */
+ public static URL registerDefaultResource(String resource) {
+ URL resURL = getResourceUrl(resource);
+ if (resURL != null) {
+ Configuration.addDefaultResource(resource);
+ }
+ return resURL;
+ }
+
+ /**
+ * Load a configuration from a resource on this classpath.
+ * If the resource is not found, an empty configuration is returned
+ * @param resource the resource name
+ * @return the loaded configuration.
+ */
+ public static Configuration loadFromResource(String resource) {
+ Configuration conf = new Configuration(false);
+ URL resURL = getResourceUrl(resource);
+ if (resURL != null) {
+ log.debug("loaded resources from {}", resURL);
+ conf.addResource(resource);
+ } else{
+ log.debug("failed to find {} on the classpath", resource);
+ }
+ return conf;
+
+ }
+
+ /**
+ * Get the URL to a resource, null if not on the CP
+ * @param resource resource to look for
+ * @return the URL or null
+ */
+ public static URL getResourceUrl(String resource) {
+ return ConfigHelper.class.getClassLoader()
+ .getResource(resource);
+ }
+
+ /**
+ * This goes through the keyset of one configuration and retrieves each value
+ * from a value source -a different or the same configuration. This triggers
+ * the property resolution process of the value, resolving any variables against
+ * in-config or inherited configurations
+ * @param keysource source of keys
+ * @param valuesource the source of values
+ * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code>
+ */
+ public static Configuration resolveConfiguration(
+ Iterable<Map.Entry<String, String>> keysource,
+ Configuration valuesource) {
+ Configuration result = new Configuration(false);
+ for (Map.Entry<String, String> entry : keysource) {
+ String key = entry.getKey();
+ String value = valuesource.get(key);
+ Preconditions.checkState(value != null,
+ "no reference for \"%s\" in values", key);
+ result.set(key, value);
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java
new file mode 100644
index 0000000..a969be9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class ConfigUtils {
+ public static final String TEMPLATE_FILE = "template.file";
+
+ public static String replaceProps(Map<String, String> config, String content) {
+ Map<String, String> tokens = new HashMap<>();
+ for (Entry<String, String> entry : config.entrySet()) {
+ tokens.put("${" + entry.getKey() + "}", entry.getValue());
+ tokens.put("{{" + entry.getKey() + "}}", entry.getValue());
+ }
+ String value = content;
+ for (Map.Entry<String,String> token : tokens.entrySet()) {
+ value = value.replaceAll(Pattern.quote(token.getKey()),
+ Matcher.quoteReplacement(token.getValue()));
+ }
+ return value;
+ }
+
+ public static Map<String, String> replacePropsInConfig(
+ Map<String, String> config, Map<String, String> env) {
+ Map<String, String> tokens = new HashMap<>();
+ for (Entry<String, String> entry : env.entrySet()) {
+ tokens.put("${" + entry.getKey() + "}", entry.getValue());
+ }
+ Map<String, String> newConfig = new HashMap<>();
+ for (Entry<String, String> entry : config.entrySet()) {
+ String value = entry.getValue();
+ for (Map.Entry<String,String> token : tokens.entrySet()) {
+ value = value.replaceAll(Pattern.quote(token.getKey()),
+ Matcher.quoteReplacement(token.getValue()));
+ }
+ newConfig.put(entry.getKey(), entry.getValue());
+ }
+ return newConfig;
+ }
+
+ public static void prepConfigForTemplateOutputter(ConfigFormat configFormat,
+ Map<String, String> config, SliderFileSystem fileSystem,
+ String clusterName, String fileName) throws IOException {
+ if (!configFormat.equals(ConfigFormat.TEMPLATE)) {
+ return;
+ }
+ Path templateFile = null;
+ if (config.containsKey(TEMPLATE_FILE)) {
+ templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE));
+ if (!fileSystem.isFile(templateFile)) {
+ templateFile = fileSystem.buildResourcePath(clusterName,
+ config.get(TEMPLATE_FILE));
+ }
+ if (!fileSystem.isFile(templateFile)) {
+ throw new IOException("config specified template file " + config
+ .get(TEMPLATE_FILE) + " but " + templateFile + " doesn't exist");
+ }
+ }
+ if (templateFile == null && fileName != null) {
+ templateFile = fileSystem.buildResourcePath(fileName);
+ if (!fileSystem.isFile(templateFile)) {
+ templateFile = fileSystem.buildResourcePath(clusterName,
+ fileName);
+ }
+ }
+ if (fileSystem.isFile(templateFile)) {
+ config.put("content", fileSystem.cat(templateFile));
+ } else {
+ config.put("content", "");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
new file mode 100644
index 0000000..fa3b402
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
@@ -0,0 +1,521 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.util.Records;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class CoreFileSystem {
+ private static final Logger
+ log = LoggerFactory.getLogger(CoreFileSystem.class);
+
+ private static final String UTF_8 = "UTF-8";
+
+ protected final FileSystem fileSystem;
+ protected final Configuration configuration;
+
+ public CoreFileSystem(FileSystem fileSystem, Configuration configuration) {
+ Preconditions.checkNotNull(fileSystem,
+ "Cannot create a CoreFileSystem with a null FileSystem");
+ Preconditions.checkNotNull(configuration,
+ "Cannot create a CoreFileSystem with a null Configuration");
+ this.fileSystem = fileSystem;
+ this.configuration = configuration;
+ }
+
+ public CoreFileSystem(Configuration configuration) throws IOException {
+ Preconditions.checkNotNull(configuration,
+ "Cannot create a CoreFileSystem with a null Configuration");
+ this.fileSystem = FileSystem.get(configuration);
+ this.configuration = configuration;
+ }
+
+ /**
+ * Get the temp path for this cluster
+ * @param clustername name of the cluster
+ * @return path for temp files (is not purged)
+ */
+ public Path getTempPathForCluster(String clustername) {
+ Path clusterDir = buildClusterDirPath(clustername);
+ return new Path(clusterDir, YarnServiceConstants.TMP_DIR_PREFIX);
+ }
+
+ /**
+ * Returns the underlying FileSystem for this object.
+ *
+ * @return filesystem
+ */
+ public FileSystem getFileSystem() {
+ return fileSystem;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("CoreFileSystem{");
+ sb.append("fileSystem=").append(fileSystem.getUri());
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Build up the path string for a cluster instance -no attempt to
+ * create the directory is made
+ *
+ * @param clustername name of the cluster
+ * @return the path for persistent data
+ */
+ public Path buildClusterDirPath(String clustername) {
+ Preconditions.checkNotNull(clustername);
+ Path path = getBaseApplicationPath();
+ return new Path(path, YarnServiceConstants.SERVICES_DIRECTORY + "/" + clustername);
+ }
+
+
+ /**
+ * Build up the path string for keytab install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for keytab
+ */
+ public Path buildKeytabInstallationDirPath(String keytabFolder) {
+ Preconditions.checkNotNull(keytabFolder);
+ Path path = getBaseApplicationPath();
+ return new Path(path, YarnServiceConstants.KEYTAB_DIR + "/" + keytabFolder);
+ }
+
+ /**
+ * Build up the path string for keytab install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for keytab installation location
+ */
+ public Path buildKeytabPath(String keytabDir, String keytabName, String clusterName) {
+ Path homePath = getHomeDirectory();
+ Path baseKeytabDir;
+ if (keytabDir != null) {
+ baseKeytabDir = new Path(homePath, keytabDir);
+ } else {
+ baseKeytabDir = new Path(buildClusterDirPath(clusterName),
+ YarnServiceConstants.KEYTAB_DIR);
+ }
+ return keytabName == null ? baseKeytabDir :
+ new Path(baseKeytabDir, keytabName);
+ }
+
+ /**
+ * Build up the path string for resource install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for resource
+ */
+ public Path buildResourcePath(String resourceFolder) {
+ Preconditions.checkNotNull(resourceFolder);
+ Path path = getBaseApplicationPath();
+ return new Path(path, YarnServiceConstants.RESOURCE_DIR + "/" + resourceFolder);
+ }
+
+ /**
+ * Build up the path string for resource install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for resource
+ */
+ public Path buildResourcePath(String dirName, String fileName) {
+ Preconditions.checkNotNull(dirName);
+ Preconditions.checkNotNull(fileName);
+ Path path = getBaseApplicationPath();
+ return new Path(path, YarnServiceConstants.RESOURCE_DIR + "/" + dirName + "/" + fileName);
+ }
+
+ /**
+ * Create a directory with the given permissions.
+ *
+ * @param dir directory
+ * @param clusterPerms cluster permissions
+ * @throws IOException IO problem
+ * @throws BadClusterStateException any cluster state problem
+ */
+ @SuppressWarnings("deprecation")
+ public void createWithPermissions(Path dir, FsPermission clusterPerms) throws
+ IOException,
+ BadClusterStateException {
+ if (fileSystem.isFile(dir)) {
+ // HADOOP-9361 shows some filesystems don't correctly fail here
+ throw new BadClusterStateException(
+ "Cannot create a directory over a file %s", dir);
+ }
+ log.debug("mkdir {} with perms {}", dir, clusterPerms);
+ //no mask whatoever
+ fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");
+ fileSystem.mkdirs(dir, clusterPerms);
+ //and force set it anyway just to make sure
+ fileSystem.setPermission(dir, clusterPerms);
+ }
+
+ /**
+ * Verify that the cluster directory is not present
+ *
+ * @param clustername name of the cluster
+ * @param clusterDirectory actual directory to look for
+ * @throws IOException trouble with FS
+ * @throws SliderException If the directory exists
+ */
+ public void verifyClusterDirectoryNonexistent(String clustername,
+ Path clusterDirectory)
+ throws IOException, SliderException {
+ if (fileSystem.exists(clusterDirectory)) {
+ throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
+ ErrorStrings.PRINTF_E_INSTANCE_ALREADY_EXISTS, clustername,
+ clusterDirectory);
+ }
+ }
+ /**
+ * Verify that the given directory is not present
+ *
+ * @param clusterDirectory actual directory to look for
+ * @throws IOException trouble with FS
+ * @throws SliderException If the directory exists
+ */
+ public void verifyDirectoryNonexistent(Path clusterDirectory) throws
+ IOException,
+ SliderException {
+ if (fileSystem.exists(clusterDirectory)) {
+
+ log.error("Dir {} exists: {}",
+ clusterDirectory,
+ listFSDir(clusterDirectory));
+ throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
+ ErrorStrings.PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS,
+ clusterDirectory);
+ }
+ }
+
+ /**
+ * Verify that a user has write access to a directory.
+ * It does this by creating then deleting a temp file
+ *
+ * @param dirPath actual directory to look for
+ * @throws FileNotFoundException file not found
+ * @throws IOException trouble with FS
+ * @throws BadClusterStateException if the directory is not writeable
+ */
+ public void verifyDirectoryWriteAccess(Path dirPath) throws IOException,
+ SliderException {
+ verifyPathExists(dirPath);
+ Path tempFile = new Path(dirPath, "tmp-file-for-checks");
+ try {
+ FSDataOutputStream out ;
+ out = fileSystem.create(tempFile, true);
+ IOUtils.closeStream(out);
+ fileSystem.delete(tempFile, false);
+ } catch (IOException e) {
+ log.warn("Failed to create file {}: {}", tempFile, e);
+ throw new BadClusterStateException(e,
+ "Unable to write to directory %s : %s", dirPath, e.toString());
+ }
+ }
+
+ /**
+ * Verify that a path exists
+ * @param path path to check
+ * @throws FileNotFoundException file not found
+ * @throws IOException trouble with FS
+ */
+ public void verifyPathExists(Path path) throws IOException {
+ if (!fileSystem.exists(path)) {
+ throw new FileNotFoundException(path.toString());
+ }
+ }
+
+ /**
+ * Verify that a path exists
+ * @param path path to check
+ * @throws FileNotFoundException file not found or is not a file
+ * @throws IOException trouble with FS
+ */
+ public void verifyFileExists(Path path) throws IOException {
+ FileStatus status = fileSystem.getFileStatus(path);
+
+ if (!status.isFile()) {
+ throw new FileNotFoundException("Not a file: " + path.toString());
+ }
+ }
+
+ /**
+ * Given a path, check if it exists and is a file
+ *
+ * @param path
+ * absolute path to the file to check
+ * @return true if and only if path exists and is a file, false for all other
+ * reasons including if file check throws IOException
+ */
+ public boolean isFile(Path path) {
+ boolean isFile = false;
+ try {
+ FileStatus status = fileSystem.getFileStatus(path);
+ if (status.isFile()) {
+ isFile = true;
+ }
+ } catch (IOException e) {
+ // ignore, isFile is already set to false
+ }
+ return isFile;
+ }
+
+ /**
+ * Get the base path
+ *
+ * @return the base path optionally configured by
+ * {@link YarnServiceConf#YARN_SERVICE_BASE_PATH}
+ */
+ public Path getBaseApplicationPath() {
+ String configuredBasePath = configuration
+ .get(YarnServiceConf.YARN_SERVICE_BASE_PATH,
+ getHomeDirectory() + "/" + YarnServiceConstants.SERVICE_BASE_DIRECTORY);
+ return new Path(configuredBasePath);
+ }
+
+ /**
+ * Get slider dependency parent dir in HDFS
+ *
+ * @return the parent dir path of slider.tar.gz in HDFS
+ */
+ public Path getDependencyPath() {
+ String parentDir = YarnServiceConstants.DEPENDENCY_DIR;
+ return new Path(String.format(parentDir, VersionInfo.getVersion()));
+ }
+
+ /**
+ * Get slider.tar.gz absolute filepath in HDFS
+ *
+ * @return the absolute path to slider.tar.gz in HDFS
+ */
+ public Path getDependencyTarGzip() {
+ Path dependencyLibAmPath = getDependencyPath();
+ Path dependencyLibTarGzip = new Path(
+ dependencyLibAmPath.toUri().toString(),
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
+ + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+ return dependencyLibTarGzip;
+ }
+
+ public Path getHomeDirectory() {
+ return fileSystem.getHomeDirectory();
+ }
+
+ /**
+ * Create an AM resource from the
+ *
+ * @param destPath dest path in filesystem
+ * @param resourceType resource type
+ * @return the local resource for AM
+ */
+ public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException {
+ FileStatus destStatus = fileSystem.getFileStatus(destPath);
+ LocalResource amResource = Records.newRecord(LocalResource.class);
+ amResource.setType(resourceType);
+ // Set visibility of the resource
+ // Setting to most private option
+ amResource.setVisibility(LocalResourceVisibility.APPLICATION);
+ // Set the resource to be copied over
+ amResource.setResource(
+ URL.fromPath(fileSystem.resolvePath(destStatus.getPath())));
+ // Set timestamp and length of file so that the framework
+ // can do basic sanity checks for the local resource
+ // after it has been copied over to ensure it is the same
+ // resource the client intended to use with the application
+ amResource.setTimestamp(destStatus.getModificationTime());
+ amResource.setSize(destStatus.getLen());
+ return amResource;
+ }
+
+ /**
+ * Register all files under a fs path as a directory to push out
+ *
+ * @param srcDir src dir
+ * @param destRelativeDir dest dir (no trailing /)
+ * @return the map of entries
+ */
+ public Map<String, LocalResource> submitDirectory(Path srcDir, String destRelativeDir) throws IOException {
+ //now register each of the files in the directory to be
+ //copied to the destination
+ FileStatus[] fileset = fileSystem.listStatus(srcDir);
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>(fileset.length);
+ for (FileStatus entry : fileset) {
+
+ LocalResource resource = createAmResource(entry.getPath(),
+ LocalResourceType.FILE);
+ String relativePath = destRelativeDir + "/" + entry.getPath().getName();
+ localResources.put(relativePath, resource);
+ }
+ return localResources;
+ }
+
+ /**
+ * Submit a JAR containing a specific class, returning
+ * the resource to be mapped in
+ *
+ * @param clazz class to look for
+ * @param subdir subdirectory (expected to end in a "/")
+ * @param jarName <i>At the destination</i>
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName)
+ throws IOException, SliderException {
+ File localFile = SliderUtils.findContainingJarOrFail(clazz);
+ return submitFile(localFile, tempPath, subdir, jarName);
+ }
+
+ /**
+ * Submit a local file to the filesystem references by the instance's cluster
+ * filesystem
+ *
+ * @param localFile filename
+ * @param subdir subdirectory (expected to end in a "/")
+ * @param destFileName destination filename
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public LocalResource submitFile(File localFile, Path tempPath, String subdir, String destFileName)
+ throws IOException {
+ Path src = new Path(localFile.toString());
+ Path subdirPath = new Path(tempPath, subdir);
+ fileSystem.mkdirs(subdirPath);
+ Path destPath = new Path(subdirPath, destFileName);
+ log.debug("Copying {} (size={} bytes) to {}", localFile, localFile.length(), destPath);
+
+ fileSystem.copyFromLocalFile(false, true, src, destPath);
+
+ // Set the type of resource - file or archive
+ // archives are untarred at destination
+ // we don't need the jar file to be untarred for now
+ return createAmResource(destPath, LocalResourceType.FILE);
+ }
+
+ /**
+ * Submit the AM tar.gz resource referenced by the instance's cluster
+ * filesystem. Also, update the providerResources object with the new
+ * resource.
+ *
+ * @param providerResources
+ * the provider resource map to be updated
+ * @throws IOException
+ * trouble copying to HDFS
+ */
+ public void submitTarGzipAndUpdate(
+ Map<String, LocalResource> providerResources) throws IOException,
+ BadClusterStateException {
+ Path dependencyLibTarGzip = getDependencyTarGzip();
+ LocalResource lc = createAmResource(dependencyLibTarGzip,
+ LocalResourceType.ARCHIVE);
+ providerResources.put(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK, lc);
+ }
+
+ public void copyLocalFileToHdfs(File localPath,
+ Path destPath, FsPermission fp)
+ throws IOException {
+ if (localPath == null || destPath == null) {
+ throw new IOException("Either localPath or destPath is null");
+ }
+ fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
+ "000");
+ fileSystem.mkdirs(destPath.getParent(), fp);
+ log.info("Copying file {} to {}", localPath.toURI(),
+ fileSystem.getScheme() + ":/" + destPath.toUri());
+
+ fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()),
+ destPath);
+ // set file permissions of the destPath
+ fileSystem.setPermission(destPath, fp);
+ }
+
+ public void copyHdfsFileToLocal(Path hdfsPath, File destFile)
+ throws IOException {
+ if (hdfsPath == null || destFile == null) {
+ throw new IOException("Either hdfsPath or destPath is null");
+ }
+ log.info("Copying file {} to {}", hdfsPath.toUri(), destFile.toURI());
+
+ Path destPath = new Path(destFile.getPath());
+ fileSystem.copyToLocalFile(hdfsPath, destPath);
+ }
+
+ /**
+ * list entries in a filesystem directory
+ *
+ * @param path directory
+ * @return a listing, one to a line
+ * @throws IOException
+ */
+ public String listFSDir(Path path) throws IOException {
+ FileStatus[] stats = fileSystem.listStatus(path);
+ StringBuilder builder = new StringBuilder();
+ for (FileStatus stat : stats) {
+ builder.append(stat.getPath().toString())
+ .append("\t")
+ .append(stat.getLen())
+ .append("\n");
+ }
+ return builder.toString();
+ }
+
+ public String cat(Path path) throws IOException {
+ FileStatus status = fileSystem.getFileStatus(path);
+ byte[] b = new byte[(int) status.getLen()];
+ FSDataInputStream in = null;
+ try {
+ in = fileSystem.open(path);
+ int count = in.read(b);
+ return new String(b, 0, count, UTF_8);
+ } finally {
+ IOUtils.closeStream(in);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java
new file mode 100644
index 0000000..6fadfd3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import java.io.Closeable;
+
+/**
+ * A duration in milliseconds. This class can be used
+ * to count time, and to be polled to see if a time limit has
+ * passed.
+ */
+public class Duration implements Closeable {
+ public long start, finish;
+ public final long limit;
+
+ /**
+ * Create a duration instance with a limit of 0
+ */
+ public Duration() {
+ this(0);
+ }
+
+ /**
+ * Create a duration with a limit specified in millis
+ * @param limit duration in milliseconds
+ */
+ public Duration(long limit) {
+ this.limit = limit;
+ }
+
+ /**
+ * Start
+ * @return self
+ */
+ public Duration start() {
+ start = now();
+ return this;
+ }
+
+ /**
+ * The close operation relays to {@link #finish()}.
+ * Implementing it allows Duration instances to be automatically
+ * finish()'d in Java7 try blocks for when used in measuring durations.
+ */
+ @Override
+ public final void close() {
+ finish();
+ }
+
+ public void finish() {
+ finish = now();
+ }
+
+ protected long now() {
+ return System.nanoTime()/1000000;
+ }
+
+ public long getInterval() {
+ return finish - start;
+ }
+
+ /**
+ * return true if the limit has been exceeded
+ * @return true if a limit was set and the current time
+ * exceeds it.
+ */
+ public boolean getLimitExceeded() {
+ return limit >= 0 && ((now() - start) > limit);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("Duration");
+ if (finish >= start) {
+ builder.append(" finished at ").append(getInterval()).append(" millis;");
+ } else {
+ if (start > 0) {
+ builder.append(" started but not yet finished;");
+ } else {
+ builder.append(" unstarted;");
+ }
+ }
+ if (limit > 0) {
+ builder.append(" limit: ").append(limit).append(" millis");
+ if (getLimitExceeded()) {
+ builder.append(" - exceeded");
+ }
+ }
+ return builder.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
new file mode 100644
index 0000000..7b22e3e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Support for marshalling objects to and from JSON.
+ * This class is NOT thread safe; it constructs an object mapper
+ * as an instance field.
+ * @param <T>
+ */
+public class JsonSerDeser<T> {
+
+ private static final Logger log = LoggerFactory.getLogger(JsonSerDeser.class);
+ private static final String UTF_8 = "UTF-8";
+
+ private final Class<T> classType;
+ private final ObjectMapper mapper;
+
+ /**
+ * Create an instance bound to a specific type
+ * @param classType class type
+ */
+ public JsonSerDeser(Class<T> classType) {
+ this.classType = classType;
+ this.mapper = new ObjectMapper();
+ mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ public JsonSerDeser(Class<T> classType, PropertyNamingStrategy namingStrategy) {
+ this(classType);
+ mapper.setPropertyNamingStrategy(namingStrategy);
+ }
+
+ /**
+ * Convert from JSON
+ * @param json input
+ * @return the parsed JSON
+ * @throws IOException IO
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public T fromJson(String json)
+ throws IOException, JsonParseException, JsonMappingException {
+ try {
+ return mapper.readValue(json, classType);
+ } catch (IOException e) {
+ log.error("Exception while parsing json : " + e + "\n" + json, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Convert from a JSON file
+ * @param jsonFile input file
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public T fromFile(File jsonFile)
+ throws IOException, JsonParseException, JsonMappingException {
+ File absoluteFile = jsonFile.getAbsoluteFile();
+ try {
+ return mapper.readValue(absoluteFile, classType);
+ } catch (IOException e) {
+ log.error("Exception while parsing json file {}", absoluteFile, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Convert from a JSON file
+ * @param resource input file
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public T fromResource(String resource)
+ throws IOException, JsonParseException, JsonMappingException {
+ try(InputStream resStream = this.getClass().getResourceAsStream(resource)) {
+ if (resStream == null) {
+ throw new FileNotFoundException(resource);
+ }
+ return (T) (mapper.readValue(resStream, classType));
+ } catch (IOException e) {
+ log.error("Exception while parsing json resource {}", resource, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Convert from an input stream, closing the stream afterwards.
+ * @param stream
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ */
+ public T fromStream(InputStream stream) throws IOException {
+ try {
+ return (T) (mapper.readValue(stream, classType));
+ } catch (IOException e) {
+ log.error("Exception while parsing json input stream", e);
+ throw e;
+ } finally {
+ IOUtils.closeStream(stream);
+ }
+ }
+
+ /**
+ * clone by converting to JSON and back again.
+ * This is much less efficient than any Java clone process.
+ * @param instance instance to duplicate
+ * @return a new instance
+ * @throws IOException problems.
+ */
+ public T fromInstance(T instance) throws IOException {
+ return fromJson(toJson(instance));
+ }
+
+ /**
+ * Deserialize from a byte array
+ * @param b
+ * @return the deserialized value
+ * @throws IOException parse problems
+ */
+ public T fromBytes(byte[] b) throws IOException {
+ String json = new String(b, 0, b.length, UTF_8);
+ return fromJson(json);
+ }
+
+ /**
+ * Load from a Hadoop filesystem
+ * @param fs filesystem
+ * @param path path
+ * @return a loaded CD
+ * @throws IOException IO problems
+ * @throws JsonParseException parse problems
+ * @throws JsonMappingException O/J mapping problems
+ */
+ public T load(FileSystem fs, Path path)
+ throws IOException, JsonParseException, JsonMappingException {
+ FileStatus status = fs.getFileStatus(path);
+ long len = status.getLen();
+ byte[] b = new byte[(int) len];
+ FSDataInputStream dataInputStream = fs.open(path);
+ int count = dataInputStream.read(b);
+ if (count != len) {
+ throw new EOFException("Read of " + path +" finished prematurely");
+ }
+ return fromBytes(b);
+ }
+
+
+ /**
+ * Save to a hadoop filesystem
+ * @param fs filesystem
+ * @param path path
+ * @param instance instance to save
+ * @param overwrite should any existing file be overwritten
+ * @throws IOException IO exception
+ */
+ public void save(FileSystem fs, Path path, T instance,
+ boolean overwrite) throws
+ IOException {
+ FSDataOutputStream dataOutputStream = fs.create(path, overwrite);
+ writeJsonAsBytes(instance, dataOutputStream);
+ }
+
+ /**
+ * Save an instance to a file
+ * @param instance instance to save
+ * @param file file
+ * @throws IOException
+ */
+ public void save(T instance, File file) throws
+ IOException {
+ writeJsonAsBytes(instance, new FileOutputStream(file.getAbsoluteFile()));
+ }
+
+ /**
+ * Write the json as bytes -then close the file
+ * @param dataOutputStream an outout stream that will always be closed
+ * @throws IOException on any failure
+ */
+ private void writeJsonAsBytes(T instance,
+ OutputStream dataOutputStream) throws IOException {
+ try {
+ String json = toJson(instance);
+ byte[] b = json.getBytes(UTF_8);
+ dataOutputStream.write(b);
+ dataOutputStream.flush();
+ dataOutputStream.close();
+ } finally {
+ IOUtils.closeStream(dataOutputStream);
+ }
+ }
+
+ /**
+ * Convert an object to a JSON string
+ * @param instance instance to convert
+ * @return a JSON string description
+ * @throws JsonParseException parse problems
+ * @throws JsonMappingException O/J mapping problems
+ */
+ public String toJson(T instance) throws IOException,
+ JsonGenerationException,
+ JsonMappingException {
+ mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
+ return mapper.writeValueAsString(instance);
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
deleted file mode 100644
index 7446e82..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.RMOperationHandler;
-import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Kill a specific container
- */
-public class ActionKillContainer extends AsyncAction {
-
- /**
- * container to kill
- */
- private final ContainerId containerId;
-
- /**
- * handler for the operation
- */
- private final RMOperationHandlerActions operationHandler;
-
- /**
- * Kill a container
- * @param containerId container to kill
- * @param delay
- * @param timeUnit
- * @param operationHandler
- */
- public ActionKillContainer(
- ContainerId containerId,
- long delay,
- TimeUnit timeUnit,
- RMOperationHandlerActions operationHandler) {
- super("kill container", delay, timeUnit, ATTR_CHANGES_APP_SIZE);
- this.operationHandler = operationHandler;
- Preconditions.checkArgument(containerId != null);
-
- this.containerId = containerId;
- }
-
- /**
- * Get the container ID to kill
- * @return
- */
- public ContainerId getContainerId() {
- return containerId;
- }
-
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
- List<AbstractRMOperation> opsList = new LinkedList<>();
- ContainerReleaseOperation release = new ContainerReleaseOperation(containerId);
- opsList.add(release);
- //now apply the operations
- operationHandler.execute(opsList);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java
deleted file mode 100644
index 0d7f7d4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Asynchronous registration operation
- */
-public class ActionRegisterServiceInstance extends AsyncAction {
-
- private final String instanceName;
- private final ApplicationId appId;
- private final Application application;
- public ActionRegisterServiceInstance(String instanceName,
- ApplicationId appId, Application application) {
- super("ActionRegisterServiceInstance");
- this.instanceName = instanceName;
- this.appId = appId;
- this.application = application;
- }
-
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
-
- // YARN Registry do the registration
- appMaster.registerServiceInstance(instanceName, appId, application);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
deleted file mode 100644
index 358c844..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-
-import java.util.Locale;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Start a container
- * @see SliderAppMaster#startContainer(Container, ContainerLaunchContext, RoleInstance)
- */
-public class ActionStartContainer extends AsyncAction {
-
- private final Container container;
- private final ContainerLaunchContext ctx;
- private final RoleInstance instance;
-
- public ActionStartContainer(String name,
- Container container,
- ContainerLaunchContext ctx,
- RoleInstance instance,
- long delay, TimeUnit timeUnit) {
- super(
- String.format(Locale.ENGLISH,
- "%s %s: /",
- name , container.getId().toString()),
- delay,
- timeUnit);
- this.container = container;
- this.ctx = ctx;
- this.instance = instance;
- }
-
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
- appMaster.startContainer(container, ctx, instance);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
deleted file mode 100644
index 08e8086..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Action to tell a queue executor to stop -after handing this on/executing it
- */
-public class ActionStopQueue extends AsyncAction {
- private static final Logger log =
- LoggerFactory.getLogger(ActionStopQueue.class);
-
- public ActionStopQueue(long delay) {
- super("stop queue", delay);
- }
-
- public ActionStopQueue(long delay,
- TimeUnit timeUnit) {
- super("stop queue", delay, timeUnit);
- }
-
- public ActionStopQueue(String name,
- long delay,
- TimeUnit timeUnit) {
- super(name, delay, timeUnit);
- }
-
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
- log.warn("STOP");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
deleted file mode 100644
index 055cea5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.slider.core.exceptions.ExceptionConverter;
-import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.core.main.ExitCodeProvider;
-import org.apache.slider.core.main.LauncherExitCodes;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Trigger an AM exit. This is used to build the exit status message for YARN
- */
-public class ActionStopSlider extends AsyncAction {
-
- private int exitCode;
- private FinalApplicationStatus finalApplicationStatus;
- private String message;
- private final Exception ex;
-
- /**
- * Simple constructor
- * @param name action name
- */
- public ActionStopSlider(String name) {
- super(name);
- this.ex = null;
- }
-
- /**
- * Stop slider
- * @param name action name
- * @param delay execution delay
- * @param timeUnit delay time unit
- * @param exitCode process exit code
- * @param finalApplicationStatus yarn status
- * @param message message for AM
- */
- public ActionStopSlider(String name,
- long delay,
- TimeUnit timeUnit,
- int exitCode,
- FinalApplicationStatus finalApplicationStatus,
- String message) {
- super(name, delay, timeUnit, ATTR_HALTS_APP);
- this.exitCode = exitCode;
- this.finalApplicationStatus = finalApplicationStatus;
- this.message = message;
- this.ex = null;
- }
-
- /**
- * Stop slider
- * @param name action name
- * @param exitCode process exit code
- * @param finalApplicationStatus yarn status
- * @param message message for AM
- */
- public ActionStopSlider(String name,
- int exitCode,
- FinalApplicationStatus finalApplicationStatus,
- String message) {
- super(name);
- this.exitCode = exitCode;
- this.finalApplicationStatus = finalApplicationStatus;
- this.message = message;
- this.ex = null;
- }
-
- /**
- * Simple constructor
- * @param ex teardown exception
- */
- public ActionStopSlider(TriggerClusterTeardownException ex) {
- this("stop",
- ex.getExitCode(),
- ex.getFinalApplicationStatus(),
- ex.getMessage());
- }
-
- /**
- * Build from an exception.
- * <p>
- * If the exception implements
- * {@link ExitCodeProvider} then the exit code is extracted from that
- * @param ex exception.
- */
- public ActionStopSlider(Exception ex) {
- super("stop");
- if (ex instanceof ExitCodeProvider) {
- setExitCode(((ExitCodeProvider)ex).getExitCode());
- } else {
- setExitCode(LauncherExitCodes.EXIT_EXCEPTION_THROWN);
- }
- setFinalApplicationStatus(FinalApplicationStatus.FAILED);
- setMessage(ex.getMessage());
- this.ex = ex;
- }
-
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
- SliderAppMaster.getLog().info("SliderAppMasterApi.stopCluster: {}",
- message);
- appMaster.onAMStop(this);
- }
-
- @Override
- public String toString() {
- return String.format("%s: exit code = %d, %s: %s;",
- name, exitCode, finalApplicationStatus, message) ;
- }
-
- public int getExitCode() {
- return exitCode;
- }
-
- public void setExitCode(int exitCode) {
- this.exitCode = exitCode;
- }
-
- public FinalApplicationStatus getFinalApplicationStatus() {
- return finalApplicationStatus;
- }
-
- public void setFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus) {
- this.finalApplicationStatus = finalApplicationStatus;
- }
-
- public String getMessage() {
- return message;
- }
-
- public void setMessage(String message) {
- this.message = message;
- }
-
- public Exception getEx() {
- return ex;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionUpgradeContainers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionUpgradeContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionUpgradeContainers.java
deleted file mode 100644
index 05fcbcc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionUpgradeContainers.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-
-public class ActionUpgradeContainers extends AsyncAction {
- private int exitCode;
- private FinalApplicationStatus finalApplicationStatus;
- private String message;
- private Set<String> containers = new HashSet<>();
- private Set<String> components = new HashSet<>();
-
- public ActionUpgradeContainers(String name,
- long delay,
- TimeUnit timeUnit,
- int exitCode,
- FinalApplicationStatus finalApplicationStatus,
- List<String> containers,
- List<String> components,
- String message) {
- super(name, delay, timeUnit);
- this.exitCode = exitCode;
- this.finalApplicationStatus = finalApplicationStatus;
- this.containers.addAll(containers);
- this.components.addAll(components);
- this.message = message;
- }
-
- @Override
- public void execute(SliderAppMaster appMaster, QueueAccess queueService,
- AppState appState) throws Exception {
- if (CollectionUtils.isNotEmpty(this.containers)
- || CollectionUtils.isNotEmpty(this.components)) {
- SliderAppMaster.getLog().info("SliderAppMaster.upgradeContainers: {}",
- message);
- appMaster.onUpgradeContainers(this);
- }
- }
-
- public int getExitCode() {
- return exitCode;
- }
-
- public void setExitCode(int exitCode) {
- this.exitCode = exitCode;
- }
-
- public FinalApplicationStatus getFinalApplicationStatus() {
- return finalApplicationStatus;
- }
-
- public void setFinalApplicationStatus(
- FinalApplicationStatus finalApplicationStatus) {
- this.finalApplicationStatus = finalApplicationStatus;
- }
-
- public String getMessage() {
- return message;
- }
-
- public void setMessage(String message) {
- this.message = message;
- }
-
- public Set<String> getContainers() {
- return containers;
- }
-
- public void setContainers(Set<String> containers) {
- this.containers = containers;
- }
-
- public Set<String> getComponents() {
- return components;
- }
-
- public void setComponents(Set<String> components) {
- this.components = components;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
deleted file mode 100644
index f9a1fd5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.io.IOException;
-import java.util.concurrent.Delayed;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-public abstract class AsyncAction implements Delayed {
-
- private static final AtomicLong sequencer = new AtomicLong(0);
-
- public final String name;
- private long nanos;
- public final int attrs;
- private final long sequenceNumber = sequencer.incrementAndGet();
-
-
- protected AsyncAction(String name) {
- this(name, 0);
- }
-
- protected AsyncAction(String name,
- long delayMillis) {
- this(name, delayMillis, TimeUnit.MILLISECONDS);
- }
-
- protected AsyncAction(String name,
- long delay,
- TimeUnit timeUnit) {
- this(name, delay, timeUnit, 0);
- }
-
- protected AsyncAction(String name,
- long delay,
- TimeUnit timeUnit,
- int attrs) {
- this.name = name;
- this.setNanos(convertAndOffset(delay, timeUnit));
- this.attrs = attrs;
- }
-
- protected long convertAndOffset(long delay, TimeUnit timeUnit) {
- return now() + TimeUnit.NANOSECONDS.convert(delay, timeUnit);
- }
-
- /**
- * The current time in nanos
- * @return now
- */
- protected long now() {
- return System.nanoTime();
- }
-
- @Override
- public long getDelay(TimeUnit unit) {
- return unit.convert(getNanos() - now(), TimeUnit.NANOSECONDS);
- }
-
- @Override
- public int compareTo(Delayed that) {
- if (this == that) {
- return 0;
- }
- return SliderUtils.compareTo(
- getDelay(TimeUnit.NANOSECONDS),
- that.getDelay(TimeUnit.NANOSECONDS));
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder(super.toString());
- sb.append(" name='").append(name).append('\'');
- sb.append(", delay=").append(getDelay(TimeUnit.SECONDS));
- sb.append(", attrs=").append(attrs);
- sb.append(", sequenceNumber=").append(sequenceNumber);
- sb.append('}');
- return sb.toString();
- }
-
- protected int getAttrs() {
- return attrs;
- }
-
- /**
- * Ask if an action has an of the specified bits set.
- * This is not an equality test.
- * @param attr attribute
- * @return true iff the action has any of the bits in the attr arg set
- */
- public boolean hasAttr(int attr) {
- return (attrs & attr) != 0;
- }
-
- /**
- * Actual application
- * @param appMaster
- * @param queueService
- * @param appState
- * @throws IOException
- */
- public abstract void execute(SliderAppMaster appMaster,
- QueueAccess queueService, AppState appState) throws Exception;
-
- public long getNanos() {
- return nanos;
- }
-
- public void setNanos(long nanos) {
- this.nanos = nanos;
- }
-
- public static final int ATTR_CHANGES_APP_SIZE = 1;
- public static final int ATTR_HALTS_APP = 2;
- public static final int ATTR_REVIEWS_APP_SIZE = 4;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/EscalateOutstandingRequests.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/EscalateOutstandingRequests.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/EscalateOutstandingRequests.java
deleted file mode 100644
index 2c545ea..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/EscalateOutstandingRequests.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Escalate outstanding requests by asking AM
- */
-public class EscalateOutstandingRequests extends AsyncAction {
-
- public EscalateOutstandingRequests() {
- super("EscalateOutstandingRequests");
- }
-
- public EscalateOutstandingRequests(long delay,
- TimeUnit timeUnit) {
- super("EscalateOutstandingRequests", delay, timeUnit, ATTR_REVIEWS_APP_SIZE);
- }
-
- @Override
- public void execute(SliderAppMaster appMaster, QueueAccess queueService, AppState appState) throws
- Exception {
- appMaster.escalateOutstandingRequests();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/MonitorComponentInstances.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/MonitorComponentInstances.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/MonitorComponentInstances.java
deleted file mode 100644
index f7aa871..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/MonitorComponentInstances.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-
-/**
- * Execute readiness checks on component instances.
- */
-public class MonitorComponentInstances extends AsyncAction {
-
- public MonitorComponentInstances() {
- super("MonitorComponentInstance");
- }
-
- @Override
- public void execute(SliderAppMaster appMaster, QueueAccess queueService,
- AppState appState) throws Exception {
- appMaster.monitorComponentInstances();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
deleted file mode 100644
index 957a35f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-
-public class ProviderStartupCompleted extends AsyncAction {
-
- public ProviderStartupCompleted() {
- super("ProviderStartupCompleted");
- }
-
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
- appMaster.eventCallbackEvent(null);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
deleted file mode 100644
index 0396891..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-/**
- * Access for queue operations
- */
-public interface QueueAccess {
- /**
- * Put an action on the immediate queue -to be executed when the queue
- * reaches it.
- * @param action action to queue
- */
- void put(AsyncAction action);
-
- /**
- * Put a delayed action: this will only be added to the main queue
- * after its action time has been reached
- * @param action action to queue
- */
- void schedule(AsyncAction action);
-
- /**
- * Remove an action from the queues.
- * @param action action to remove
- * @return true if the action was removed
- */
- boolean remove(AsyncAction action);
-
- /**
- * Add a named renewing action
- * @param name name
- * @param renewingAction wrapped action
- */
- void renewing(String name,
- RenewingAction<? extends AsyncAction> renewingAction);
-
- /**
- * Look up a renewing action
- * @param name name of the action
- * @return the action or null if none was found
- */
- RenewingAction<? extends AsyncAction> lookupRenewingAction(String name);
-
- /**
- * Remove a renewing action
- * @param name action name name of the action
- * @return true if the action was found and removed.
- */
- boolean removeRenewingAction(String name);
-
- /**
- * Look in the immediate queue for any actions of a specific attribute
- */
- boolean hasQueuedActionWithAttribute(int attr);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
deleted file mode 100644
index d0fc2cf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Executor for async actions - hands them off to the AM as
- * appropriate
- */
-public class QueueExecutor implements Runnable {
- private static final Logger log =
- LoggerFactory.getLogger(QueueExecutor.class);
-
- private final SliderAppMaster appMaster;
- private final QueueService actionQueues;
- private final AppState appState;
-
-
- public QueueExecutor(SliderAppMaster appMaster,
- QueueService actionQueues) {
- Preconditions.checkNotNull(appMaster);
- Preconditions.checkNotNull(actionQueues);
-
- this.appMaster = appMaster;
- this.actionQueues = actionQueues;
- this.appState = appMaster.getAppState();
- }
-
- @VisibleForTesting
- public QueueExecutor(QueueService actionQueues) {
- Preconditions.checkNotNull(actionQueues);
- this.appMaster = null;
- this.appState = null;
- this.actionQueues = actionQueues;
- }
-
- /**
- * Run until the queue has been told to stop
- */
- @Override
- public void run() {
- AsyncAction take = null;
- try {
- log.info("Queue Executor run() started");
- do {
- take = actionQueues.actionQueue.take();
- log.debug("Executing {}", take);
-
- take.execute(appMaster, actionQueues, appState);
- log.debug("Completed {}", take);
-
- } while (!(take instanceof ActionStopQueue));
- log.info("Queue Executor run() stopped");
- } catch (InterruptedException e) {
- // interrupted: exit
- } catch (Throwable e) {
- log.error("Exception processing {}: {}", take, e, e);
- if (appMaster != null) {
- appMaster.onExceptionInThread(Thread.currentThread(), e);
- }
- }
- // tag completed
- actionQueues.complete();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
deleted file mode 100644
index 34acade..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-
-import org.apache.slider.server.services.workflow.ServiceThreadFactory;
-import org.apache.slider.server.services.workflow.WorkflowExecutorService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Iterator;
-import java.util.Map;
-import java.util.concurrent.BlockingDeque;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.DelayQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * The Queue service provides immediate and scheduled queues, as well
- * as an executor thread that moves queued actions from the scheduled
- * queue to the immediate one.
- *
- * <p>
- * This code to be revisited to see if all that was needed is the single scheduled
- * queue, implicitly making actions immediate by giving them an execution
- * time of "now". It would force having a sequence number to all actions, one
- * which the queue would have to set from its (monotonic, thread-safe) counter
- * on every submission, with a modified comparison operator. This would guarantee
- * that earlier submissions were picked before later ones.
- */
-public class QueueService extends WorkflowExecutorService<ExecutorService>
-implements Runnable, QueueAccess {
- private static final Logger log =
- LoggerFactory.getLogger(QueueService.class);
- public static final String NAME = "Action Queue";
- private final AtomicBoolean completed = new AtomicBoolean(false);
-
- /**
- * Immediate actions.
- */
- public final BlockingDeque<AsyncAction> actionQueue =
- new LinkedBlockingDeque<>();
-
- /**
- * Actions to be scheduled in the future
- */
- public final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<>();
-
- /**
- * Map of renewing actions by name ... this is to allow them to
- * be cancelled by name
- */
- private final Map<String, RenewingAction<? extends AsyncAction>> renewingActions
- = new ConcurrentHashMap<>();
-
- /**
- * Create a queue instance with a single thread executor
- */
- public QueueService() {
- super(NAME,
- ServiceThreadFactory.singleThreadExecutor(NAME, true));
- }
-
- @Override
- public void put(AsyncAction action) {
- log.debug("Queueing {}", action);
- actionQueue.add(action);
- }
-
- @Override
- public void schedule(AsyncAction action) {
- log.debug("Scheduling {}", action);
- scheduledActions.add(action);
- }
-
- @Override
- public boolean remove(AsyncAction action) {
- boolean removedFromDelayQueue = scheduledActions.remove(action);
- boolean removedFromActions = actionQueue.remove(action);
- return removedFromActions || removedFromDelayQueue;
- }
-
- @Override
- public void renewing(String name,
- RenewingAction<? extends AsyncAction> renewingAction) {
- log.debug("Adding renewing Action \"{}\": {}", name,
- renewingAction.getAction());
- if (removeRenewingAction(name)) {
- log.debug("Removed predecessor action");
- }
- renewingActions.put(name, renewingAction);
- schedule(renewingAction);
- }
-
- @Override
- public RenewingAction<? extends AsyncAction> lookupRenewingAction(String name) {
- return renewingActions.get(name);
- }
-
- @Override
- public boolean removeRenewingAction(String name) {
- RenewingAction<? extends AsyncAction> action = renewingActions.remove(name);
- return action != null && remove(action);
- }
-
- /**
- * Stop the service by scheduling an {@link ActionStopQueue} action
- * ..if the processor thread is working this will propagate through
- * and stop the queue handling after all other actions complete.
- * @throws Exception
- */
- @Override
- protected void serviceStop() throws Exception {
- ActionStopQueue stopQueue = new ActionStopQueue("serviceStop: "+ this,
- 0, TimeUnit.MILLISECONDS);
- schedule(stopQueue);
- super.serviceStop();
- }
-
- /**
- * Flush an action queue of all types of a specific action
- * @param clazz
- */
- protected void flushActionQueue(Class<? extends AsyncAction> clazz) {
- Iterator<AsyncAction> iterator =
- actionQueue.descendingIterator();
- while (iterator.hasNext()) {
- AsyncAction next = iterator.next();
- if (next.getClass().equals(clazz)) {
- iterator.remove();
- }
- }
- }
-
- @Override
- public boolean hasQueuedActionWithAttribute(int attr) {
- for (AsyncAction action : actionQueue) {
- if (action.hasAttr(attr)) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * Run until the queue has been told to stop
- */
- @Override
- public void run() {
- try {
-
- log.info("QueueService processor started");
-
- AsyncAction take;
- do {
- take = scheduledActions.take();
- log.debug("Propagating {}", take);
- actionQueue.put(take);
- } while (!(take instanceof ActionStopQueue));
- log.info("QueueService processor terminated");
- } catch (InterruptedException e) {
- // interrupted during actions
- }
- // the thread exits, but does not tag the service as complete. That's expected
- // to be done by the stop queue
- }
-
-
- /**
- * Check to see if the queue executor has completed
- * @return the status
- */
- public boolean isCompleted() {
- return completed.get();
- }
-
- /**
- * Package scoped method to mark the queue service as finished
- */
- void complete() {
- completed.set(true);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
deleted file mode 100644
index 3c1bed8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Notify the app master that it should register a component instance
- * in the registry
- * {@link SliderAppMaster#registerComponent(ContainerId)}
- */
-public class RegisterComponentInstance extends AsyncAction {
-
- public final ContainerId containerId;
- public final RoleInstance roleInstance;
-
- public RegisterComponentInstance(ContainerId containerId,
- RoleInstance roleInstance,
- long delay,
- TimeUnit timeUnit) {
- super("RegisterComponentInstance :" + containerId,
- delay, timeUnit);
- this.roleInstance = roleInstance;
- Preconditions.checkArgument(containerId != null);
- this.containerId = containerId;
- }
-
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
-
- appMaster.registerComponent(containerId, roleInstance);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
deleted file mode 100644
index f3143ea..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import com.google.common.base.Preconditions;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-/**
- * This action executes then reschedules an inner action; a limit
- * can specify the number of times to run
- */
-
-public class RenewingAction<A extends AsyncAction> extends AsyncAction {
- private static final Logger log =
- LoggerFactory.getLogger(RenewingAction.class);
- private final A action;
- private long interval;
- private TimeUnit timeUnit;
- public final AtomicInteger executionCount = new AtomicInteger();
- private final ReentrantReadWriteLock intervalLock = new ReentrantReadWriteLock();
- private final Lock intervalReadLock = intervalLock.readLock();
- private final Lock intervalWriteLock = intervalLock.writeLock();
- public final int limit;
-
-
- /**
- * Rescheduling action
- * @param action action to execute
- * @param initialDelay initial delay
- * @param interval interval for later delays
- * @param timeUnit time unit for all times
- * @param limit limit on the no. of executions. If 0 or less: no limit
- */
- public RenewingAction(A action,
- long initialDelay,
- long interval,
- TimeUnit timeUnit,
- int limit) {
- super("renewing " + action.name, initialDelay, timeUnit, action.getAttrs());
- Preconditions.checkArgument(interval > 0, "invalid interval: " + interval);
- this.action = action;
- this.interval = interval;
- this.timeUnit = timeUnit;
- this.limit = limit;
- }
-
- /**
- * Execute the inner action then reschedule ourselves
- * @param appMaster
- * @param queueService
- * @param appState
- * @throws Exception
- */
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState)
- throws Exception {
- long exCount = executionCount.incrementAndGet();
- log.debug("{}: Executing inner action count # {}", this, exCount);
- action.execute(appMaster, queueService, appState);
- boolean reschedule = true;
- if (limit > 0) {
- reschedule = limit > exCount;
- }
- if (reschedule) {
- this.setNanos(convertAndOffset(getInterval(), getTimeUnit()));
- log.debug("{}: rescheduling, new offset {} mS ", this,
- getDelay(TimeUnit.MILLISECONDS));
- queueService.schedule(this);
- }
- }
-
- /**
- * Get the action
- * @return
- */
- public A getAction() {
- return action;
- }
-
- public long getInterval() {
- intervalReadLock.lock();
- try {
- return interval;
- } finally {
- intervalReadLock.unlock();
- }
- }
-
- public void updateInterval(long delay, TimeUnit timeUnit) {
- intervalWriteLock.lock();
- try {
- interval = delay;
- this.timeUnit = timeUnit;
- } finally {
- intervalWriteLock.unlock();
- }
- }
-
- public TimeUnit getTimeUnit() {
- intervalReadLock.lock();
- try {
- return timeUnit;
- } finally {
- intervalReadLock.unlock();
- }
- }
-
- public int getExecutionCount() {
- return executionCount.get();
- }
-
- public int getLimit() {
- return limit;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ResetFailureWindow.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ResetFailureWindow.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ResetFailureWindow.java
deleted file mode 100644
index 36f58dd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ResetFailureWindow.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
-import org.apache.slider.server.appmaster.state.AppState;
-
-/**
- * Requests the AM to reset the failure window
- */
-public class ResetFailureWindow extends AsyncAction {
- private final RMOperationHandlerActions operationHandler;
-
- public ResetFailureWindow(RMOperationHandlerActions operationHandler) {
- super("ResetFailureWindow");
- this.operationHandler = operationHandler;
- }
-
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
- synchronized (appMaster) {
- appState.resetFailureCounts();
- AbstractRMOperation blacklistOperation = appState.updateBlacklist();
- if (blacklistOperation != null) {
- blacklistOperation.execute(operationHandler);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ReviewAndFlexApplicationSize.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ReviewAndFlexApplicationSize.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ReviewAndFlexApplicationSize.java
deleted file mode 100644
index bf7edf9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ReviewAndFlexApplicationSize.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Tell the AM to execute the full flex review operation
- */
-public class ReviewAndFlexApplicationSize extends AsyncAction {
-
- public ReviewAndFlexApplicationSize(String name,
- long delay,
- TimeUnit timeUnit) {
- super(name, delay, timeUnit, ATTR_REVIEWS_APP_SIZE);
- }
-
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
- appMaster.handleReviewAndFlexApplicationSize(this);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
deleted file mode 100644
index ac86333..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Tell AM to unregister this component instance
- */
-public class UnregisterComponentInstance extends AsyncAction {
-
-
- public final RoleInstance roleInstance;
-
- public UnregisterComponentInstance(long delay, TimeUnit timeUnit,
- RoleInstance roleInstance) {
- super("UnregisterComponentInstance :" + roleInstance.getCompInstanceName()
- + ", ContainerId = " + roleInstance.getContainerId(),
- delay, timeUnit);
- this.roleInstance = roleInstance;
- }
-
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
- appMaster.unregisterComponent(roleInstance);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetric.java
deleted file mode 100644
index 89dfbfd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetric.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Metric;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * A bool metric, mapped to an integer. true maps to 1, false to zero,
- */
-public class BoolMetric implements Metric, Gauge<Integer> {
-
- private final AtomicBoolean value;
-
- public BoolMetric(boolean b) {
- value = new AtomicBoolean(b);
- }
-
- public void set(boolean b) {
- value.set(b);
- }
-
- public boolean get() {
- return value.get();
- }
-
- @Override
- public Integer getValue() {
- return value.get() ? 1 : 0;
- }
-
- @Override
- public String toString() {
- return value.toString();
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- BoolMetric that = (BoolMetric) o;
- return get() == that.get();
- }
-
- @Override
- public int hashCode() {
- return value.hashCode();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetricPredicate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetricPredicate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetricPredicate.java
deleted file mode 100644
index 82bcd3a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetricPredicate.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Metric;
-
-/**
- * A metric which takes a predicate and returns 1 if the predicate evaluates
- * to true. The predicate is evaluated whenever the metric is read.
- */
-public class BoolMetricPredicate implements Metric, Gauge<Integer> {
-
- private final Eval predicate;
-
- public BoolMetricPredicate(Eval predicate) {
- this.predicate = predicate;
- }
-
- @Override
- public Integer getValue() {
- return predicate.eval() ? 1: 0;
- }
-
- public interface Eval {
- boolean eval();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
deleted file mode 100644
index c93467b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Metric;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * This is a {@link AtomicLong} which acts as a metrics gauge: its state can be exposed as
- * a metrics.
- * It also exposes some of the same method names as the Codahale Counter class, so that
- * it's easy to swap in.
- *
- */
-public class LongGauge extends AtomicLong implements Metric, Gauge<Long> {
-
- /**
- * Instantiate
- * @param val current value
- */
- public LongGauge(long val) {
- super(val);
- }
-
- /**
- * Instantiate with value 0
- */
- public LongGauge() {
- this(0);
- }
-
- /**
- * Get the value as a metric
- * @return current value
- */
- @Override
- public Long getValue() {
- return get();
- }
-
- /**
- * Method from {@Code counter}; used here for drop-in replacement
- * without any recompile
- * @return current value
- */
- public Long getCount() {
- return get();
- }
-
- /**
- * {@code ++}
- */
- public void inc() {
- incrementAndGet();
- }
-
- /**
- * {@code --}
- */
- public void dec() {
- decrementAndGet();
- }
-
- /**
- * Decrement to the floor of 0. Operations in parallel may cause confusion here,
- * but it will still never go below zero
- * @param delta delta
- * @return the current value
- */
- public long decToFloor(long delta) {
- long l = get();
- long r = l - delta;
- if (r < 0) {
- r = 0;
- }
- // if this fails, the decrement has been lost
- compareAndSet(l, r);
- return get();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongMetricFunction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongMetricFunction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongMetricFunction.java
deleted file mode 100644
index 1de7345..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongMetricFunction.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Metric;
-
-/**
- * A metric which takes a function to generate a long value.
- * The function is evaluated whenever the metric is read.
- */
-public class LongMetricFunction implements Metric, Gauge<Long> {
-
- private final Eval function;
-
- public LongMetricFunction(Eval function) {
- this.function = function;
- }
-
- @Override
- public Long getValue() {
- return function.eval();
- }
-
- public interface Eval {
- long eval();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MeterAndCounter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MeterAndCounter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MeterAndCounter.java
deleted file mode 100644
index 02ab7bc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MeterAndCounter.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.MetricRegistry;
-
-/**
- * A combined meter and counter that can be used to measure load.
- * Hash and equality are derived from the name
- */
-public class MeterAndCounter {
-
- /**
- * suffix for counters: {@value}
- */
- public static final String COUNTER = ".counter";
-
- /**
- * suffix for meters: {@value}
- */
- public static final String METER = ".meter";
-
- final Meter meter;
- final Counter counter;
- final String name;
-
- /**
- * Construct an instance
- * @param metrics metrics to bond to
- * @param name name before suffixes are appended
- */
- public MeterAndCounter(MetricRegistry metrics, String name) {
- this.name = name;
- counter = metrics.counter(name + COUNTER);
- meter = metrics.meter(name + METER);
- }
-
- /**
- * Construct an instance
- * @param metrics metrics to bond to
- * @param clazz class to use to derive name
- * @param name name before suffixes are appended
- */
-
- public MeterAndCounter(MetricRegistry metrics, Class clazz, String name) {
- this.name = name;
- counter = metrics.counter(MetricRegistry.name(clazz, name + COUNTER));
- meter = metrics.meter(MetricRegistry.name(clazz, name + METER));
- }
-
- /**
- * Increment the counter, mark the meter
- */
- public void mark() {
- counter.inc();
- meter.mark();
- }
-
- public void inc() {
- mark();
- }
-
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- MeterAndCounter that = (MeterAndCounter) o;
-
- return name.equals(that.name);
-
- }
-
- @Override
- public int hashCode() {
- return name.hashCode();
- }
-
- /**
- * Get the count.
- * @return the current count
- */
- public long getCount() {
- return counter.getCount();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
deleted file mode 100644
index 1fe8ea6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-import com.codahale.metrics.Metric;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.MetricSet;
-import com.codahale.metrics.health.HealthCheckRegistry;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.CompositeService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * Class for all metrics and monitoring
- */
-public class MetricsAndMonitoring extends CompositeService {
- protected static final Logger log =
- LoggerFactory.getLogger(MetricsAndMonitoring.class);
- public MetricsAndMonitoring(String name) {
- super(name);
- }
-
- public MetricsAndMonitoring() {
- super("MetricsAndMonitoring");
- }
-
- /**
- * Singleton of metrics registry
- */
- final MetricRegistry metrics = new MetricRegistry();
-
- final HealthCheckRegistry health = new HealthCheckRegistry();
-
- private final Map<String, MeterAndCounter> meterAndCounterMap
- = new ConcurrentHashMap<>();
-
- private final List<MetricSet> metricSets = new ArrayList<>();
-
- public static final int EVENT_LIMIT = 1000;
-
- public MetricRegistry getMetrics() {
- return metrics;
- }
-
- public HealthCheckRegistry getHealth() {
- return health;
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- addService(new MetricsBindingService("MetricsBindingService",
- metrics));
- super.serviceInit(conf);
- }
-
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
- for (MetricSet set : metricSets) {
- unregister(set);
- }
- }
-
- public MeterAndCounter getMeterAndCounter(String name) {
- return meterAndCounterMap.get(name);
- }
-
- /**
- * Get or create the meter/counter pair
- * @param name name of instance
- * @return an instance
- */
- public MeterAndCounter getOrCreateMeterAndCounter(String name) {
- MeterAndCounter instance = meterAndCounterMap.get(name);
- if (instance == null) {
- synchronized (this) {
- // check in a sync block
- instance = meterAndCounterMap.get(name);
- if (instance == null) {
- instance = new MeterAndCounter(metrics, name);
- meterAndCounterMap.put(name, instance);
- }
- }
- }
- return instance;
- }
-
- /**
- * Get a specific meter and mark it. This will create and register it on demand.
- * @param name name of meter/counter
- */
- public void markMeterAndCounter(String name) {
- MeterAndCounter meter = getOrCreateMeterAndCounter(name);
- meter.mark();
- }
-
- /**
- * Given a {@link Metric}, registers it under the given name.
- *
- * @param name the name of the metric
- * @param metric the metric
- * @param <T> the type of the metric
- * @return {@code metric}
- * @throws IllegalArgumentException if the name is already registered
- */
- public <T extends Metric> T register(String name, T metric) throws IllegalArgumentException {
- return metrics.register(name, metric);
- }
-
- public <T extends Metric> T register(Class<?> klass, T metric, String... names)
- throws IllegalArgumentException {
- return register(MetricRegistry.name(klass, names), metric);
- }
-
- /**
- * Add a metric set for registering and deregistration on service stop
- * @param metricSet metric set
- */
- public void addMetricSet(MetricSet metricSet) {
- metricSets.add(metricSet);
- metrics.registerAll(metricSet);
- }
-
- /**
- * add a metric set, giving each entry a prefix
- * @param prefix prefix (a trailing "." is automatically added)
- * @param metricSet the metric set to register
- */
- public void addMetricSet(String prefix, MetricSet metricSet) {
- addMetricSet(new PrefixedMetricsSet(prefix, metricSet));
- }
-
- /**
- * Unregister a metric set; robust
- * @param metricSet metric set to unregister
- */
- public void unregister(MetricSet metricSet) {
- for (String s : metricSet.getMetrics().keySet()) {
- try {
- metrics.remove(s);
- } catch (IllegalArgumentException e) {
- // log but continue
- log.info("Exception when trying to unregister {}", s, e);
- }
- }
- }
-}
-
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
new file mode 100644
index 0000000..415392a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
@@ -0,0 +1,1699 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.GlobFilter;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.zookeeper.server.util.KerberosUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * These are slider-specific Util methods
+ */
+public final class SliderUtils {
+
+ private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
+
+ /**
+ * Atomic bool to track whether or not process security has already been
+ * turned on (prevents re-entrancy)
+ */
+ private static final AtomicBoolean processSecurityAlreadyInitialized =
+ new AtomicBoolean(false);
+ public static final String JAVA_SECURITY_KRB5_REALM =
+ "java.security.krb5.realm";
+ public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
+
+ /**
+ * Winutils
+ */
+ public static final String WINUTILS = "WINUTILS.EXE";
+ /**
+ * name of openssl program
+ */
+ public static final String OPENSSL = "openssl";
+
+ /**
+ * name of python program
+ */
+ public static final String PYTHON = "python";
+
+ /**
+ * type of docker standalone application
+ */
+ public static final String DOCKER = "docker";
+ /**
+ * type of docker on yarn application
+ */
+ public static final String DOCKER_YARN = "yarn_docker";
+
+ public static final int NODE_LIST_LIMIT = 10;
+
+ private SliderUtils() {
+ }
+
+ /**
+ * Implementation of set-ness, groovy definition of true/false for a string
+ * @param s string
+ * @return true iff the string is neither null nor empty
+ */
+ public static boolean isUnset(String s) {
+ return s == null || s.isEmpty();
+ }
+
+ public static boolean isSet(String s) {
+ return !isUnset(s);
+ }
+
+ public static boolean isEmpty(Collection l) {
+ return l == null || l.isEmpty();
+ }
+
+ /**
+ * Probe for a collection existing and not being empty
+ * @param l collection
+ * @return true if the reference is valid and it contains entries
+ */
+
+ public static boolean isNotEmpty(Collection l) {
+ return l != null && !l.isEmpty();
+ }
+
+ /**
+ * Probe for a map existing and not being empty
+ * @param m map
+ * @return true if the reference is valid and it contains map entries
+ */
+ public static boolean isNotEmpty(Map m) {
+ return m != null && !m.isEmpty();
+ }
+
+ /*
+ * Validates whether num is an integer
+ * @param num
+ * @param msg the message to be shown in exception
+ */
+ @SuppressWarnings("ResultOfMethodCallIgnored")
+ private static void validateNumber(String num, String msg) throws
+ BadConfigException {
+ try {
+ Integer.parseInt(num);
+ } catch (NumberFormatException nfe) {
+ throw new BadConfigException(msg + num);
+ }
+ }
+
+ /*
+ * Translates the trailing JVM heapsize unit: g, G, m, M
+ * This assumes designated unit of 'm'
+ * @param heapsize
+ * @return heapsize in MB
+ */
+ public static String translateTrailingHeapUnit(String heapsize) throws
+ BadConfigException {
+ String errMsg = "Bad heapsize: ";
+ if (heapsize.endsWith("m") || heapsize.endsWith("M")) {
+ String num = heapsize.substring(0, heapsize.length() - 1);
+ validateNumber(num, errMsg);
+ return num;
+ }
+ if (heapsize.endsWith("g") || heapsize.endsWith("G")) {
+ String num = heapsize.substring(0, heapsize.length() - 1) + "000";
+ validateNumber(num, errMsg);
+ return num;
+ }
+ // check if specified heap size is a number
+ validateNumber(heapsize, errMsg);
+ return heapsize;
+ }
+
+ /**
+ * recursive directory delete
+ * @param dir dir to delete
+ * @throws IOException on any problem
+ */
+ public static void deleteDirectoryTree(File dir) throws IOException {
+ if (dir.exists()) {
+ if (dir.isDirectory()) {
+ log.info("Cleaning up {}", dir);
+ //delete the children
+ File[] files = dir.listFiles();
+ if (files == null) {
+ throw new IOException("listfiles() failed for " + dir);
+ }
+ for (File file : files) {
+ log.info("deleting {}", file);
+ if (!file.delete()) {
+ log.warn("Unable to delete " + file);
+ }
+ }
+ if (!dir.delete()) {
+ log.warn("Unable to delete " + dir);
+ }
+ } else {
+ throw new IOException("Not a directory " + dir);
+ }
+ } else {
+ //not found, do nothing
+ log.debug("No output dir yet");
+ }
+ }
+
+ /**
+ * Find a containing JAR
+ * @param clazz class to find
+ * @return the file
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ * @throws FileNotFoundException if the class did not resolve to a file
+ */
+ public static File findContainingJarOrFail(Class clazz) throws IOException {
+ File localFile = SliderUtils.findContainingJar(clazz);
+ if (null == localFile) {
+ throw new FileNotFoundException("Could not find JAR containing " + clazz);
+ }
+ return localFile;
+ }
+
+
+ /**
+ * Find a containing JAR
+ * @param my_class class to find
+ * @return the file or null if it is not found
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ */
+ public static File findContainingJar(Class my_class) throws IOException {
+ ClassLoader loader = my_class.getClassLoader();
+ if (loader == null) {
+ throw new IOException(
+ "Class " + my_class + " does not have a classloader!");
+ }
+ String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
+ Enumeration<URL> urlEnumeration = loader.getResources(class_file);
+ for (; urlEnumeration.hasMoreElements(); ) {
+ URL url = urlEnumeration.nextElement();
+ if ("jar".equals(url.getProtocol())) {
+ String toReturn = url.getPath();
+ if (toReturn.startsWith("file:")) {
+ toReturn = toReturn.substring("file:".length());
+ }
+ // URLDecoder is a misnamed class, since it actually decodes
+ // x-www-form-urlencoded MIME type rather than actual
+ // URL encoding (which the file path has). Therefore it would
+ // decode +s to ' 's which is incorrect (spaces are actually
+ // either unencoded or encoded as "%20"). Replace +s first, so
+ // that they are kept sacred during the decoding process.
+ toReturn = toReturn.replaceAll("\\+", "%2B");
+ toReturn = URLDecoder.decode(toReturn, "UTF-8");
+ String jarFilePath = toReturn.replaceAll("!.*$", "");
+ return new File(jarFilePath);
+ } else {
+ log.info("could not locate JAR containing {} URL={}", my_class, url);
+ }
+ }
+ return null;
+ }
+
+ public static void checkPort(String hostname, int port, int connectTimeout)
+ throws IOException {
+ InetSocketAddress addr = new InetSocketAddress(hostname, port);
+ checkPort(hostname, addr, connectTimeout);
+ }
+
+ @SuppressWarnings("SocketOpenedButNotSafelyClosed")
+ public static void checkPort(String name,
+ InetSocketAddress address,
+ int connectTimeout)
+ throws IOException {
+ try(Socket socket = new Socket()) {
+ socket.connect(address, connectTimeout);
+ } catch (Exception e) {
+ throw new IOException("Failed to connect to " + name
+ + " at " + address
+ + " after " + connectTimeout + "milliseconds"
+ + ": " + e,
+ e);
+ }
+ }
+
+ public static void checkURL(String name, String url, int timeout) throws
+ IOException {
+ InetSocketAddress address = NetUtils.createSocketAddr(url);
+ checkPort(name, address, timeout);
+ }
+
+ /**
+ * A required file
+ * @param role role of the file (for errors)
+ * @param filename the filename
+ * @throws ExitUtil.ExitException if the file is missing
+ * @return the file
+ */
+ public static File requiredFile(String filename, String role) throws
+ IOException {
+ if (filename.isEmpty()) {
+ throw new ExitUtil.ExitException(-1, role + " file not defined");
+ }
+ File file = new File(filename);
+ if (!file.exists()) {
+ throw new ExitUtil.ExitException(-1,
+ role + " file not found: " +
+ file.getCanonicalPath());
+ }
+ return file;
+ }
+
+ private static final PatternValidator clusternamePattern
+ = new PatternValidator("[a-z][a-z0-9_-]*");
+
+ private static final PatternValidator compNamePattern
+ = new PatternValidator("[a-z][a-z0-9-]*");
+
+ public static void validateCompName(String compName) {
+ compNamePattern.validate(compName);
+ }
+
+ /**
+ * Normalize a cluster name then verify that it is valid
+ * @param name proposed cluster name
+ * @return true iff it is valid
+ */
+ public static boolean isClusternameValid(String name) {
+ return name != null && clusternamePattern.matches(name);
+ }
+
+ public static boolean oldIsClusternameValid(String name) {
+ if (name == null || name.isEmpty()) {
+ return false;
+ }
+ int first = name.charAt(0);
+ if (0 == (Character.getType(first) & Character.LOWERCASE_LETTER)) {
+ return false;
+ }
+
+ for (int i = 0; i < name.length(); i++) {
+ int elt = (int) name.charAt(i);
+ int t = Character.getType(elt);
+ if (0 == (t & Character.LOWERCASE_LETTER)
+ && 0 == (t & Character.DECIMAL_DIGIT_NUMBER)
+ && elt != '-'
+ && elt != '_') {
+ return false;
+ }
+ if (!Character.isLetterOrDigit(elt) && elt != '-' && elt != '_') {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Copy a directory to a new FS -both paths must be qualified. If
+ * a directory needs to be created, supplied permissions can override
+ * the default values. Existing directories are not touched
+ * @param conf conf file
+ * @param srcDirPath src dir
+ * @param destDirPath dest dir
+ * @param permission permission for the dest directory; null means "default"
+ * @return # of files copies
+ */
+ @SuppressWarnings("deprecation")
+ public static int copyDirectory(Configuration conf,
+ Path srcDirPath,
+ Path destDirPath,
+ FsPermission permission) throws
+ IOException,
+ BadClusterStateException {
+ FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf);
+ FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf);
+ //list all paths in the src.
+ if (!srcFS.exists(srcDirPath)) {
+ throw new FileNotFoundException("Source dir not found " + srcDirPath);
+ }
+ if (!srcFS.isDirectory(srcDirPath)) {
+ throw new FileNotFoundException(
+ "Source dir not a directory " + srcDirPath);
+ }
+ GlobFilter dotFilter = new GlobFilter("[!.]*");
+ FileStatus[] entries = srcFS.listStatus(srcDirPath, dotFilter);
+ int srcFileCount = entries.length;
+ if (srcFileCount == 0) {
+ return 0;
+ }
+ if (permission == null) {
+ permission = FsPermission.getDirDefault();
+ }
+ if (!destFS.exists(destDirPath)) {
+ new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath,
+ permission);
+ }
+ Path[] sourcePaths = new Path[srcFileCount];
+ for (int i = 0; i < srcFileCount; i++) {
+ FileStatus e = entries[i];
+ Path srcFile = e.getPath();
+ if (srcFS.isDirectory(srcFile)) {
+ String msg = "Configuration dir " + srcDirPath
+ + " contains a directory " + srcFile;
+ log.warn(msg);
+ throw new IOException(msg);
+ }
+ log.debug("copying src conf file {}", srcFile);
+ sourcePaths[i] = srcFile;
+ }
+ log.debug("Copying {} files from {} to dest {}", srcFileCount,
+ srcDirPath,
+ destDirPath);
+ FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf);
+ return srcFileCount;
+ }
+
+ /**
+ * Copy a file to a new FS -both paths must be qualified.
+ * @param conf conf file
+ * @param srcFile src file
+ * @param destFile dest file
+ */
+ @SuppressWarnings("deprecation")
+ public static void copy(Configuration conf,
+ Path srcFile,
+ Path destFile) throws
+ IOException,
+ BadClusterStateException {
+ FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
+ //list all paths in the src.
+ if (!srcFS.exists(srcFile)) {
+ throw new FileNotFoundException("Source file not found " + srcFile);
+ }
+ if (!srcFS.isFile(srcFile)) {
+ throw new FileNotFoundException(
+ "Source file not a file " + srcFile);
+ }
+ FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
+ FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
+ }
+
+ /**
+ * Take a collection, return a list containing the string value of every
+ * element in the collection.
+ * @param c collection
+ * @return a stringified list
+ */
+ public static List<String> collectionToStringList(Collection c) {
+ List<String> l = new ArrayList<>(c.size());
+ for (Object o : c) {
+ l.add(o.toString());
+ }
+ return l;
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -including at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @return the joined entries
+ */
+ public static String join(Collection collection, String separator) {
+ return join(collection, separator, true);
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(Collection collection,
+ String separator,
+ boolean trailing) {
+ StringBuilder b = new StringBuilder();
+ // fast return on empty collection
+ if (collection.isEmpty()) {
+ return trailing ? separator : "";
+ }
+ for (Object o : collection) {
+ b.append(o);
+ b.append(separator);
+ }
+ int length = separator.length();
+ String s = b.toString();
+ return (trailing || s.isEmpty()) ?
+ s : (b.substring(0, b.length() - length));
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -including at the end
+ * @param collection strings
+ * @param separator separator string
+ * @return the joined entries
+ */
+ public static String join(String[] collection, String separator) {
+ return join(collection, separator, true);
+
+
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection strings
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(String[] collection, String separator,
+ boolean trailing) {
+ return join(Arrays.asList(collection), separator, trailing);
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -except at the end
+ * @param collection strings
+ * @param separator separator string
+ * @return the list
+ */
+ public static String joinWithInnerSeparator(String separator,
+ Object... collection) {
+ StringBuilder b = new StringBuilder();
+ boolean first = true;
+
+ for (Object o : collection) {
+ if (first) {
+ first = false;
+ } else {
+ b.append(separator);
+ }
+ b.append(o.toString());
+ b.append(separator);
+ }
+ return b.toString();
+ }
+
+ /**
+ * Resolve a mandatory environment variable
+ * @param key env var
+ * @return the resolved value
+ * @throws BadClusterStateException
+ */
+ public static String mandatoryEnvVariable(String key) throws
+ BadClusterStateException {
+ String v = System.getenv(key);
+ if (v == null) {
+ throw new BadClusterStateException("Missing Environment variable " + key);
+ }
+ return v;
+ }
+
+ public static String appReportToString(ApplicationReport r,
+ String separator) {
+ StringBuilder builder = new StringBuilder(512);
+ builder.append("application ")
+ .append(
+ r.getName())
+ .append("/")
+ .append(r.getApplicationType())
+ .append(separator);
+ Set<String> tags = r.getApplicationTags();
+ if (!tags.isEmpty()) {
+ for (String tag : tags) {
+ builder.append(tag).append(separator);
+ }
+ }
+ DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
+ dateFormat.setTimeZone(TimeZone.getDefault());
+ builder.append("state: ").append(r.getYarnApplicationState());
+ String trackingUrl = r.getTrackingUrl();
+ if (isSet(trackingUrl)) {
+ builder.append(separator).append("URL: ").append(trackingUrl);
+ }
+ builder.append(separator)
+ .append("Started: ")
+ .append(dateFormat.format(new Date(r.getStartTime())));
+ long finishTime = r.getFinishTime();
+ if (finishTime > 0) {
+ builder.append(separator)
+ .append("Finished: ")
+ .append(dateFormat.format(new Date(finishTime)));
+ }
+ String rpcHost = r.getHost();
+ if (!isSet(rpcHost)) {
+ builder.append(separator)
+ .append("RPC :")
+ .append(rpcHost)
+ .append(':')
+ .append(r.getRpcPort());
+ }
+ String diagnostics = r.getDiagnostics();
+ if (!isSet(diagnostics)) {
+ builder.append(separator).append("Diagnostics :").append(diagnostics);
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Filter a string value given a single filter
+ *
+ * @param value
+ * the string value to check
+ * @param filter
+ * a single string filter
+ * @return return true if value should be trapped, false if it should be let
+ * through
+ */
+ public static boolean filter(String value, String filter) {
+ return !(StringUtils.isEmpty(filter) || filter.equals(value));
+ }
+
+ /**
+ * Filter a string value given a set of filters
+ *
+ * @param value
+ * the string value to check
+ * @param filters
+ * a set of string filters
+ * @return return true if value should be trapped, false if it should be let
+ * through
+ */
+ public static boolean filter(String value, Set<String> filters) {
+ return !(filters.isEmpty() || filters.contains(value));
+ }
+
+ /**
+ * Sorts the given list of application reports, most recently started
+ * or finished instance first.
+ *
+ * @param instances list of instances
+ */
+ public static void sortApplicationsByMostRecent(List<ApplicationReport> instances) {
+ Collections.sort(instances, new MostRecentlyStartedOrFinishedFirst());
+ }
+
+ /**
+ * Sorts the given list of application reports
+ * Finished instances are ordered by finished time and running/accepted instances are
+ * ordered by start time
+ * Finally Instance are order by finished instances coming after running instances
+ *
+ * @param instances list of instances
+ */
+ public static void sortApplicationReport(List<ApplicationReport> instances) {
+ if (instances.size() <= 1) {
+ return;
+ }
+ List<ApplicationReport> nonLiveInstance =
+ new ArrayList<>(instances.size());
+ List<ApplicationReport> liveInstance =
+ new ArrayList<>(instances.size());
+
+ for (ApplicationReport report : instances) {
+ if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
+ ||
+ report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
+ liveInstance.add(report);
+ } else {
+ nonLiveInstance.add(report);
+ }
+ }
+
+ if (liveInstance.size() > 1) {
+ Collections.sort(liveInstance, new MostRecentlyStartedAppFirst());
+ }
+ if (nonLiveInstance.size() > 1) {
+ Collections.sort(nonLiveInstance, new MostRecentAppFinishFirst());
+ }
+ instances.clear();
+ instances.addAll(liveInstance);
+ instances.addAll(nonLiveInstance);
+ }
+
+ /**
+ * Merge in one map to another -all entries in the second map are
+ * merged into the first -overwriting any duplicate keys.
+ * @param first first map -the updated one.
+ * @param second the map that is merged in
+ * @return the first map
+ */
+ public static Map<String, String> mergeMap(Map<String, String> first,
+ Map<String, String> second) {
+ first.putAll(second);
+ return first;
+ }
+
+ /**
+ * Merge a set of entries into a map. This will take the entryset of
+ * a map, or a Hadoop collection itself
+ * @param dest destination
+ * @param entries entries
+ * @return dest -with the entries merged in
+ */
+ public static Map<String, String> mergeEntries(Map<String, String> dest,
+ Iterable<Map.Entry<String, String>> entries) {
+ for (Map.Entry<String, String> entry : entries) {
+ dest.put(entry.getKey(), entry.getValue());
+ }
+ return dest;
+ }
+
+ /**
+ * Generic map merge logic
+ * @param first first map
+ * @param second second map
+ * @param <T1> key type
+ * @param <T2> value type
+ * @return 'first' merged with the second
+ */
+ public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
+ Map<T1, T2> second) {
+ first.putAll(second);
+ return first;
+ }
+
+ /**
+ * Generic map merge logic
+ * @param first first map
+ * @param second second map
+ * @param <T1> key type
+ * @param <T2> value type
+ * @return 'first' merged with the second
+ */
+ public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
+ Map<T1, T2> second) {
+ Preconditions.checkArgument(first != null, "Null 'first' value");
+ Preconditions.checkArgument(second != null, "Null 'second' value");
+ for (Map.Entry<T1, T2> entry : second.entrySet()) {
+ T1 key = entry.getKey();
+ if (!first.containsKey(key)) {
+ first.put(key, entry.getValue());
+ }
+ }
+ return first;
+ }
+
+ /**
+ * Convert a map to a multi-line string for printing
+ * @param map map to stringify
+ * @return a string representation of the map
+ */
+ public static String stringifyMap(Map<String, String> map) {
+ StringBuilder builder = new StringBuilder();
+ for (Map.Entry<String, String> entry : map.entrySet()) {
+ builder.append(entry.getKey())
+ .append("=\"")
+ .append(entry.getValue())
+ .append("\"\n");
+
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Parse an int value, replacing it with defval if undefined;
+ * @param errorKey key to use in exceptions
+ * @param defVal default value to use if the key is not in the map
+ * @param min min value or -1 for do not check
+ * @param max max value or -1 for do not check
+ * @return the int value the integer value
+ * @throws BadConfigException if the value could not be parsed
+ */
+ public static int parseAndValidate(String errorKey,
+ String valS,
+ int defVal,
+ int min, int max) throws
+ BadConfigException {
+ if (valS == null) {
+ valS = Integer.toString(defVal);
+ }
+ String trim = valS.trim();
+ int val;
+ try {
+ val = Integer.decode(trim);
+ } catch (NumberFormatException e) {
+ throw new BadConfigException("Failed to parse value of "
+ + errorKey + ": \"" + trim + "\"");
+ }
+ if (min >= 0 && val < min) {
+ throw new BadConfigException("Value of "
+ + errorKey + ": " + val + ""
+ + "is less than the minimum of " + min);
+ }
+ if (max >= 0 && val > max) {
+ throw new BadConfigException("Value of "
+ + errorKey + ": " + val + ""
+ + "is more than the maximum of " + max);
+ }
+ return val;
+ }
+
+ public static InetSocketAddress getRmAddress(Configuration conf) {
+ return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
+ }
+
+ public static InetSocketAddress getRmSchedulerAddress(Configuration conf) {
+ return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
+ }
+
+ /**
+ * probe to see if the RM scheduler is defined
+ * @param conf config
+ * @return true if the RM scheduler address is set to
+ * something other than 0.0.0.0
+ */
+ public static boolean isRmSchedulerAddressDefined(Configuration conf) {
+ InetSocketAddress address = getRmSchedulerAddress(conf);
+ return isAddressDefined(address);
+ }
+
+ /**
+ * probe to see if the address
+ * @param address network address
+ * @return true if the scheduler address is set to
+ * something other than 0.0.0.0
+ */
+ public static boolean isAddressDefined(InetSocketAddress address) {
+ if (address == null || address.getHostString() == null) {
+ return false;
+ }
+ return !(address.getHostString().equals("0.0.0.0"));
+ }
+
+ public static void setRmAddress(Configuration conf, String rmAddr) {
+ conf.set(YarnConfiguration.RM_ADDRESS, rmAddr);
+ }
+
+ public static void setRmSchedulerAddress(Configuration conf, String rmAddr) {
+ conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);
+ }
+
+ public static boolean hasAppFinished(ApplicationReport report) {
+ return report == null ||
+ report.getYarnApplicationState().ordinal() >=
+ YarnApplicationState.FINISHED.ordinal();
+ }
+
+ public static String containerToString(Container container) {
+ if (container == null) {
+ return "null container";
+ }
+ return String.format(Locale.ENGLISH,
+ "ContainerID=%s nodeID=%s http=%s priority=%s resource=%s",
+ container.getId(),
+ container.getNodeId(),
+ container.getNodeHttpAddress(),
+ container.getPriority(),
+ container.getResource());
+ }
+
+ /**
+ * convert an AM report to a string for diagnostics
+ * @param report the report
+ * @return the string value
+ */
+ public static String reportToString(ApplicationReport report) {
+ if (report == null) {
+ return "Null application report";
+ }
+
+ return "App " + report.getName() + "/" + report.getApplicationType() +
+ "# " +
+ report.getApplicationId() + " user " + report.getUser() +
+ " is in state " + report.getYarnApplicationState() +
+ " RPC: " + report.getHost() + ":" + report.getRpcPort() +
+ " URL: " + report.getOriginalTrackingUrl();
+ }
+
+ /**
+ * Convert a YARN URL into a string value of a normal URL
+ * @param url URL
+ * @return string representatin
+ */
+ public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(url.getScheme()).append("://");
+ if (url.getHost() != null) {
+ builder.append(url.getHost()).append(":").append(url.getPort());
+ }
+ builder.append(url.getFile());
+ return builder.toString();
+ }
+
+ /**
+ * Get a random open port
+ * @return true if the port was available for listening on
+ */
+ public static int getOpenPort() throws IOException {
+ ServerSocket socket = null;
+ try {
+ socket = new ServerSocket(0);
+ return socket.getLocalPort();
+ } finally {
+ if (socket != null) {
+ socket.close();
+ }
+ }
+ }
+
+ /**
+ * See if a port is available for listening on by trying to listen
+ * on it and seeing if that works or fails.
+ * @param port port to listen to
+ * @return true if the port was available for listening on
+ */
+ public static boolean isPortAvailable(int port) {
+ try {
+ ServerSocket socket = new ServerSocket(port);
+ socket.close();
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ // Build env map: key -> value;
+ // value will be replaced by the corresponding value in tokenMap, if any.
+ public static Map<String, String> buildEnvMap(
+ org.apache.hadoop.yarn.service.api.records.Configuration conf,
+ Map<String,String> tokenMap) {
+ if (tokenMap == null) {
+ return conf.getEnv();
+ }
+ Map<String, String> env = new HashMap<>();
+ for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) {
+ String key = entry.getKey();
+ String val = entry.getValue();
+ for (Map.Entry<String,String> token : tokenMap.entrySet()) {
+ val = val.replaceAll(Pattern.quote(token.getKey()),
+ token.getValue());
+ }
+ env.put(key,val);
+ }
+ return env;
+ }
+
+ /**
+ * Apply a set of command line options to a cluster role map
+ * @param clusterRoleMap cluster role map to merge onto
+ * @param commandOptions command opts
+ */
+ public static void applyCommandLineRoleOptsToRoleMap(
+ Map<String, Map<String, String>> clusterRoleMap,
+ Map<String, Map<String, String>> commandOptions) {
+ for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
+ String key = entry.getKey();
+ Map<String, String> optionMap = entry.getValue();
+ Map<String, String> existingMap = clusterRoleMap.get(key);
+ if (existingMap == null) {
+ existingMap = new HashMap<String, String>();
+ }
+ log.debug("Overwriting role options with command line values {}",
+ stringifyMap(optionMap));
+ mergeMap(existingMap, optionMap);
+ //set or overwrite the role
+ clusterRoleMap.put(key, existingMap);
+ }
+ }
+
+ /**
+ * verify that the supplied cluster name is valid
+ * @param clustername cluster name
+ * @throws BadCommandArgumentsException if it is invalid
+ */
+ public static void validateClusterName(String clustername)
+ throws BadCommandArgumentsException {
+ if (!isClusternameValid(clustername)) {
+ throw new BadCommandArgumentsException(
+ "Illegal cluster name: " + clustername);
+ }
+ }
+
+ /**
+ * Verify that a Kerberos principal has been set -if not fail
+ * with an error message that actually tells you what is missing
+ * @param conf configuration to look at
+ * @param principal key of principal
+ * @throws BadConfigException if the key is not set
+ */
+ public static void verifyPrincipalSet(Configuration conf,
+ String principal) throws
+ BadConfigException {
+ String principalName = conf.get(principal);
+ if (principalName == null) {
+ throw new BadConfigException("Unset Kerberos principal : %s",
+ principal);
+ }
+ log.debug("Kerberos princial {}={}", principal, principalName);
+ }
+
+ /**
+ * Flag to indicate whether the cluster is in secure mode
+ * @param conf configuration to look at
+ * @return true if the slider client/service should be in secure mode
+ */
+ public static boolean isHadoopClusterSecure(Configuration conf) {
+ return SecurityUtil.getAuthenticationMethod(conf) !=
+ UserGroupInformation.AuthenticationMethod.SIMPLE;
+ }
+
+ /**
+ * Init security if the cluster configuration declares the cluster is secure
+ * @param conf configuration to look at
+ * @return true if the cluster is secure
+ * @throws IOException cluster is secure
+ * @throws SliderException the configuration/process is invalid
+ */
+ public static boolean maybeInitSecurity(Configuration conf) throws
+ IOException,
+ SliderException {
+ boolean clusterSecure = isHadoopClusterSecure(conf);
+ if (clusterSecure) {
+ log.debug("Enabling security");
+ initProcessSecurity(conf);
+ }
+ return clusterSecure;
+ }
+
+ /**
+ * Turn on security. This is setup to only run once.
+ * @param conf configuration to build up security
+ * @return true if security was initialized in this call
+ * @throws IOException IO/Net problems
+ * @throws BadConfigException the configuration and system state are inconsistent
+ */
+ public static boolean initProcessSecurity(Configuration conf) throws
+ IOException,
+ SliderException {
+
+ if (processSecurityAlreadyInitialized.compareAndSet(true, true)) {
+ //security is already inited
+ return false;
+ }
+
+ log.info("JVM initialized into secure mode with kerberos realm {}",
+ SliderUtils.getKerberosRealm());
+ //this gets UGI to reset its previous world view (i.e simple auth)
+ //security
+ log.debug("java.security.krb5.realm={}",
+ System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
+ log.debug("java.security.krb5.kdc={}",
+ System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
+ log.debug("hadoop.security.authentication={}",
+ conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION));
+ log.debug("hadoop.security.authorization={}",
+ conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION));
+ UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation authUser = UserGroupInformation.getCurrentUser();
+ log.debug("Authenticating as {}", authUser);
+ log.debug("Login user is {}", UserGroupInformation.getLoginUser());
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ throw new SliderException(LauncherExitCodes.EXIT_UNAUTHORIZED,
+ "Although secure mode is enabled," +
+ "the application has already set up its user as an insecure entity %s",
+ authUser);
+ }
+ if (authUser.getAuthenticationMethod() ==
+ UserGroupInformation.AuthenticationMethod.SIMPLE) {
+ throw new BadConfigException("Auth User is not Kerberized %s" +
+ " -security has already been set up with the wrong authentication method. "
+ + "This can occur if a file system has already been created prior to the loading of "
+ + "the security configuration.",
+ authUser);
+
+ }
+
+ SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
+ SliderUtils.verifyPrincipalSet(conf, "dfs.namenode.kerberos.principal");
+ return true;
+ }
+
+ /**
+ * Force an early login: This catches any auth problems early rather than
+ * in RPC operations
+ * @throws IOException if the login fails
+ */
+ public static void forceLogin() throws IOException {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ if (UserGroupInformation.isLoginKeytabBased()) {
+ UserGroupInformation.getLoginUser().reloginFromKeytab();
+ } else {
+ UserGroupInformation.getLoginUser().reloginFromTicketCache();
+ }
+ }
+ }
+
+ public static String getLibDir() {
+ String[] libDirs = getLibDirs();
+ if (libDirs == null || libDirs.length == 0) {
+ return null;
+ }
+ return libDirs[0];
+ }
+
+ public static String[] getLibDirs() {
+ String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
+ if (isUnset(libDirStr)) {
+ return ArrayUtils.EMPTY_STRING_ARRAY;
+ }
+ return StringUtils.split(libDirStr, ',');
+ }
+
+ /**
+ * Submit a JAR containing a specific class and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param clazz class to look for
+ * @param libdir lib directory
+ * @param jarName <i>At the destination</i>
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public static LocalResource putJar(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Class clazz,
+ Path tempPath,
+ String libdir,
+ String jarName
+ )
+ throws IOException, SliderException {
+ LocalResource res = sliderFileSystem.submitJarWithClass(
+ clazz,
+ tempPath,
+ libdir,
+ jarName);
+ providerResources.put(libdir + "/" + jarName, res);
+ return res;
+ }
+
+ /**
+ * Submit a JAR containing and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param libDir lib directory
+ * @param srcPath copy jars from
+ */
+ public static void putAllJars(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libDir,
+ String srcPath) throws IOException, SliderException {
+ log.info("Loading all dependencies from {}", srcPath);
+ if (SliderUtils.isSet(srcPath)) {
+ File srcFolder = new File(srcPath);
+ FilenameFilter jarFilter = createJarFilter();
+ File[] listOfJars = srcFolder.listFiles(jarFilter);
+ if (listOfJars == null || listOfJars.length == 0) {
+ return;
+ }
+ for (File jarFile : listOfJars) {
+ LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
+ providerResources.put(libDir + "/" + jarFile.getName(), res);
+ }
+ }
+ }
+
+ /**
+ * Accept all filenames ending with {@code .jar}
+ * @return a filename filter
+ */
+ public static FilenameFilter createJarFilter() {
+ return new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
+ }
+ };
+ }
+
+ /**
+ * Submit the AM tar.gz containing all dependencies and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ */
+ public static void putAmTarGzipAndUpdate(
+ Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem
+ ) throws IOException, SliderException {
+ log.info("Loading all dependencies from {}{}",
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME,
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+ sliderFileSystem.submitTarGzipAndUpdate(providerResources);
+ }
+
+ public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
+ Map<String, Map<String, String>> dest = new HashMap<>();
+ for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) {
+ dest.put(entry.getKey(), stringMapClone(entry.getValue()));
+ }
+ return dest;
+ }
+
+ public static Map<String, String> stringMapClone(Map<String, String> src) {
+ Map<String, String> dest = new HashMap<>();
+ return mergeEntries(dest, src.entrySet());
+ }
+
+ /**
+ * List a directory in the local filesystem
+ * @param dir directory
+ * @return a listing, one to a line
+ */
+ public static String listDir(File dir) {
+ if (dir == null) {
+ return "";
+ }
+ String[] confDirEntries = dir.list();
+ if (confDirEntries == null) {
+ return "";
+ }
+ StringBuilder builder = new StringBuilder();
+ for (String entry : confDirEntries) {
+ builder.append(entry).append("\n");
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Create a file:// path from a local file
+ * @param file file to point the path
+ * @return a new Path
+ */
+ public static Path createLocalPath(File file) {
+ return new Path(file.toURI());
+ }
+
+ public static String getKerberosRealm() {
+ try {
+ return KerberosUtil.getDefaultRealm();
+ } catch (Exception e) {
+ log.debug("introspection into JVM internals failed", e);
+ return "(unknown)";
+
+ }
+ }
+
+ /**
+ * Build up the classpath for execution
+ * -behaves very differently on a mini test cluster vs a production
+ * production one.
+ *
+ * @param sliderConfDir relative path to the dir containing slider config
+ * options to put on the classpath -or null
+ * @param libdir directory containing the JAR files
+ * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
+ * (and hence the current classpath should be used, not anything built up)
+ * @return a classpath
+ */
+ public static ClasspathConstructor buildClasspath(String sliderConfDir,
+ String libdir,
+ SliderFileSystem sliderFileSystem,
+ boolean usingMiniMRCluster) {
+
+ ClasspathConstructor classpath = new ClasspathConstructor();
+ classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
+
+ // add the runtime classpath needed for tests to work
+ if (usingMiniMRCluster) {
+ // for mini cluster we pass down the java CP properties
+ // and nothing else
+ classpath.appendAll(classpath.localJVMClasspath());
+ } else {
+ if (sliderConfDir != null) {
+ classpath.addClassDirectory(sliderConfDir);
+ }
+ classpath.addLibDir(libdir);
+ if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
+ classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
+ } else {
+ log.info(
+ "For faster submission of apps, upload dependencies using cmd {} {}",
+ SliderActions.ACTION_DEPENDENCY, Arguments.ARG_UPLOAD);
+ }
+ classpath.addRemoteClasspathEnvVar();
+ classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
+ }
+ return classpath;
+ }
+
+ /**
+ * Verify that a path refers to a directory. If not
+ * logs the parent dir then throws an exception
+ * @param dir the directory
+ * @param errorlog log for output on an error
+ * @throws FileNotFoundException if it is not a directory
+ */
+ public static void verifyIsDir(File dir, Logger errorlog) throws
+ FileNotFoundException {
+ if (!dir.exists()) {
+ errorlog.warn("contents of {}: {}", dir,
+ listDir(dir.getParentFile()));
+ throw new FileNotFoundException(dir.toString());
+ }
+ if (!dir.isDirectory()) {
+ errorlog.info("contents of {}: {}", dir,
+ listDir(dir.getParentFile()));
+ throw new FileNotFoundException(
+ "Not a directory: " + dir);
+ }
+ }
+
+ /**
+ * Verify that a file exists
+ * @param file file
+ * @param errorlog log for output on an error
+ * @throws FileNotFoundException
+ */
+ public static void verifyFileExists(File file, Logger errorlog) throws
+ FileNotFoundException {
+ if (!file.exists()) {
+ errorlog.warn("contents of {}: {}", file,
+ listDir(file.getParentFile()));
+ throw new FileNotFoundException(file.toString());
+ }
+ if (!file.isFile()) {
+ throw new FileNotFoundException("Not a file: " + file.toString());
+ }
+ }
+
+ /**
+ * verify that a config option is set
+ * @param configuration config
+ * @param key key
+ * @return the value, in case it needs to be verified too
+ * @throws BadConfigException if the key is missing
+ */
+ public static String verifyOptionSet(Configuration configuration, String key,
+ boolean allowEmpty) throws BadConfigException {
+ String val = configuration.get(key);
+ if (val == null) {
+ throw new BadConfigException(
+ "Required configuration option \"%s\" not defined ", key);
+ }
+ if (!allowEmpty && val.isEmpty()) {
+ throw new BadConfigException(
+ "Configuration option \"%s\" must not be empty", key);
+ }
+ return val;
+ }
+
+ /**
+ * Verify that a keytab property is defined and refers to a non-empty file
+ *
+ * @param siteConf configuration
+ * @param prop property to look for
+ * @return the file referenced
+ * @throws BadConfigException on a failure
+ */
+ public static File verifyKeytabExists(Configuration siteConf,
+ String prop) throws
+ BadConfigException {
+ String keytab = siteConf.get(prop);
+ if (keytab == null) {
+ throw new BadConfigException("Missing keytab property %s",
+ prop);
+
+ }
+ File keytabFile = new File(keytab);
+ if (!keytabFile.exists()) {
+ throw new BadConfigException("Missing keytab file %s defined in %s",
+ keytabFile,
+ prop);
+ }
+ if (keytabFile.length() == 0 || !keytabFile.isFile()) {
+ throw new BadConfigException("Invalid keytab file %s defined in %s",
+ keytabFile,
+ prop);
+ }
+ return keytabFile;
+ }
+
+ /**
+ * Add a subpath to an existing URL. This extends
+ * the path, inserting a / between all entries
+ * if needed.
+ * @param base base path/URL
+ * @param path subpath
+ * @return base+"/"+subpath
+ */
+ public static String appendToURL(String base, String path) {
+ StringBuilder fullpath = new StringBuilder(base);
+ if (!base.endsWith("/")) {
+ fullpath.append("/");
+ }
+ if (path.startsWith("/")) {
+ fullpath.append(path.substring(1));
+ } else {
+ fullpath.append(path);
+ }
+ return fullpath.toString();
+ }
+
+ /**
+ * Truncate the given string to a maximum length provided
+ * with a pad (...) added to the end if expected size if more than 10.
+ * @param toTruncate string to truncate; may be null
+ * @param maxSize maximum size
+ * @return the truncated/padded string.
+ */
+ public static String truncate(String toTruncate, int maxSize) {
+ if (toTruncate == null || maxSize < 1
+ || toTruncate.length() <= maxSize) {
+ return toTruncate;
+ }
+
+ String pad = "...";
+ if (maxSize < 10) {
+ pad = "";
+ }
+ return toTruncate.substring(0, maxSize - pad.length()).concat(pad);
+ }
+
+ /**
+ * Given a source folder create zipped file
+ *
+ * @param srcFolder
+ * @param zipFile
+ *
+ * @throws IOException
+ */
+ public static void zipFolder(File srcFolder, File zipFile) throws IOException {
+ log.info("Zipping folder {} to {}", srcFolder.getAbsolutePath(), zipFile.getAbsolutePath());
+ List<String> files = new ArrayList<>();
+ generateFileList(files, srcFolder, srcFolder, true);
+
+ byte[] buffer = new byte[1024];
+
+ try (FileOutputStream fos = new FileOutputStream(zipFile)) {
+ try (ZipOutputStream zos = new ZipOutputStream(fos)) {
+
+ for (String file : files) {
+ ZipEntry ze = new ZipEntry(file);
+ zos.putNextEntry(ze);
+ try (FileInputStream in = new FileInputStream(srcFolder + File.separator + file)) {
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ zos.write(buffer, 0, len);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Given a source folder create a tar.gz file
+ *
+ * @param libDirs
+ * @param tarGzipFile
+ *
+ * @throws IOException
+ */
+ public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
+ FilenameFilter filter) throws IOException {
+ log.info("Tar-gzipping folders {} to {}", libDirs,
+ tarGzipFile.getAbsolutePath());
+
+ try(TarArchiveOutputStream taos =
+ new TarArchiveOutputStream(new GZIPOutputStream(
+ new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
+ for (String libDir : libDirs) {
+ File srcFolder = new File(libDir);
+ List<String> files = new ArrayList<>();
+ generateFileList(files, srcFolder, srcFolder, true, filter);
+ for (String file : files) {
+ File srcFile = new File(srcFolder, file);
+ TarArchiveEntry tarEntry = new TarArchiveEntry(
+ srcFile, file);
+ taos.putArchiveEntry(tarEntry);
+ try(FileInputStream in = new FileInputStream(srcFile)) {
+ org.apache.commons.io.IOUtils.copy(in, taos);
+ }
+ taos.flush();
+ taos.closeArchiveEntry();
+ }
+ }
+ }
+ }
+
+ private static void generateFileList(List<String> fileList, File node,
+ File rootFolder, Boolean relative) {
+ generateFileList(fileList, node, rootFolder, relative, null);
+ }
+
+ private static void generateFileList(List<String> fileList, File node,
+ File rootFolder, Boolean relative, FilenameFilter filter) {
+ if (node.isFile()) {
+ String fileFullPath = node.toString();
+ if (relative) {
+ fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1,
+ fileFullPath.length()));
+ } else {
+ fileList.add(fileFullPath);
+ }
+ }
+
+ if (node.isDirectory()) {
+ String[] subNode = node.list(filter);
+ if (subNode == null || subNode.length == 0) {
+ return;
+ }
+ for (String filename : subNode) {
+ generateFileList(fileList, new File(node, filename), rootFolder,
+ relative, filter);
+ }
+ }
+ }
+
+ /**
+ * Check for any needed libraries being present. On Unix none are needed;
+ * on windows they must be present
+ * @return true if all is well
+ */
+ public static String checkForRequiredNativeLibraries() {
+
+ if (!Shell.WINDOWS) {
+ return "";
+ }
+ StringBuilder errorText = new StringBuilder("");
+ if (!NativeIO.isAvailable()) {
+ errorText.append("No native IO library. ");
+ }
+ try {
+ String path = Shell.getQualifiedBinPath(WINUTILS);
+ log.debug("winutils is at {}", path);
+ } catch (IOException e) {
+ errorText.append("No " + WINUTILS);
+ log.warn("No winutils: {}", e, e);
+ }
+ try {
+ File target = new File("target");
+ FileUtil.canRead(target);
+ } catch (UnsatisfiedLinkError e) {
+ log.warn("Failing to link to native IO methods: {}", e, e);
+ errorText.append("No native IO methods");
+ }
+ return errorText.toString();
+ }
+
+ /**
+ * Strictly verify that windows utils is present.
+ * Checks go as far as opening the file and looking for
+ * the headers.
+ * @throws IOException on any problem reading the file
+ * @throws FileNotFoundException if the file is not considered valid
+ */
+ public static void maybeVerifyWinUtilsValid() throws
+ IOException,
+ SliderException {
+ String errorText = SliderUtils.checkForRequiredNativeLibraries();
+ if (!errorText.isEmpty()) {
+ throw new BadClusterStateException(errorText);
+ }
+ }
+
+ /**
+ * Write bytes to a file
+ * @param outfile output file
+ * @param data data to write
+ * @throws IOException on any IO problem
+ */
+ public static void write(File outfile, byte[] data)
+ throws IOException {
+ File parentDir = outfile.getCanonicalFile().getParentFile();
+ if (parentDir == null) {
+ throw new IOException(outfile.getPath() + " has no parent dir");
+ }
+ if (!parentDir.exists()) {
+ if(!parentDir.mkdirs()) {
+ throw new IOException("Failed to create parent directory " + parentDir);
+ }
+ }
+ SliderUtils.verifyIsDir(parentDir, log);
+ try(FileOutputStream out = new FileOutputStream(outfile)) {
+ out.write(data);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recent app comes first
+ * Specifically: the one whose start time value is greater.
+ */
+ private static class MostRecentlyStartedAppFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long x = r1.getStartTime();
+ long y = r2.getStartTime();
+ return compareTwoLongsReverse(x, y);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recent app comes first.
+ * "Recent"== the app whose start time <i>or finish time</i> is the greatest.
+ */
+ private static class MostRecentlyStartedOrFinishedFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long started1 = r1.getStartTime();
+ long started2 = r2.getStartTime();
+ long finished1 = r1.getFinishTime();
+ long finished2 = r2.getFinishTime();
+ long lastEvent1 = Math.max(started1, finished1);
+ long lastEvent2 = Math.max(started2, finished2);
+ return compareTwoLongsReverse(lastEvent1, lastEvent2);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recently finished app comes first
+ * Specifically: the one whose finish time value is greater.
+ */
+ private static class MostRecentAppFinishFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long x = r1.getFinishTime();
+ long y = r2.getFinishTime();
+ return compareTwoLongsReverse(x, y);
+ }
+ }
+
+ /**
+ * Compare two long values for sorting. As the return value for
+ * comparators must be int, the simple value of <code>x-y</code>
+ * is inapplicable
+ * @param x x value
+ * @param y y value
+ * @return +ve if x is less than y, -ve if y is greater than x; 0 for equality
+ */
+ public static int compareTwoLongsReverse(long x, long y) {
+ return (x < y) ? 1 : ((x == y) ? 0 : -1);
+ }
+
+ public static String createNameTag(String name) {
+ return "Name: " + name;
+ }
+
+ public static String createVersionTag(String version) {
+ return "Version: " + version;
+ }
+
+ public static String createDescriptionTag(String description) {
+ return "Description: " + description;
+ }
+
+ public static final String DAYS = ".days";
+ public static final String HOURS = ".hours";
+ public static final String MINUTES = ".minutes";
+ public static final String SECONDS = ".seconds";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
new file mode 100644
index 0000000..1fa07ce
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.net.HostAndPort;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ZookeeperUtils {
+ public static final int DEFAULT_PORT = 2181;
+
+ public static String buildConnectionString(String zkHosts, int port) {
+ String zkPort = Integer.toString(port);
+ //parse the hosts
+ String[] hostlist = zkHosts.split(",", 0);
+ String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
+ return quorum;
+ }
+
+ /**
+ * Take a quorum list and split it to (trimmed) pairs
+ * @param hostPortQuorumList list of form h1:port, h2:port2,...
+ * @return a possibly empty list of values between commas. They may not be
+ * valid hostname:port pairs
+ */
+ public static List<String> splitToPairs(String hostPortQuorumList) {
+ // split an address hot
+ String[] strings = StringUtils.getStrings(hostPortQuorumList);
+ int len = 0;
+ if (strings != null) {
+ len = strings.length;
+ }
+ List<String> tuples = new ArrayList<String>(len);
+ if (strings != null) {
+ for (String s : strings) {
+ tuples.add(s.trim());
+ }
+ }
+ return tuples;
+ }
+
+ /**
+ * Split a quorum list into a list of hostnames and ports
+ * @param hostPortQuorumList split to a list of hosts and ports
+ * @return a list of values
+ */
+ public static List<HostAndPort> splitToHostsAndPorts(String hostPortQuorumList) {
+ // split an address hot
+ String[] strings = StringUtils.getStrings(hostPortQuorumList);
+ int len = 0;
+ if (strings != null) {
+ len = strings.length;
+ }
+ List<HostAndPort> list = new ArrayList<HostAndPort>(len);
+ if (strings != null) {
+ for (String s : strings) {
+ list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
+ }
+ }
+ return list;
+ }
+
+ /**
+ * Build up to a hosts only list
+ * @param hostAndPorts
+ * @return a list of the hosts only
+ */
+ public static String buildHostsOnlyList(List<HostAndPort> hostAndPorts) {
+ StringBuilder sb = new StringBuilder();
+ for (HostAndPort hostAndPort : hostAndPorts) {
+ sb.append(hostAndPort.getHostText()).append(",");
+ }
+ if (sb.length() > 0) {
+ sb.delete(sb.length() - 1, sb.length());
+ }
+ return sb.toString();
+ }
+
+ public static String buildQuorumEntry(HostAndPort hostAndPort,
+ int defaultPort) {
+ String s = hostAndPort.toString();
+ if (hostAndPort.hasPort()) {
+ return s;
+ } else {
+ return s + ":" + defaultPort;
+ }
+ }
+
+ /**
+ * Build a quorum list, injecting a ":defaultPort" ref if needed on
+ * any entry without one
+ * @param hostAndPorts
+ * @param defaultPort
+ * @return
+ */
+ public static String buildQuorum(List<HostAndPort> hostAndPorts, int defaultPort) {
+ List<String> entries = new ArrayList<String>(hostAndPorts.size());
+ for (HostAndPort hostAndPort : hostAndPorts) {
+ entries.add(buildQuorumEntry(hostAndPort, defaultPort));
+ }
+ return SliderUtils.join(entries, ",", false);
+ }
+
+ public static String convertToHostsOnlyList(String quorum) throws
+ BadConfigException {
+ List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
+ return ZookeeperUtils.buildHostsOnlyList(hostAndPorts);
+ }
+
+ public static List<HostAndPort> splitToHostsAndPortsStrictly(String quorum) throws
+ BadConfigException {
+ List<HostAndPort> hostAndPorts =
+ ZookeeperUtils.splitToHostsAndPorts(quorum);
+ if (hostAndPorts.isEmpty()) {
+ throw new BadConfigException("empty zookeeper quorum");
+ }
+ return hostAndPorts;
+ }
+
+ public static int getFirstPort(String quorum, int defVal) throws
+ BadConfigException {
+ List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
+ int port = hostAndPorts.get(0).getPortOrDefault(defVal);
+ return port;
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/Slider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/Slider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/Slider.java
deleted file mode 100644
index 5fc8618..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/Slider.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider;
-
-import org.apache.slider.client.SliderClient;
-import org.apache.slider.core.main.ServiceLauncher;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * This is just the entry point class
- */
-public class Slider extends SliderClient {
-
-
- public static final String SERVICE_CLASSNAME = "org.apache.slider.Slider";
-
- /**
- * This is the main entry point for the service launcher.
- * @param args command line arguments.
- */
- public static void main(String[] args) {
-
- //turn the args to a list
- List<String> argsList = Arrays.asList(args);
- //create a new list, as the ArrayList type doesn't push() on an insert
- List<String> extendedArgs = new ArrayList<String>(argsList);
- //insert the service name
- extendedArgs.add(0, SERVICE_CLASSNAME);
- //now have the service launcher do its work
- ServiceLauncher.serviceMain(extendedArgs);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
deleted file mode 100644
index 8b0a563..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.api.proto.Messages;
-import org.codehaus.jackson.JsonParseException;
-import org.codehaus.jackson.annotate.JsonIgnore;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.JsonMappingException;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * Describe a specific node in the cluster
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL )
-public final class ClusterNode implements Cloneable {
- protected static final Logger
- LOG = LoggerFactory.getLogger(ClusterNode.class);
-
- @JsonIgnore
- public ContainerId containerId;
-
- /**
- * server name
- */
- public String name;
-
-
- /**
- * UUID of container used in Slider RPC to refer to instances
- */
- public String id;
-
- public String role;
-
- public int roleId;
-
- public long createTime;
- public long startTime;
- /**
- * flag set when it is released, to know if it has
- * already been targeted for termination
- */
- public boolean released;
- public String host;
- public String ip;
- public String hostname;
- public String hostUrl;
-
- /**
- * state from {@link ClusterDescription}
- */
- public int state;
-
- /**
- * Exit code: only valid if the state >= STOPPED
- */
- public int exitCode;
-
- /**
- * what was the command executed?
- */
- public String command;
-
- /**
- * Any diagnostics
- */
- public String diagnostics;
-
- /**
- * What is the tail output from the executed process (or [] if not started
- * or the log cannot be picked up
- */
- public String[] output;
-
- /**
- * Any environment details
- */
- public String[] environment;
-
- /**
- * server-side ctor takes the container ID and builds the name from it
- * @param containerId container ID; can be null
- */
- public ClusterNode(ContainerId containerId) {
- if (containerId != null) {
- this.containerId = containerId;
- this.name = containerId.toString();
- }
- }
-
- /**
- * ctor for deserialization
- */
- public ClusterNode() {
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append(name).append(": ");
- builder.append(state).append("\n");
- builder.append("state: ").append(state).append("\n");
- builder.append("role: ").append(role).append("\n");
- append(builder, "host", host);
- append(builder, "hostURL", hostUrl);
- append(builder, "command", command);
- if (output != null) {
- for (String line : output) {
- builder.append(line).append("\n");
- }
- }
- append(builder, "diagnostics", diagnostics);
- return builder.toString();
- }
-
- private void append(StringBuilder builder, String key, Object val) {
- if (val != null) {
- builder.append(key).append(": ").append(val.toString()).append("\n");
- }
- }
-
- /**
- * Convert to a JSON string
- * @return a JSON string description
- * @throws IOException Problems mapping/writing the object
- */
- public String toJsonString() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.writeValueAsString(this);
- }
-
-
- /**
- * Convert from JSON
- * @param json input
- * @return the parsed JSON
- * @throws IOException IO
- */
- public static ClusterNode fromJson(String json)
- throws IOException, JsonParseException, JsonMappingException {
- ObjectMapper mapper = new ObjectMapper();
- try {
- return mapper.readValue(json, ClusterNode.class);
- } catch (IOException e) {
- LOG.error("Exception while parsing json : {}\n{}", e , json, e);
- throw e;
- }
- }
-
- /**
- * Build from a protobuf response
- * @param message
- * @return the deserialized node
- */
- public static ClusterNode fromProtobuf(Messages.RoleInstanceState message) {
- ClusterNode node = new ClusterNode();
- node.name = message.getName();
- node.command = message.getCommand();
- node.diagnostics = message.getDiagnostics();
- String[] arr;
- int environmentCount = message.getEnvironmentCount();
- if (environmentCount > 0) {
- arr = new String[environmentCount];
- node.environment = message.getEnvironmentList().toArray(arr);
- }
- node.exitCode = message.getExitCode();
- int outputCount = message.getOutputCount();
- if (outputCount > 0) {
- arr = new String[outputCount];
- node.output = message.getOutputList().toArray(arr);
- }
- node.role = message.getRole();
- node.roleId = message.getRoleId();
- node.state = message.getState();
- node.host = message.getHost();
- node.hostUrl = message.getHostURL();
- node.createTime = message.getCreateTime();
- node.startTime = message.getStartTime();
- node.released = message.getReleased();
- return node;
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException {
- return super.clone();
- }
-
- public ClusterNode doClone() {
- try {
- return (ClusterNode)clone();
- } catch (CloneNotSupportedException e) {
- //not going to happen. This is a final class
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
deleted file mode 100644
index 0e3b535..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api;
-
-/**
- * Keys for internal use, go into `internal.json` and not intended for normal
- * use except when tuning Slider AM operations
- */
-public interface InternalKeys {
-
-
- /**
- * Home dir of the app: {@value}
- * If set, implies there is a home dir to use
- */
- String INTERNAL_APPLICATION_HOME = "internal.application.home";
- /**
- * Path to an image file containing the app: {@value}
- */
- String INTERNAL_APPLICATION_IMAGE_PATH = "internal.application.image.path";
- /**
- * Time in milliseconds to wait after forking any in-AM
- * process before attempting to start up the containers: {@value}
- *
- * A shorter value brings the cluster up faster, but means that if the
- * in AM process fails (due to a bad configuration), then time
- * is wasted starting containers on a cluster that isn't going to come
- * up
- */
- String INTERNAL_CONTAINER_STARTUP_DELAY = "internal.container.startup.delay";
- /**
- * internal temp directory: {@value}
- */
- String INTERNAL_AM_TMP_DIR = "internal.am.tmp.dir";
- /**
- * internal temp directory: {@value}
- */
- String INTERNAL_TMP_DIR = "internal.tmp.dir";
- /**
- * where a snapshot of the original conf dir is: {@value}
- */
- String INTERNAL_SNAPSHOT_CONF_PATH = "internal.snapshot.conf.path";
- /**
- * where a snapshot of the original conf dir is: {@value}
- */
- String INTERNAL_GENERATED_CONF_PATH = "internal.generated.conf.path";
- /**
- * where a snapshot of the original conf dir is: {@value}
- */
- String INTERNAL_PROVIDER_NAME = "internal.provider.name";
- /**
- * where a snapshot of the original conf dir is: {@value}
- */
- String INTERNAL_DATA_DIR_PATH = "internal.data.dir.path";
- /**
- * where the app def is stored
- */
- String INTERNAL_APPDEF_DIR_PATH = "internal.appdef.dir.path";
- /**
- * where addons for the app are stored
- */
- String INTERNAL_ADDONS_DIR_PATH = "internal.addons.dir.path";
- /**
- * Time in milliseconds to wait after forking any in-AM
- * process before attempting to start up the containers: {@value}
- *
- * A shorter value brings the cluster up faster, but means that if the
- * in AM process fails (due to a bad configuration), then time
- * is wasted starting containers on a cluster that isn't going to come
- * up
- */
- int DEFAULT_INTERNAL_CONTAINER_STARTUP_DELAY = 5000;
- /**
- * Time in seconds before a container is considered long-lived.
- * Shortlived containers are interpreted as a problem with the role
- * and/or the host: {@value}
- */
- String INTERNAL_CONTAINER_FAILURE_SHORTLIFE =
- "internal.container.failure.shortlife";
- /**
- * Default short life threshold: {@value}
- */
- int DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE = 60;
-
- /**
- * Version of the app: {@value}
- */
- String KEYTAB_LOCATION = "internal.keytab.location";
-
- /**
- * Queue used to deploy the app: {@value}
- */
- String INTERNAL_QUEUE = "internal.queue";
-
- /**
- * Flag to indicate whether or not the chaos monkey is enabled:
- * {@value}
- */
- String CHAOS_MONKEY_ENABLED = "internal.chaos.monkey.enabled";
- boolean DEFAULT_CHAOS_MONKEY_ENABLED = false;
-
-
- /**
- * Rate
- */
-
- String CHAOS_MONKEY_INTERVAL = "internal.chaos.monkey.interval";
- String CHAOS_MONKEY_INTERVAL_DAYS = CHAOS_MONKEY_INTERVAL + ".days";
- String CHAOS_MONKEY_INTERVAL_HOURS = CHAOS_MONKEY_INTERVAL + ".hours";
- String CHAOS_MONKEY_INTERVAL_MINUTES = CHAOS_MONKEY_INTERVAL + ".minutes";
- String CHAOS_MONKEY_INTERVAL_SECONDS = CHAOS_MONKEY_INTERVAL + ".seconds";
-
- long DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS = 0;
- long DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS = 0;
- long DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES = 0;
-
- String CHAOS_MONKEY_DELAY = "internal.chaos.monkey.delay";
- String CHAOS_MONKEY_DELAY_DAYS = CHAOS_MONKEY_DELAY + ".days";
- String CHAOS_MONKEY_DELAY_HOURS = CHAOS_MONKEY_DELAY + ".hours";
- String CHAOS_MONKEY_DELAY_MINUTES = CHAOS_MONKEY_DELAY + ".minutes";
- String CHAOS_MONKEY_DELAY_SECONDS = CHAOS_MONKEY_DELAY + ".seconds";
-
- int DEFAULT_CHAOS_MONKEY_STARTUP_DELAY = 0;
-
- /**
- * Prefix for all chaos monkey probabilities
- */
- String CHAOS_MONKEY_PROBABILITY =
- "internal.chaos.monkey.probability";
- /**
- * Probabilies are out of 10000 ; 100==1%
- */
-
- /**
- * Probability of a monkey check killing the AM: {@value}
- */
- String CHAOS_MONKEY_PROBABILITY_AM_FAILURE =
- CHAOS_MONKEY_PROBABILITY + ".amfailure";
-
- /**
- * Default probability of a monkey check killing the AM: {@value}
- */
- int DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = 0;
-
- /**
- * Probability of a monkey check killing the AM: {@value}
- */
- String CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE =
- CHAOS_MONKEY_PROBABILITY + ".amlaunchfailure";
-
- /**
- * Probability of a monkey check killing a container: {@value}
- */
-
- String CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE =
- CHAOS_MONKEY_PROBABILITY + ".containerfailure";
-
- /**
- * Default probability of a monkey check killing the a container: {@value}
- */
- int DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE = 0;
-
-
- /**
- * 1% of chaos
- */
- int PROBABILITY_PERCENT_1 = 100;
-
- /**
- * 100% for chaos values
- */
- int PROBABILITY_PERCENT_100 = 100 * PROBABILITY_PERCENT_1;
-
- /**
- * interval between checks for escalation: {@value}
- */
- String ESCALATION_CHECK_INTERVAL = "escalation.check.interval.seconds";
-
- /**
- * default value: {@value}
- */
- int DEFAULT_ESCALATION_CHECK_INTERVAL = 30;
-
-
- /**
- * interval between readiness checks: {@value}
- */
- String MONITOR_INTERVAL = "monitor.interval.seconds";
-
- /**
- * default value: {@value}
- */
- int DEFAULT_MONITOR_INTERVAL = 30;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
deleted file mode 100644
index 988627d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api;
-
-/**
- * Keys for entries in the <code>options</code> section
- * of a cluster description.
- */
-public interface OptionKeys extends InternalKeys {
-
- /**
- * Time in milliseconds to wait after forking any in-AM
- * process before attempting to start up the containers: {@value}
- *
- * A shorter value brings the cluster up faster, but means that if the
- * in AM process fails (due to a bad configuration), then time
- * is wasted starting containers on a cluster that isn't going to come
- * up
- */
- String APPLICATION_TYPE = "application.type";
-
- String APPLICATION_NAME = "application.name";
-
- /**
- * Prefix for site.xml options: {@value}
- */
- String SITE_XML_PREFIX = "site.";
- /**
- * Prefix for config file options: {@value}
- */
- String CONF_FILE_PREFIX = "conf.";
- /**
- * Prefix for package options: {@value}
- */
- String PKG_FILE_PREFIX = "pkg.";
- /**
- * Prefix for export options: {@value}
- */
- String EXPORT_PREFIX = "export.";
- /**
- * Type suffix for config file and package options: {@value}
- */
- String TYPE_SUFFIX = ".type";
- /**
- * Name suffix for config file and package options: {@value}
- */
- String NAME_SUFFIX = ".name";
- /**
- * Per component suffix for config file options: {@value}
- */
- String PER_COMPONENT = ".per.component";
- /**
- * Per group suffix for config file options: {@value}
- */
- String PER_GROUP = ".per.group";
-
- /**
- * Zookeeper quorum host list: {@value}
- */
- String ZOOKEEPER_QUORUM = "zookeeper.quorum";
- String ZOOKEEPER_HOSTS = "zookeeper.hosts";
- String ZOOKEEPER_PORT = "zookeeper.port";
-
- /**
- * Zookeeper path value (string): {@value}
- */
- String ZOOKEEPER_PATH = "zookeeper.path";
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[60/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
new file mode 100644
index 0000000..6e6f4dd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
@@ -0,0 +1,1654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.GlobFilter;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.zookeeper.server.util.KerberosUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * These are slider-specific Util methods
+ */
+public final class SliderUtils {
+
+ private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
+
+ /**
+ * Atomic bool to track whether or not process security has already been
+ * turned on (prevents re-entrancy)
+ */
+ private static final AtomicBoolean processSecurityAlreadyInitialized =
+ new AtomicBoolean(false);
+ public static final String JAVA_SECURITY_KRB5_REALM =
+ "java.security.krb5.realm";
+ public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
+
+ /**
+ * Winutils
+ */
+ public static final String WINUTILS = "WINUTILS.EXE";
+ /**
+ * name of openssl program
+ */
+ public static final String OPENSSL = "openssl";
+
+ /**
+ * name of python program
+ */
+ public static final String PYTHON = "python";
+
+ /**
+ * type of docker standalone service
+ */
+ public static final String DOCKER = "docker";
+ /**
+ * type of docker on yarn service
+ */
+ public static final String DOCKER_YARN = "yarn_docker";
+
+ public static final int NODE_LIST_LIMIT = 10;
+
+ private SliderUtils() {
+ }
+
+ /**
+ * Implementation of set-ness, groovy definition of true/false for a string
+ * @param s string
+ * @return true iff the string is neither null nor empty
+ */
+ public static boolean isUnset(String s) {
+ return s == null || s.isEmpty();
+ }
+
+ public static boolean isSet(String s) {
+ return !isUnset(s);
+ }
+
+ public static boolean isEmpty(Collection l) {
+ return l == null || l.isEmpty();
+ }
+
+ /**
+ * Probe for a collection existing and not being empty
+ * @param l collection
+ * @return true if the reference is valid and it contains entries
+ */
+
+ public static boolean isNotEmpty(Collection l) {
+ return l != null && !l.isEmpty();
+ }
+
+ /**
+ * Probe for a map existing and not being empty
+ * @param m map
+ * @return true if the reference is valid and it contains map entries
+ */
+ public static boolean isNotEmpty(Map m) {
+ return m != null && !m.isEmpty();
+ }
+
+ /*
+ * Validates whether num is an integer
+ * @param num
+ * @param msg the message to be shown in exception
+ */
+ @SuppressWarnings("ResultOfMethodCallIgnored")
+ private static void validateNumber(String num, String msg) throws
+ BadConfigException {
+ try {
+ Integer.parseInt(num);
+ } catch (NumberFormatException nfe) {
+ throw new BadConfigException(msg + num);
+ }
+ }
+
+ /*
+ * Translates the trailing JVM heapsize unit: g, G, m, M
+ * This assumes designated unit of 'm'
+ * @param heapsize
+ * @return heapsize in MB
+ */
+ public static String translateTrailingHeapUnit(String heapsize) throws
+ BadConfigException {
+ String errMsg = "Bad heapsize: ";
+ if (heapsize.endsWith("m") || heapsize.endsWith("M")) {
+ String num = heapsize.substring(0, heapsize.length() - 1);
+ validateNumber(num, errMsg);
+ return num;
+ }
+ if (heapsize.endsWith("g") || heapsize.endsWith("G")) {
+ String num = heapsize.substring(0, heapsize.length() - 1) + "000";
+ validateNumber(num, errMsg);
+ return num;
+ }
+ // check if specified heap size is a number
+ validateNumber(heapsize, errMsg);
+ return heapsize;
+ }
+
+ /**
+ * recursive directory delete
+ * @param dir dir to delete
+ * @throws IOException on any problem
+ */
+ public static void deleteDirectoryTree(File dir) throws IOException {
+ if (dir.exists()) {
+ if (dir.isDirectory()) {
+ log.info("Cleaning up {}", dir);
+ //delete the children
+ File[] files = dir.listFiles();
+ if (files == null) {
+ throw new IOException("listfiles() failed for " + dir);
+ }
+ for (File file : files) {
+ log.info("deleting {}", file);
+ if (!file.delete()) {
+ log.warn("Unable to delete " + file);
+ }
+ }
+ if (!dir.delete()) {
+ log.warn("Unable to delete " + dir);
+ }
+ } else {
+ throw new IOException("Not a directory " + dir);
+ }
+ } else {
+ //not found, do nothing
+ log.debug("No output dir yet");
+ }
+ }
+
+ /**
+ * Find a containing JAR
+ * @param clazz class to find
+ * @return the file
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ * @throws FileNotFoundException if the class did not resolve to a file
+ */
+ public static File findContainingJarOrFail(Class clazz) throws IOException {
+ File localFile = SliderUtils.findContainingJar(clazz);
+ if (null == localFile) {
+ throw new FileNotFoundException("Could not find JAR containing " + clazz);
+ }
+ return localFile;
+ }
+
+
+ /**
+ * Find a containing JAR
+ * @param my_class class to find
+ * @return the file or null if it is not found
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ */
+ public static File findContainingJar(Class my_class) throws IOException {
+ ClassLoader loader = my_class.getClassLoader();
+ if (loader == null) {
+ throw new IOException(
+ "Class " + my_class + " does not have a classloader!");
+ }
+ String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
+ Enumeration<URL> urlEnumeration = loader.getResources(class_file);
+ for (; urlEnumeration.hasMoreElements(); ) {
+ URL url = urlEnumeration.nextElement();
+ if ("jar".equals(url.getProtocol())) {
+ String toReturn = url.getPath();
+ if (toReturn.startsWith("file:")) {
+ toReturn = toReturn.substring("file:".length());
+ }
+ // URLDecoder is a misnamed class, since it actually decodes
+ // x-www-form-urlencoded MIME type rather than actual
+ // URL encoding (which the file path has). Therefore it would
+ // decode +s to ' 's which is incorrect (spaces are actually
+ // either unencoded or encoded as "%20"). Replace +s first, so
+ // that they are kept sacred during the decoding process.
+ toReturn = toReturn.replaceAll("\\+", "%2B");
+ toReturn = URLDecoder.decode(toReturn, "UTF-8");
+ String jarFilePath = toReturn.replaceAll("!.*$", "");
+ return new File(jarFilePath);
+ } else {
+ log.info("could not locate JAR containing {} URL={}", my_class, url);
+ }
+ }
+ return null;
+ }
+
+ public static void checkPort(String hostname, int port, int connectTimeout)
+ throws IOException {
+ InetSocketAddress addr = new InetSocketAddress(hostname, port);
+ checkPort(hostname, addr, connectTimeout);
+ }
+
+ @SuppressWarnings("SocketOpenedButNotSafelyClosed")
+ public static void checkPort(String name,
+ InetSocketAddress address,
+ int connectTimeout)
+ throws IOException {
+ try(Socket socket = new Socket()) {
+ socket.connect(address, connectTimeout);
+ } catch (Exception e) {
+ throw new IOException("Failed to connect to " + name
+ + " at " + address
+ + " after " + connectTimeout + "milliseconds"
+ + ": " + e,
+ e);
+ }
+ }
+
+ public static void checkURL(String name, String url, int timeout) throws
+ IOException {
+ InetSocketAddress address = NetUtils.createSocketAddr(url);
+ checkPort(name, address, timeout);
+ }
+
+ /**
+ * A required file
+ * @param role role of the file (for errors)
+ * @param filename the filename
+ * @throws ExitUtil.ExitException if the file is missing
+ * @return the file
+ */
+ public static File requiredFile(String filename, String role) throws
+ IOException {
+ if (filename.isEmpty()) {
+ throw new ExitUtil.ExitException(-1, role + " file not defined");
+ }
+ File file = new File(filename);
+ if (!file.exists()) {
+ throw new ExitUtil.ExitException(-1,
+ role + " file not found: " +
+ file.getCanonicalPath());
+ }
+ return file;
+ }
+
+ private static final PatternValidator clusternamePattern
+ = new PatternValidator("[a-z][a-z0-9_-]*");
+
+ /**
+ * Normalize a cluster name then verify that it is valid
+ * @param name proposed cluster name
+ * @return true iff it is valid
+ */
+ public static boolean isClusternameValid(String name) {
+ return name != null && clusternamePattern.matches(name);
+ }
+
+ /**
+ * Copy a directory to a new FS -both paths must be qualified. If
+ * a directory needs to be created, supplied permissions can override
+ * the default values. Existing directories are not touched
+ * @param conf conf file
+ * @param srcDirPath src dir
+ * @param destDirPath dest dir
+ * @param permission permission for the dest directory; null means "default"
+ * @return # of files copies
+ */
+ @SuppressWarnings("deprecation")
+ public static int copyDirectory(Configuration conf,
+ Path srcDirPath,
+ Path destDirPath,
+ FsPermission permission) throws
+ IOException,
+ BadClusterStateException {
+ FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf);
+ FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf);
+ //list all paths in the src.
+ if (!srcFS.exists(srcDirPath)) {
+ throw new FileNotFoundException("Source dir not found " + srcDirPath);
+ }
+ if (!srcFS.isDirectory(srcDirPath)) {
+ throw new FileNotFoundException(
+ "Source dir not a directory " + srcDirPath);
+ }
+ GlobFilter dotFilter = new GlobFilter("[!.]*");
+ FileStatus[] entries = srcFS.listStatus(srcDirPath, dotFilter);
+ int srcFileCount = entries.length;
+ if (srcFileCount == 0) {
+ return 0;
+ }
+ if (permission == null) {
+ permission = FsPermission.getDirDefault();
+ }
+ if (!destFS.exists(destDirPath)) {
+ new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath,
+ permission);
+ }
+ Path[] sourcePaths = new Path[srcFileCount];
+ for (int i = 0; i < srcFileCount; i++) {
+ FileStatus e = entries[i];
+ Path srcFile = e.getPath();
+ if (srcFS.isDirectory(srcFile)) {
+ String msg = "Configuration dir " + srcDirPath
+ + " contains a directory " + srcFile;
+ log.warn(msg);
+ throw new IOException(msg);
+ }
+ log.debug("copying src conf file {}", srcFile);
+ sourcePaths[i] = srcFile;
+ }
+ log.debug("Copying {} files from {} to dest {}", srcFileCount,
+ srcDirPath,
+ destDirPath);
+ FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf);
+ return srcFileCount;
+ }
+
+ /**
+ * Copy a file to a new FS -both paths must be qualified.
+ * @param conf conf file
+ * @param srcFile src file
+ * @param destFile dest file
+ */
+ @SuppressWarnings("deprecation")
+ public static void copy(Configuration conf,
+ Path srcFile,
+ Path destFile) throws
+ IOException,
+ BadClusterStateException {
+ FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
+ //list all paths in the src.
+ if (!srcFS.exists(srcFile)) {
+ throw new FileNotFoundException("Source file not found " + srcFile);
+ }
+ if (!srcFS.isFile(srcFile)) {
+ throw new FileNotFoundException(
+ "Source file not a file " + srcFile);
+ }
+ FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
+ FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
+ }
+
+ /**
+ * Take a collection, return a list containing the string value of every
+ * element in the collection.
+ * @param c collection
+ * @return a stringified list
+ */
+ public static List<String> collectionToStringList(Collection c) {
+ List<String> l = new ArrayList<>(c.size());
+ for (Object o : c) {
+ l.add(o.toString());
+ }
+ return l;
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -including at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @return the joined entries
+ */
+ public static String join(Collection collection, String separator) {
+ return join(collection, separator, true);
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(Collection collection,
+ String separator,
+ boolean trailing) {
+ StringBuilder b = new StringBuilder();
+ // fast return on empty collection
+ if (collection.isEmpty()) {
+ return trailing ? separator : "";
+ }
+ for (Object o : collection) {
+ b.append(o);
+ b.append(separator);
+ }
+ int length = separator.length();
+ String s = b.toString();
+ return (trailing || s.isEmpty()) ?
+ s : (b.substring(0, b.length() - length));
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -including at the end
+ * @param collection strings
+ * @param separator separator string
+ * @return the joined entries
+ */
+ public static String join(String[] collection, String separator) {
+ return join(collection, separator, true);
+
+
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection strings
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(String[] collection, String separator,
+ boolean trailing) {
+ return join(Arrays.asList(collection), separator, trailing);
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -except at the end
+ * @param collection strings
+ * @param separator separator string
+ * @return the list
+ */
+ public static String joinWithInnerSeparator(String separator,
+ Object... collection) {
+ StringBuilder b = new StringBuilder();
+ boolean first = true;
+
+ for (Object o : collection) {
+ if (first) {
+ first = false;
+ } else {
+ b.append(separator);
+ }
+ b.append(o.toString());
+ b.append(separator);
+ }
+ return b.toString();
+ }
+
+ /**
+ * Resolve a mandatory environment variable
+ * @param key env var
+ * @return the resolved value
+ * @throws BadClusterStateException
+ */
+ public static String mandatoryEnvVariable(String key) throws
+ BadClusterStateException {
+ String v = System.getenv(key);
+ if (v == null) {
+ throw new BadClusterStateException("Missing Environment variable " + key);
+ }
+ return v;
+ }
+
+ public static String appReportToString(ApplicationReport r,
+ String separator) {
+ StringBuilder builder = new StringBuilder(512);
+ builder.append("service ")
+ .append(
+ r.getName())
+ .append("/")
+ .append(r.getApplicationType())
+ .append(separator);
+ Set<String> tags = r.getApplicationTags();
+ if (!tags.isEmpty()) {
+ for (String tag : tags) {
+ builder.append(tag).append(separator);
+ }
+ }
+ DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
+ dateFormat.setTimeZone(TimeZone.getDefault());
+ builder.append("state: ").append(r.getYarnApplicationState());
+ String trackingUrl = r.getTrackingUrl();
+ if (isSet(trackingUrl)) {
+ builder.append(separator).append("URL: ").append(trackingUrl);
+ }
+ builder.append(separator)
+ .append("Started: ")
+ .append(dateFormat.format(new Date(r.getStartTime())));
+ long finishTime = r.getFinishTime();
+ if (finishTime > 0) {
+ builder.append(separator)
+ .append("Finished: ")
+ .append(dateFormat.format(new Date(finishTime)));
+ }
+ String rpcHost = r.getHost();
+ if (!isSet(rpcHost)) {
+ builder.append(separator)
+ .append("RPC :")
+ .append(rpcHost)
+ .append(':')
+ .append(r.getRpcPort());
+ }
+ String diagnostics = r.getDiagnostics();
+ if (!isSet(diagnostics)) {
+ builder.append(separator).append("Diagnostics :").append(diagnostics);
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Filter a string value given a single filter
+ *
+ * @param value
+ * the string value to check
+ * @param filter
+ * a single string filter
+ * @return return true if value should be trapped, false if it should be let
+ * through
+ */
+ public static boolean filter(String value, String filter) {
+ return !(StringUtils.isEmpty(filter) || filter.equals(value));
+ }
+
+ /**
+ * Filter a string value given a set of filters
+ *
+ * @param value
+ * the string value to check
+ * @param filters
+ * a set of string filters
+ * @return return true if value should be trapped, false if it should be let
+ * through
+ */
+ public static boolean filter(String value, Set<String> filters) {
+ return !(filters.isEmpty() || filters.contains(value));
+ }
+
+ /**
+ * Sorts the given list of service reports, most recently started
+ * or finished instance first.
+ *
+ * @param instances list of instances
+ */
+ public static void sortApplicationsByMostRecent(List<ApplicationReport> instances) {
+ Collections.sort(instances, new MostRecentlyStartedOrFinishedFirst());
+ }
+
+ /**
+ * Sorts the given list of service reports
+ * Finished instances are ordered by finished time and running/accepted instances are
+ * ordered by start time
+ * Finally Instance are order by finished instances coming after running instances
+ *
+ * @param instances list of instances
+ */
+ public static void sortApplicationReport(List<ApplicationReport> instances) {
+ if (instances.size() <= 1) {
+ return;
+ }
+ List<ApplicationReport> nonLiveInstance =
+ new ArrayList<>(instances.size());
+ List<ApplicationReport> liveInstance =
+ new ArrayList<>(instances.size());
+
+ for (ApplicationReport report : instances) {
+ if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
+ ||
+ report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
+ liveInstance.add(report);
+ } else {
+ nonLiveInstance.add(report);
+ }
+ }
+
+ if (liveInstance.size() > 1) {
+ Collections.sort(liveInstance, new MostRecentlyStartedAppFirst());
+ }
+ if (nonLiveInstance.size() > 1) {
+ Collections.sort(nonLiveInstance, new MostRecentAppFinishFirst());
+ }
+ instances.clear();
+ instances.addAll(liveInstance);
+ instances.addAll(nonLiveInstance);
+ }
+
+ /**
+ * Merge in one map to another -all entries in the second map are
+ * merged into the first -overwriting any duplicate keys.
+ * @param first first map -the updated one.
+ * @param second the map that is merged in
+ * @return the first map
+ */
+ public static Map<String, String> mergeMap(Map<String, String> first,
+ Map<String, String> second) {
+ first.putAll(second);
+ return first;
+ }
+
+ /**
+ * Merge a set of entries into a map. This will take the entryset of
+ * a map, or a Hadoop collection itself
+ * @param dest destination
+ * @param entries entries
+ * @return dest -with the entries merged in
+ */
+ public static Map<String, String> mergeEntries(Map<String, String> dest,
+ Iterable<Map.Entry<String, String>> entries) {
+ for (Map.Entry<String, String> entry : entries) {
+ dest.put(entry.getKey(), entry.getValue());
+ }
+ return dest;
+ }
+
+ /**
+ * Generic map merge logic
+ * @param first first map
+ * @param second second map
+ * @param <T1> key type
+ * @param <T2> value type
+ * @return 'first' merged with the second
+ */
+ public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
+ Map<T1, T2> second) {
+ first.putAll(second);
+ return first;
+ }
+
+ /**
+ * Generic map merge logic
+ * @param first first map
+ * @param second second map
+ * @param <T1> key type
+ * @param <T2> value type
+ * @return 'first' merged with the second
+ */
+ public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
+ Map<T1, T2> second) {
+ Preconditions.checkArgument(first != null, "Null 'first' value");
+ Preconditions.checkArgument(second != null, "Null 'second' value");
+ for (Map.Entry<T1, T2> entry : second.entrySet()) {
+ T1 key = entry.getKey();
+ if (!first.containsKey(key)) {
+ first.put(key, entry.getValue());
+ }
+ }
+ return first;
+ }
+
+ /**
+ * Convert a map to a multi-line string for printing
+ * @param map map to stringify
+ * @return a string representation of the map
+ */
+ public static String stringifyMap(Map<String, String> map) {
+ StringBuilder builder = new StringBuilder();
+ for (Map.Entry<String, String> entry : map.entrySet()) {
+ builder.append(entry.getKey())
+ .append("=\"")
+ .append(entry.getValue())
+ .append("\"\n");
+
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Parse an int value, replacing it with defval if undefined;
+ * @param errorKey key to use in exceptions
+ * @param defVal default value to use if the key is not in the map
+ * @param min min value or -1 for do not check
+ * @param max max value or -1 for do not check
+ * @return the int value the integer value
+ * @throws BadConfigException if the value could not be parsed
+ */
+ public static int parseAndValidate(String errorKey,
+ String valS,
+ int defVal,
+ int min, int max) throws
+ BadConfigException {
+ if (valS == null) {
+ valS = Integer.toString(defVal);
+ }
+ String trim = valS.trim();
+ int val;
+ try {
+ val = Integer.decode(trim);
+ } catch (NumberFormatException e) {
+ throw new BadConfigException("Failed to parse value of "
+ + errorKey + ": \"" + trim + "\"");
+ }
+ if (min >= 0 && val < min) {
+ throw new BadConfigException("Value of "
+ + errorKey + ": " + val + ""
+ + "is less than the minimum of " + min);
+ }
+ if (max >= 0 && val > max) {
+ throw new BadConfigException("Value of "
+ + errorKey + ": " + val + ""
+ + "is more than the maximum of " + max);
+ }
+ return val;
+ }
+
+ public static InetSocketAddress getRmAddress(Configuration conf) {
+ return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
+ }
+
+ public static InetSocketAddress getRmSchedulerAddress(Configuration conf) {
+ return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
+ }
+
+ /**
+ * probe to see if the RM scheduler is defined
+ * @param conf config
+ * @return true if the RM scheduler address is set to
+ * something other than 0.0.0.0
+ */
+ public static boolean isRmSchedulerAddressDefined(Configuration conf) {
+ InetSocketAddress address = getRmSchedulerAddress(conf);
+ return isAddressDefined(address);
+ }
+
+ /**
+ * probe to see if the address
+ * @param address network address
+ * @return true if the scheduler address is set to
+ * something other than 0.0.0.0
+ */
+ public static boolean isAddressDefined(InetSocketAddress address) {
+ if (address == null || address.getHostString() == null) {
+ return false;
+ }
+ return !(address.getHostString().equals("0.0.0.0"));
+ }
+
+ public static void setRmAddress(Configuration conf, String rmAddr) {
+ conf.set(YarnConfiguration.RM_ADDRESS, rmAddr);
+ }
+
+ public static void setRmSchedulerAddress(Configuration conf, String rmAddr) {
+ conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);
+ }
+
+ public static boolean hasAppFinished(ApplicationReport report) {
+ return report == null ||
+ report.getYarnApplicationState().ordinal() >=
+ YarnApplicationState.FINISHED.ordinal();
+ }
+
+ public static String containerToString(Container container) {
+ if (container == null) {
+ return "null container";
+ }
+ return String.format(Locale.ENGLISH,
+ "ContainerID=%s nodeID=%s http=%s priority=%s resource=%s",
+ container.getId(),
+ container.getNodeId(),
+ container.getNodeHttpAddress(),
+ container.getPriority(),
+ container.getResource());
+ }
+
+ /**
+ * convert an AM report to a string for diagnostics
+ * @param report the report
+ * @return the string value
+ */
+ public static String reportToString(ApplicationReport report) {
+ if (report == null) {
+ return "Null service report";
+ }
+
+ return "App " + report.getName() + "/" + report.getApplicationType() +
+ "# " +
+ report.getApplicationId() + " user " + report.getUser() +
+ " is in state " + report.getYarnApplicationState() +
+ " RPC: " + report.getHost() + ":" + report.getRpcPort() +
+ " URL: " + report.getOriginalTrackingUrl();
+ }
+
+ /**
+ * Convert a YARN URL into a string value of a normal URL
+ * @param url URL
+ * @return string representatin
+ */
+ public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(url.getScheme()).append("://");
+ if (url.getHost() != null) {
+ builder.append(url.getHost()).append(":").append(url.getPort());
+ }
+ builder.append(url.getFile());
+ return builder.toString();
+ }
+
+ /**
+ * Get a random open port
+ * @return true if the port was available for listening on
+ */
+ public static int getOpenPort() throws IOException {
+ ServerSocket socket = null;
+ try {
+ socket = new ServerSocket(0);
+ return socket.getLocalPort();
+ } finally {
+ if (socket != null) {
+ socket.close();
+ }
+ }
+ }
+
+ /**
+ * See if a port is available for listening on by trying to listen
+ * on it and seeing if that works or fails.
+ * @param port port to listen to
+ * @return true if the port was available for listening on
+ */
+ public static boolean isPortAvailable(int port) {
+ try {
+ ServerSocket socket = new ServerSocket(port);
+ socket.close();
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ // Build env map: key -> value;
+ // value will be replaced by the corresponding value in tokenMap, if any.
+ public static Map<String, String> buildEnvMap(
+ org.apache.hadoop.yarn.service.api.records.Configuration conf,
+ Map<String,String> tokenMap) {
+ if (tokenMap == null) {
+ return conf.getEnv();
+ }
+ Map<String, String> env = new HashMap<>();
+ for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) {
+ String key = entry.getKey();
+ String val = entry.getValue();
+ for (Map.Entry<String,String> token : tokenMap.entrySet()) {
+ val = val.replaceAll(Pattern.quote(token.getKey()),
+ token.getValue());
+ }
+ env.put(key,val);
+ }
+ return env;
+ }
+
+ /**
+ * Apply a set of command line options to a cluster role map
+ * @param clusterRoleMap cluster role map to merge onto
+ * @param commandOptions command opts
+ */
+ public static void applyCommandLineRoleOptsToRoleMap(
+ Map<String, Map<String, String>> clusterRoleMap,
+ Map<String, Map<String, String>> commandOptions) {
+ for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
+ String key = entry.getKey();
+ Map<String, String> optionMap = entry.getValue();
+ Map<String, String> existingMap = clusterRoleMap.get(key);
+ if (existingMap == null) {
+ existingMap = new HashMap<String, String>();
+ }
+ log.debug("Overwriting role options with command line values {}",
+ stringifyMap(optionMap));
+ mergeMap(existingMap, optionMap);
+ //set or overwrite the role
+ clusterRoleMap.put(key, existingMap);
+ }
+ }
+
+ /**
+ * Verify that a Kerberos principal has been set -if not fail
+ * with an error message that actually tells you what is missing
+ * @param conf configuration to look at
+ * @param principal key of principal
+ * @throws BadConfigException if the key is not set
+ */
+ public static void verifyPrincipalSet(Configuration conf,
+ String principal) throws
+ BadConfigException {
+ String principalName = conf.get(principal);
+ if (principalName == null) {
+ throw new BadConfigException("Unset Kerberos principal : %s",
+ principal);
+ }
+ log.debug("Kerberos princial {}={}", principal, principalName);
+ }
+
+ /**
+ * Flag to indicate whether the cluster is in secure mode
+ * @param conf configuration to look at
+ * @return true if the slider client/service should be in secure mode
+ */
+ public static boolean isHadoopClusterSecure(Configuration conf) {
+ return SecurityUtil.getAuthenticationMethod(conf) !=
+ UserGroupInformation.AuthenticationMethod.SIMPLE;
+ }
+
+ /**
+ * Init security if the cluster configuration declares the cluster is secure
+ * @param conf configuration to look at
+ * @return true if the cluster is secure
+ * @throws IOException cluster is secure
+ * @throws SliderException the configuration/process is invalid
+ */
+ public static boolean maybeInitSecurity(Configuration conf) throws
+ IOException,
+ SliderException {
+ boolean clusterSecure = isHadoopClusterSecure(conf);
+ if (clusterSecure) {
+ log.debug("Enabling security");
+ initProcessSecurity(conf);
+ }
+ return clusterSecure;
+ }
+
+ /**
+ * Turn on security. This is setup to only run once.
+ * @param conf configuration to build up security
+ * @return true if security was initialized in this call
+ * @throws IOException IO/Net problems
+ * @throws BadConfigException the configuration and system state are inconsistent
+ */
+ public static boolean initProcessSecurity(Configuration conf) throws
+ IOException,
+ SliderException {
+
+ if (processSecurityAlreadyInitialized.compareAndSet(true, true)) {
+ //security is already inited
+ return false;
+ }
+
+ log.info("JVM initialized into secure mode with kerberos realm {}",
+ SliderUtils.getKerberosRealm());
+ //this gets UGI to reset its previous world view (i.e simple auth)
+ //security
+ log.debug("java.security.krb5.realm={}",
+ System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
+ log.debug("java.security.krb5.kdc={}",
+ System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
+ log.debug("hadoop.security.authentication={}",
+ conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION));
+ log.debug("hadoop.security.authorization={}",
+ conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION));
+ UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation authUser = UserGroupInformation.getCurrentUser();
+ log.debug("Authenticating as {}", authUser);
+ log.debug("Login user is {}", UserGroupInformation.getLoginUser());
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ throw new SliderException(LauncherExitCodes.EXIT_UNAUTHORIZED,
+ "Although secure mode is enabled," +
+ "the service has already set up its user as an insecure entity %s",
+ authUser);
+ }
+ if (authUser.getAuthenticationMethod() ==
+ UserGroupInformation.AuthenticationMethod.SIMPLE) {
+ throw new BadConfigException("Auth User is not Kerberized %s" +
+ " -security has already been set up with the wrong authentication method. "
+ + "This can occur if a file system has already been created prior to the loading of "
+ + "the security configuration.",
+ authUser);
+
+ }
+
+ SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
+ SliderUtils.verifyPrincipalSet(conf, "dfs.namenode.kerberos.principal");
+ return true;
+ }
+
+ /**
+ * Force an early login: This catches any auth problems early rather than
+ * in RPC operations
+ * @throws IOException if the login fails
+ */
+ public static void forceLogin() throws IOException {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ if (UserGroupInformation.isLoginKeytabBased()) {
+ UserGroupInformation.getLoginUser().reloginFromKeytab();
+ } else {
+ UserGroupInformation.getLoginUser().reloginFromTicketCache();
+ }
+ }
+ }
+
+ public static String getLibDir() {
+ String[] libDirs = getLibDirs();
+ if (libDirs == null || libDirs.length == 0) {
+ return null;
+ }
+ return libDirs[0];
+ }
+
+ public static String[] getLibDirs() {
+ String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
+ if (isUnset(libDirStr)) {
+ return ArrayUtils.EMPTY_STRING_ARRAY;
+ }
+ return StringUtils.split(libDirStr, ',');
+ }
+
+ /**
+ * Submit a JAR containing a specific class and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param clazz class to look for
+ * @param libdir lib directory
+ * @param jarName <i>At the destination</i>
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public static LocalResource putJar(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Class clazz,
+ Path tempPath,
+ String libdir,
+ String jarName
+ )
+ throws IOException, SliderException {
+ LocalResource res = sliderFileSystem.submitJarWithClass(
+ clazz,
+ tempPath,
+ libdir,
+ jarName);
+ providerResources.put(libdir + "/" + jarName, res);
+ return res;
+ }
+
+ /**
+ * Submit a JAR containing and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param libDir lib directory
+ * @param srcPath copy jars from
+ */
+ public static void putAllJars(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libDir,
+ String srcPath) throws IOException, SliderException {
+ log.info("Loading all dependencies from {}", srcPath);
+ if (SliderUtils.isSet(srcPath)) {
+ File srcFolder = new File(srcPath);
+ FilenameFilter jarFilter = createJarFilter();
+ File[] listOfJars = srcFolder.listFiles(jarFilter);
+ if (listOfJars == null || listOfJars.length == 0) {
+ return;
+ }
+ for (File jarFile : listOfJars) {
+ LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
+ providerResources.put(libDir + "/" + jarFile.getName(), res);
+ }
+ }
+ }
+
+ /**
+ * Accept all filenames ending with {@code .jar}
+ * @return a filename filter
+ */
+ public static FilenameFilter createJarFilter() {
+ return new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
+ }
+ };
+ }
+
+ /**
+ * Submit the AM tar.gz containing all dependencies and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ */
+ public static void putAmTarGzipAndUpdate(
+ Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem
+ ) throws IOException, SliderException {
+ log.info("Loading all dependencies from {}{}",
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME,
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+ sliderFileSystem.submitTarGzipAndUpdate(providerResources);
+ }
+
+ public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
+ Map<String, Map<String, String>> dest = new HashMap<>();
+ for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) {
+ dest.put(entry.getKey(), stringMapClone(entry.getValue()));
+ }
+ return dest;
+ }
+
+ public static Map<String, String> stringMapClone(Map<String, String> src) {
+ Map<String, String> dest = new HashMap<>();
+ return mergeEntries(dest, src.entrySet());
+ }
+
+ /**
+ * List a directory in the local filesystem
+ * @param dir directory
+ * @return a listing, one to a line
+ */
+ public static String listDir(File dir) {
+ if (dir == null) {
+ return "";
+ }
+ String[] confDirEntries = dir.list();
+ if (confDirEntries == null) {
+ return "";
+ }
+ StringBuilder builder = new StringBuilder();
+ for (String entry : confDirEntries) {
+ builder.append(entry).append("\n");
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Create a file:// path from a local file
+ * @param file file to point the path
+ * @return a new Path
+ */
+ public static Path createLocalPath(File file) {
+ return new Path(file.toURI());
+ }
+
+ public static String getKerberosRealm() {
+ try {
+ return KerberosUtil.getDefaultRealm();
+ } catch (Exception e) {
+ log.debug("introspection into JVM internals failed", e);
+ return "(unknown)";
+
+ }
+ }
+
+ /**
+ * Build up the classpath for execution
+ * -behaves very differently on a mini test cluster vs a production
+ * production one.
+ *
+ * @param sliderConfDir relative path to the dir containing slider config
+ * options to put on the classpath -or null
+ * @param libdir directory containing the JAR files
+ * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
+ * (and hence the current classpath should be used, not anything built up)
+ * @return a classpath
+ */
+ public static ClasspathConstructor buildClasspath(String sliderConfDir,
+ String libdir,
+ SliderFileSystem sliderFileSystem,
+ boolean usingMiniMRCluster) {
+
+ ClasspathConstructor classpath = new ClasspathConstructor();
+ classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
+
+ // add the runtime classpath needed for tests to work
+ if (usingMiniMRCluster) {
+ // for mini cluster we pass down the java CP properties
+ // and nothing else
+ classpath.appendAll(classpath.localJVMClasspath());
+ } else {
+ if (sliderConfDir != null) {
+ classpath.addClassDirectory(sliderConfDir);
+ }
+ classpath.addLibDir(libdir);
+ if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
+ classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
+ } else {
+ log.info(
+ "For faster submission of apps, upload dependencies using cmd {} {}",
+ SliderActions.ACTION_DEPENDENCY, Arguments.ARG_UPLOAD);
+ }
+ classpath.addRemoteClasspathEnvVar();
+ classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
+ }
+ return classpath;
+ }
+
+ /**
+ * Verify that a path refers to a directory. If not
+ * logs the parent dir then throws an exception
+ * @param dir the directory
+ * @param errorlog log for output on an error
+ * @throws FileNotFoundException if it is not a directory
+ */
+ public static void verifyIsDir(File dir, Logger errorlog) throws
+ FileNotFoundException {
+ if (!dir.exists()) {
+ errorlog.warn("contents of {}: {}", dir,
+ listDir(dir.getParentFile()));
+ throw new FileNotFoundException(dir.toString());
+ }
+ if (!dir.isDirectory()) {
+ errorlog.info("contents of {}: {}", dir,
+ listDir(dir.getParentFile()));
+ throw new FileNotFoundException(
+ "Not a directory: " + dir);
+ }
+ }
+
+ /**
+ * Verify that a file exists
+ * @param file file
+ * @param errorlog log for output on an error
+ * @throws FileNotFoundException
+ */
+ public static void verifyFileExists(File file, Logger errorlog) throws
+ FileNotFoundException {
+ if (!file.exists()) {
+ errorlog.warn("contents of {}: {}", file,
+ listDir(file.getParentFile()));
+ throw new FileNotFoundException(file.toString());
+ }
+ if (!file.isFile()) {
+ throw new FileNotFoundException("Not a file: " + file.toString());
+ }
+ }
+
+ /**
+ * verify that a config option is set
+ * @param configuration config
+ * @param key key
+ * @return the value, in case it needs to be verified too
+ * @throws BadConfigException if the key is missing
+ */
+ public static String verifyOptionSet(Configuration configuration, String key,
+ boolean allowEmpty) throws BadConfigException {
+ String val = configuration.get(key);
+ if (val == null) {
+ throw new BadConfigException(
+ "Required configuration option \"%s\" not defined ", key);
+ }
+ if (!allowEmpty && val.isEmpty()) {
+ throw new BadConfigException(
+ "Configuration option \"%s\" must not be empty", key);
+ }
+ return val;
+ }
+
+ /**
+ * Verify that a keytab property is defined and refers to a non-empty file
+ *
+ * @param siteConf configuration
+ * @param prop property to look for
+ * @return the file referenced
+ * @throws BadConfigException on a failure
+ */
+ public static File verifyKeytabExists(Configuration siteConf,
+ String prop) throws
+ BadConfigException {
+ String keytab = siteConf.get(prop);
+ if (keytab == null) {
+ throw new BadConfigException("Missing keytab property %s",
+ prop);
+
+ }
+ File keytabFile = new File(keytab);
+ if (!keytabFile.exists()) {
+ throw new BadConfigException("Missing keytab file %s defined in %s",
+ keytabFile,
+ prop);
+ }
+ if (keytabFile.length() == 0 || !keytabFile.isFile()) {
+ throw new BadConfigException("Invalid keytab file %s defined in %s",
+ keytabFile,
+ prop);
+ }
+ return keytabFile;
+ }
+
+ /**
+ * Add a subpath to an existing URL. This extends
+ * the path, inserting a / between all entries
+ * if needed.
+ * @param base base path/URL
+ * @param path subpath
+ * @return base+"/"+subpath
+ */
+ public static String appendToURL(String base, String path) {
+ StringBuilder fullpath = new StringBuilder(base);
+ if (!base.endsWith("/")) {
+ fullpath.append("/");
+ }
+ if (path.startsWith("/")) {
+ fullpath.append(path.substring(1));
+ } else {
+ fullpath.append(path);
+ }
+ return fullpath.toString();
+ }
+
+ /**
+ * Truncate the given string to a maximum length provided
+ * with a pad (...) added to the end if expected size if more than 10.
+ * @param toTruncate string to truncate; may be null
+ * @param maxSize maximum size
+ * @return the truncated/padded string.
+ */
+ public static String truncate(String toTruncate, int maxSize) {
+ if (toTruncate == null || maxSize < 1
+ || toTruncate.length() <= maxSize) {
+ return toTruncate;
+ }
+
+ String pad = "...";
+ if (maxSize < 10) {
+ pad = "";
+ }
+ return toTruncate.substring(0, maxSize - pad.length()).concat(pad);
+ }
+
+ /**
+ * Given a source folder create zipped file
+ *
+ * @param srcFolder
+ * @param zipFile
+ *
+ * @throws IOException
+ */
+ public static void zipFolder(File srcFolder, File zipFile) throws IOException {
+ log.info("Zipping folder {} to {}", srcFolder.getAbsolutePath(), zipFile.getAbsolutePath());
+ List<String> files = new ArrayList<>();
+ generateFileList(files, srcFolder, srcFolder, true);
+
+ byte[] buffer = new byte[1024];
+
+ try (FileOutputStream fos = new FileOutputStream(zipFile)) {
+ try (ZipOutputStream zos = new ZipOutputStream(fos)) {
+
+ for (String file : files) {
+ ZipEntry ze = new ZipEntry(file);
+ zos.putNextEntry(ze);
+ try (FileInputStream in = new FileInputStream(srcFolder + File.separator + file)) {
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ zos.write(buffer, 0, len);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Given a source folder create a tar.gz file
+ *
+ * @param libDirs
+ * @param tarGzipFile
+ *
+ * @throws IOException
+ */
+ public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
+ FilenameFilter filter) throws IOException {
+ log.info("Tar-gzipping folders {} to {}", libDirs,
+ tarGzipFile.getAbsolutePath());
+
+ try(TarArchiveOutputStream taos =
+ new TarArchiveOutputStream(new GZIPOutputStream(
+ new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
+ for (String libDir : libDirs) {
+ File srcFolder = new File(libDir);
+ List<String> files = new ArrayList<>();
+ generateFileList(files, srcFolder, srcFolder, true, filter);
+ for (String file : files) {
+ File srcFile = new File(srcFolder, file);
+ TarArchiveEntry tarEntry = new TarArchiveEntry(
+ srcFile, file);
+ taos.putArchiveEntry(tarEntry);
+ try(FileInputStream in = new FileInputStream(srcFile)) {
+ org.apache.commons.io.IOUtils.copy(in, taos);
+ }
+ taos.flush();
+ taos.closeArchiveEntry();
+ }
+ }
+ }
+ }
+
+ private static void generateFileList(List<String> fileList, File node,
+ File rootFolder, Boolean relative) {
+ generateFileList(fileList, node, rootFolder, relative, null);
+ }
+
+ private static void generateFileList(List<String> fileList, File node,
+ File rootFolder, Boolean relative, FilenameFilter filter) {
+ if (node.isFile()) {
+ String fileFullPath = node.toString();
+ if (relative) {
+ fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1,
+ fileFullPath.length()));
+ } else {
+ fileList.add(fileFullPath);
+ }
+ }
+
+ if (node.isDirectory()) {
+ String[] subNode = node.list(filter);
+ if (subNode == null || subNode.length == 0) {
+ return;
+ }
+ for (String filename : subNode) {
+ generateFileList(fileList, new File(node, filename), rootFolder,
+ relative, filter);
+ }
+ }
+ }
+
+ /**
+ * Check for any needed libraries being present. On Unix none are needed;
+ * on windows they must be present
+ * @return true if all is well
+ */
+ public static String checkForRequiredNativeLibraries() {
+
+ if (!Shell.WINDOWS) {
+ return "";
+ }
+ StringBuilder errorText = new StringBuilder("");
+ if (!NativeIO.isAvailable()) {
+ errorText.append("No native IO library. ");
+ }
+ try {
+ String path = Shell.getQualifiedBinPath(WINUTILS);
+ log.debug("winutils is at {}", path);
+ } catch (IOException e) {
+ errorText.append("No " + WINUTILS);
+ log.warn("No winutils: {}", e, e);
+ }
+ try {
+ File target = new File("target");
+ FileUtil.canRead(target);
+ } catch (UnsatisfiedLinkError e) {
+ log.warn("Failing to link to native IO methods: {}", e, e);
+ errorText.append("No native IO methods");
+ }
+ return errorText.toString();
+ }
+
+ /**
+ * Strictly verify that windows utils is present.
+ * Checks go as far as opening the file and looking for
+ * the headers.
+ * @throws IOException on any problem reading the file
+ * @throws FileNotFoundException if the file is not considered valid
+ */
+ public static void maybeVerifyWinUtilsValid() throws
+ IOException,
+ SliderException {
+ String errorText = SliderUtils.checkForRequiredNativeLibraries();
+ if (!errorText.isEmpty()) {
+ throw new BadClusterStateException(errorText);
+ }
+ }
+
+ /**
+ * Write bytes to a file
+ * @param outfile output file
+ * @param data data to write
+ * @throws IOException on any IO problem
+ */
+ public static void write(File outfile, byte[] data)
+ throws IOException {
+ File parentDir = outfile.getCanonicalFile().getParentFile();
+ if (parentDir == null) {
+ throw new IOException(outfile.getPath() + " has no parent dir");
+ }
+ if (!parentDir.exists()) {
+ if(!parentDir.mkdirs()) {
+ throw new IOException("Failed to create parent directory " + parentDir);
+ }
+ }
+ SliderUtils.verifyIsDir(parentDir, log);
+ try(FileOutputStream out = new FileOutputStream(outfile)) {
+ out.write(data);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recent app comes first
+ * Specifically: the one whose start time value is greater.
+ */
+ private static class MostRecentlyStartedAppFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long x = r1.getStartTime();
+ long y = r2.getStartTime();
+ return compareTwoLongsReverse(x, y);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recent app comes first.
+ * "Recent"== the app whose start time <i>or finish time</i> is the greatest.
+ */
+ private static class MostRecentlyStartedOrFinishedFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long started1 = r1.getStartTime();
+ long started2 = r2.getStartTime();
+ long finished1 = r1.getFinishTime();
+ long finished2 = r2.getFinishTime();
+ long lastEvent1 = Math.max(started1, finished1);
+ long lastEvent2 = Math.max(started2, finished2);
+ return compareTwoLongsReverse(lastEvent1, lastEvent2);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recently finished app comes first
+ * Specifically: the one whose finish time value is greater.
+ */
+ private static class MostRecentAppFinishFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long x = r1.getFinishTime();
+ long y = r2.getFinishTime();
+ return compareTwoLongsReverse(x, y);
+ }
+ }
+
+ /**
+ * Compare two long values for sorting. As the return value for
+ * comparators must be int, the simple value of <code>x-y</code>
+ * is inapplicable
+ * @param x x value
+ * @param y y value
+ * @return +ve if x is less than y, -ve if y is greater than x; 0 for equality
+ */
+ public static int compareTwoLongsReverse(long x, long y) {
+ return (x < y) ? 1 : ((x == y) ? 0 : -1);
+ }
+
+ public static String createNameTag(String name) {
+ return "Name: " + name;
+ }
+
+ public static String createVersionTag(String version) {
+ return "Version: " + version;
+ }
+
+ public static String createDescriptionTag(String description) {
+ return "Description: " + description;
+ }
+
+ public static final String DAYS = ".days";
+ public static final String HOURS = ".hours";
+ public static final String MINUTES = ".minutes";
+ public static final String SECONDS = ".seconds";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
new file mode 100644
index 0000000..1fa07ce
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.net.HostAndPort;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ZookeeperUtils {
+ public static final int DEFAULT_PORT = 2181;
+
+ public static String buildConnectionString(String zkHosts, int port) {
+ String zkPort = Integer.toString(port);
+ //parse the hosts
+ String[] hostlist = zkHosts.split(",", 0);
+ String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
+ return quorum;
+ }
+
+ /**
+ * Take a quorum list and split it to (trimmed) pairs
+ * @param hostPortQuorumList list of form h1:port, h2:port2,...
+ * @return a possibly empty list of values between commas. They may not be
+ * valid hostname:port pairs
+ */
+ public static List<String> splitToPairs(String hostPortQuorumList) {
+ // split an address hot
+ String[] strings = StringUtils.getStrings(hostPortQuorumList);
+ int len = 0;
+ if (strings != null) {
+ len = strings.length;
+ }
+ List<String> tuples = new ArrayList<String>(len);
+ if (strings != null) {
+ for (String s : strings) {
+ tuples.add(s.trim());
+ }
+ }
+ return tuples;
+ }
+
+ /**
+ * Split a quorum list into a list of hostnames and ports
+ * @param hostPortQuorumList split to a list of hosts and ports
+ * @return a list of values
+ */
+ public static List<HostAndPort> splitToHostsAndPorts(String hostPortQuorumList) {
+ // split an address hot
+ String[] strings = StringUtils.getStrings(hostPortQuorumList);
+ int len = 0;
+ if (strings != null) {
+ len = strings.length;
+ }
+ List<HostAndPort> list = new ArrayList<HostAndPort>(len);
+ if (strings != null) {
+ for (String s : strings) {
+ list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
+ }
+ }
+ return list;
+ }
+
+ /**
+ * Build up to a hosts only list
+ * @param hostAndPorts
+ * @return a list of the hosts only
+ */
+ public static String buildHostsOnlyList(List<HostAndPort> hostAndPorts) {
+ StringBuilder sb = new StringBuilder();
+ for (HostAndPort hostAndPort : hostAndPorts) {
+ sb.append(hostAndPort.getHostText()).append(",");
+ }
+ if (sb.length() > 0) {
+ sb.delete(sb.length() - 1, sb.length());
+ }
+ return sb.toString();
+ }
+
+ public static String buildQuorumEntry(HostAndPort hostAndPort,
+ int defaultPort) {
+ String s = hostAndPort.toString();
+ if (hostAndPort.hasPort()) {
+ return s;
+ } else {
+ return s + ":" + defaultPort;
+ }
+ }
+
+ /**
+ * Build a quorum list, injecting a ":defaultPort" ref if needed on
+ * any entry without one
+ * @param hostAndPorts
+ * @param defaultPort
+ * @return
+ */
+ public static String buildQuorum(List<HostAndPort> hostAndPorts, int defaultPort) {
+ List<String> entries = new ArrayList<String>(hostAndPorts.size());
+ for (HostAndPort hostAndPort : hostAndPorts) {
+ entries.add(buildQuorumEntry(hostAndPort, defaultPort));
+ }
+ return SliderUtils.join(entries, ",", false);
+ }
+
+ public static String convertToHostsOnlyList(String quorum) throws
+ BadConfigException {
+ List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
+ return ZookeeperUtils.buildHostsOnlyList(hostAndPorts);
+ }
+
+ public static List<HostAndPort> splitToHostsAndPortsStrictly(String quorum) throws
+ BadConfigException {
+ List<HostAndPort> hostAndPorts =
+ ZookeeperUtils.splitToHostsAndPorts(quorum);
+ if (hostAndPorts.isEmpty()) {
+ throw new BadConfigException("empty zookeeper quorum");
+ }
+ return hostAndPorts;
+ }
+
+ public static int getFirstPort(String quorum, int defVal) throws
+ BadConfigException {
+ List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
+ int port = hostAndPorts.get(0).getPortOrDefault(defVal);
+ return port;
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto
new file mode 100644
index 0000000..0a21c24
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.yarn.proto";
+option java_outer_classname = "ClientAMProtocol";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.yarn;
+
+service ClientAMProtocolService {
+ rpc flexComponents(FlexComponentsRequestProto) returns (FlexComponentsResponseProto);
+ rpc getStatus(GetStatusRequestProto) returns (GetStatusResponseProto);
+ rpc stop(StopRequestProto) returns (StopResponseProto);
+}
+
+message FlexComponentsRequestProto {
+ repeated ComponentCountProto components = 1;
+}
+
+message ComponentCountProto {
+ optional string name = 1;
+ optional int64 numberOfContainers = 2;
+}
+
+message FlexComponentsResponseProto{
+}
+
+message GetStatusRequestProto {
+
+}
+message GetStatusResponseProto {
+ optional string status = 1;
+}
+
+message StopRequestProto {
+
+}
+
+message StopResponseProto {
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
new file mode 100644
index 0000000..d343a03
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import com.google.common.base.Supplier;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.NMClient;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.service.component.ComponentState;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
+import static org.mockito.Mockito.mock;
+
+public class MockServiceAM extends ServiceMaster {
+
+ Service service;
+ // The list of containers fed by tests to be returned on
+ // AMRMClientCallBackHandler#onContainersAllocated
+ final List<Container> feedContainers =
+ Collections.synchronizedList(new LinkedList<>());
+
+ public MockServiceAM(Service service) {
+ super(service.getName());
+ this.service = service;
+ }
+
+
+ @Override
+ protected ContainerId getAMContainerId()
+ throws BadClusterStateException {
+ return ContainerId.newContainerId(ApplicationAttemptId
+ .newInstance(ApplicationId.fromString(service.getId()), 1), 1);
+ }
+
+ @Override
+ protected Path getAppDir() {
+ Path path = new Path(new Path("target", "apps"), service.getName());
+ System.out.println("Service path: " + path);
+ return path;
+ }
+
+ @Override
+ protected ServiceScheduler createServiceScheduler(ServiceContext context)
+ throws IOException, YarnException {
+ return new ServiceScheduler(context) {
+
+ @Override
+ protected YarnRegistryViewForProviders createYarnRegistryOperations(
+ ServiceContext context, RegistryOperations registryClient) {
+ return mock(YarnRegistryViewForProviders.class);
+ }
+
+ @Override
+ protected AMRMClientAsync<AMRMClient.ContainerRequest> createAMRMClient() {
+ AMRMClientImpl client1 = new AMRMClientImpl() {
+ @Override public AllocateResponse allocate(float progressIndicator)
+ throws YarnException, IOException {
+
+ AllocateResponse.AllocateResponseBuilder builder =
+ AllocateResponse.newBuilder();
+ synchronized (feedContainers) {
+ if (feedContainers.isEmpty()) {
+ System.out.println("Allocating........ no containers");
+ return builder.build();
+ } else {
+ // The AMRMClient will return containers for compoenent that are
+ // at FLEXING state
+ List<Container> allocatedContainers = new LinkedList<>();
+ Iterator<Container> itor = feedContainers.iterator();
+ while (itor.hasNext()) {
+ Container c = itor.next();
+ org.apache.hadoop.yarn.service.component.Component component =
+ componentsById.get(c.getAllocationRequestId());
+ if (component.getState() == ComponentState.FLEXING) {
+ System.out.println("Allocated container " + c.getId());
+ allocatedContainers.add(c);
+ itor.remove();
+ }
+ }
+ return builder.allocatedContainers(allocatedContainers).build();
+ }
+ }
+ }
+
+ @Override
+ public RegisterApplicationMasterResponse registerApplicationMaster(
+ String appHostName, int appHostPort, String appTrackingUrl) {
+ return mock(RegisterApplicationMasterResponse.class);
+ }
+
+ @Override public void unregisterApplicationMaster(
+ FinalApplicationStatus appStatus, String appMessage,
+ String appTrackingUrl) {
+ // DO nothing
+ }
+ };
+
+ return AMRMClientAsync
+ .createAMRMClientAsync(client1, 1000,
+ this.new AMRMClientCallback());
+ }
+
+ @Override
+ public NMClientAsync createNMClient() {
+ NMClientAsync nmClientAsync = super.createNMClient();
+ nmClientAsync.setClient(mock(NMClient.class));
+ return nmClientAsync;
+ }
+ };
+ }
+
+ @Override protected void loadApplicationJson(ServiceContext context,
+ SliderFileSystem fs) throws IOException {
+ context.service = service;
+ }
+
+ /**
+ *
+ * @param service The service for the component
+ * @param id The id for the container
+ * @param compName The component to which the container is fed
+ * @return
+ */
+ public Container feedContainerToComp(Service service, int id,
+ String compName) {
+ ApplicationId applicationId = ApplicationId.fromString(service.getId());
+ ContainerId containerId = ContainerId
+ .newContainerId(ApplicationAttemptId.newInstance(applicationId, 1), id);
+ NodeId nodeId = NodeId.newInstance("localhost", 1234);
+ Container container = Container
+ .newInstance(containerId, nodeId, "localhost",
+ Resource.newInstance(100, 1), Priority.newInstance(0), null);
+
+ long allocateId =
+ context.scheduler.getAllComponents().get(compName).getAllocateId();
+ container.setAllocationRequestId(allocateId);
+ synchronized (feedContainers) {
+ feedContainers.add(container);
+ }
+ return container;
+ }
+
+ public void flexComponent(String compName, long numberOfContainers)
+ throws IOException {
+ ClientAMProtocol.ComponentCountProto componentCountProto =
+ ClientAMProtocol.ComponentCountProto.newBuilder().setName(compName)
+ .setNumberOfContainers(numberOfContainers).build();
+ ClientAMProtocol.FlexComponentsRequestProto requestProto =
+ ClientAMProtocol.FlexComponentsRequestProto.newBuilder()
+ .addComponents(componentCountProto).build();
+ context.clientAMService.flexComponents(requestProto);
+ }
+
+ public Component getComponent(String compName) {
+ return context.scheduler.getAllComponents().get(compName);
+ }
+
+ public void waitForDependenciesSatisfied(String compName)
+ throws TimeoutException, InterruptedException {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override public Boolean get() {
+ return context.scheduler.getAllComponents().get(compName)
+ .areDependenciesReady();
+ }
+ }, 1000, 20000);
+ }
+
+ public void waitForNumDesiredContainers(String compName,
+ int numDesiredContainers) throws TimeoutException, InterruptedException {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override public Boolean get() {
+ return context.scheduler.getAllComponents().get(compName)
+ .getNumDesiredInstances() == numDesiredContainers;
+ }
+ }, 1000, 20000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
new file mode 100644
index 0000000..0f4f598
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
+
+public class ServiceTestUtils {
+
+ public static final JsonSerDeser<Service> JSON_SER_DESER =
+ new JsonSerDeser<>(Service.class,
+ PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
+
+ // Example service definition
+ // 2 components, each of which has 2 containers.
+ protected Service createExampleApplication() {
+ Service exampleApp = new Service();
+ exampleApp.setName("example-app");
+ exampleApp.addComponent(createComponent("compa"));
+ exampleApp.addComponent(createComponent("compb"));
+ return exampleApp;
+ }
+
+ protected Component createComponent(String name) {
+ return createComponent(name, 2L, "sleep 1000");
+ }
+
+ protected Component createComponent(String name, long numContainers,
+ String command) {
+ Component comp1 = new Component();
+ comp1.setNumberOfContainers(numContainers);
+ comp1.setLaunchCommand(command);
+ comp1.setName(name);
+ Resource resource = new Resource();
+ comp1.setResource(resource);
+ resource.setMemory("128");
+ resource.setCpus(1);
+ return comp1;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[72/86] [abbrv] hadoop git commit: YARN-7100. Added JsonIgnore
annotation to Resource#getMemoryMB. Contributed by Eric Yang
Posted by ji...@apache.org.
YARN-7100. Added JsonIgnore annotation to Resource#getMemoryMB. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5fad4ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5fad4ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5fad4ee
Branch: refs/heads/yarn-native-services
Commit: d5fad4eee31704bd0fbb8ff5afec0f77f8da1aa5
Parents: 7bfd881
Author: Jian He <ji...@apache.org>
Authored: Tue Aug 29 20:37:38 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 25 16:37:23 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/service/api/records/Resource.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5fad4ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
index cec9de9..dfdf92a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
@@ -22,6 +22,7 @@ import io.swagger.annotations.ApiModelProperty;
import java.util.Objects;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -103,6 +104,7 @@ public class Resource extends BaseResource implements Cloneable {
this.memory = memory;
}
+ @JsonIgnore
public long getMemoryMB() {
if (this.memory == null) {
return 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
deleted file mode 100644
index f1bf2ad..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client;
-
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.types.NodeInformationList;
-import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
-import org.apache.slider.common.params.ActionAMSuicideArgs;
-import org.apache.slider.common.params.ActionClientArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
-import org.apache.slider.common.params.ActionDiagnosticArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
-import org.apache.slider.common.params.ActionFreezeArgs;
-import org.apache.slider.common.params.ActionKeytabArgs;
-import org.apache.slider.common.params.ActionNodesArgs;
-import org.apache.slider.common.params.ActionKillContainerArgs;
-import org.apache.slider.common.params.ActionListArgs;
-import org.apache.slider.common.params.ActionRegistryArgs;
-import org.apache.slider.common.params.ActionResolveArgs;
-import org.apache.slider.common.params.ActionResourceArgs;
-import org.apache.slider.common.params.ActionStatusArgs;
-import org.apache.slider.common.params.ActionThawArgs;
-import org.apache.slider.common.params.ActionUpgradeArgs;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.SliderException;
-
-import java.io.IOException;
-
-/**
- * Interface of those method calls in the slider API that are intended
- * for direct public invocation.
- * <p>
- * Stability: evolving
- */
-public interface SliderClientAPI extends Service {
-
- int actionDestroy(String clustername) throws YarnException, IOException;
-
- /**
- * AM to commit an asynchronous suicide
- */
- int actionAmSuicide(String clustername,
- ActionAMSuicideArgs args) throws YarnException, IOException;
-
- /**
- * Manage keytabs leveraged by slider
- *
- * @param keytabInfo the arguments needed to manage the keytab
- * @throws YarnException Yarn problems
- * @throws IOException other problems
- * @throws BadCommandArgumentsException bad arguments.
- */
- int actionKeytab(ActionKeytabArgs keytabInfo)
- throws YarnException, IOException;
-
- /**
- * Manage file resources leveraged by slider
- *
- * @param resourceInfo the arguments needed to manage the resource
- * @throws YarnException Yarn problems
- * @throws IOException other problems
- * @throws BadCommandArgumentsException bad arguments.
- */
- int actionResource(ActionResourceArgs resourceInfo)
- throws YarnException, IOException;
-
- /**
- * Perform client operations such as install or configure
- *
- * @param clientInfo the arguments needed for client operations
- *
- * @throws SliderException bad arguments.
- * @throws IOException problems related to package and destination folders
- */
- int actionClient(ActionClientArgs clientInfo)
- throws IOException, YarnException;
-
- /**
- * Update the cluster specification
- *
- * @param clustername cluster name
- * @param buildInfo the arguments needed to update the cluster
- * @throws YarnException Yarn problems
- * @throws IOException other problems
- */
- int actionUpdate(String clustername,
- AbstractClusterBuildingActionArgs buildInfo)
- throws YarnException, IOException;
-
- /**
- * Upgrade the cluster with a newer version of the application
- *
- * @param clustername cluster name
- * @param buildInfo the arguments needed to upgrade the cluster
- * @throws YarnException Yarn problems
- * @throws IOException other problems
- */
- int actionUpgrade(String clustername,
- ActionUpgradeArgs buildInfo)
- throws YarnException, IOException;
-
- /**
- * Implement the list action: list all nodes
- * @return exit code of 0 if a list was created
- */
- int actionList(String clustername, ActionListArgs args) throws IOException, YarnException;
-
-
- int actionFlex(String name, ActionFlexArgs args) throws YarnException,
- IOException;
-
- /**
- * Test for a cluster existing probe for a cluster of the given name existing
- * in the filesystem. If the live param is set, it must be a live cluster
- * @return exit code
- */
- int actionExists(String name, boolean checkLive) throws YarnException, IOException;
-
- /**
- * Kill a specific container of the cluster
- * @param name cluster name
- * @param args arguments
- * @return exit code
- * @throws YarnException
- * @throws IOException
- */
- int actionKillContainer(String name, ActionKillContainerArgs args)
- throws YarnException, IOException;
-
- /**
- * Status operation
- *
- * @param clustername cluster name
- * @param statusArgs status arguments
- * @return 0 -for success, else an exception is thrown
- * @throws YarnException
- * @throws IOException
- */
- int actionStatus(String clustername, ActionStatusArgs statusArgs)
- throws YarnException, IOException;
-
- /**
- * Status operation which returns the status object as a string instead of
- * printing it to the console or file.
- *
- * @param clustername cluster name
- * @return cluster status details
- * @throws YarnException
- * @throws IOException
- */
- Application actionStatus(String clustername) throws YarnException, IOException;
-
- /**
- * Version Details
- * @return exit code
- */
- int actionVersion();
-
- /**
- * Stop the cluster
- *
- * @param clustername cluster name
- * @param freezeArgs arguments to the stop
- * @return EXIT_SUCCESS if the cluster was not running by the end of the operation
- */
- int actionStop(String clustername, ActionFreezeArgs freezeArgs)
- throws YarnException, IOException;
-
- /**
- * Restore a cluster
- */
- int actionStart(String clustername, ActionThawArgs thaw) throws YarnException, IOException;
-
- /**
- * Registry operation
- *
- * @param args registry Arguments
- * @return 0 for success, -1 for some issues that aren't errors, just failures
- * to retrieve information (e.g. no configurations for that entry)
- * @throws YarnException YARN problems
- * @throws IOException Network or other problems
- */
- int actionResolve(ActionResolveArgs args)
- throws YarnException, IOException;
-
- /**
- * Registry operation
- *
- * @param registryArgs registry Arguments
- * @return 0 for success, -1 for some issues that aren't errors, just failures
- * to retrieve information (e.g. no configurations for that entry)
- * @throws YarnException YARN problems
- * @throws IOException Network or other problems
- */
- int actionRegistry(ActionRegistryArgs registryArgs)
- throws YarnException, IOException;
-
- /**
- * diagnostic operation
- *
- * @param diagnosticArgs diagnostic Arguments
- * @return 0 for success, -1 for some issues that aren't errors, just
- * failures to retrieve information (e.g. no application name
- * specified)
- * @throws YarnException YARN problems
- * @throws IOException Network or other problems
- */
- int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs);
-
- /**
- * Get the registry binding. As this may start the registry, it can take time
- * and fail
- * @return the registry
- */
- RegistryOperations getRegistryOperations()
- throws SliderException, IOException;
-
- /**
- * Upload all Slider AM and agent dependency libraries to HDFS, so that they
- * do not need to be uploaded with every create call. This operation is
- * Slider version specific. So it needs to be invoked for every single
- * version of slider/slider-client.
- *
- * @throws SliderException
- * @throws IOException
- */
- int actionDependency(ActionDependencyArgs dependencyArgs) throws IOException,
- YarnException;
-
- /**
- * List the nodes
- * @param args
- * @return
- * @throws YarnException
- * @throws IOException
- */
- NodeInformationList listYarnClusterNodes(ActionNodesArgs args)
- throws YarnException, IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
deleted file mode 100644
index e1b578c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.api.types.NodeInformationList;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.slider.common.tools.Duration;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-//TODO, Remove this class and YarnAppListClient
-// why do we need so many yarn client wrappers ?
-// - yarn client already provides most of functionality already
-
-/**
- * A class that extends visibility to some of the YarnClientImpl
- * members and data structures, and factors out pure-YARN operations
- * from the slider entry point service
- */
-public class SliderYarnClientImpl extends YarnClientImpl {
- protected static final Logger log = LoggerFactory.getLogger(SliderYarnClientImpl.class);
-
- /**
- * Get the RM Client RPC interface
- * @return an RPC interface valid after initialization and authentication
- */
- public ApplicationClientProtocol getRmClient() {
- return rmClient;
- }
-
- /**
- * List Slider <i>deployed</i>instances belonging to a specific user in a
- * given set of states and filtered by an application name tag.
- * <p>
- * Deployed means: known about in the YARN cluster; it will include all apps
- * in the specified set of states and tagged with the specified app name.
- *
- * @param user
- * user: "" means all users
- * @param appStates
- * filter by a set of YarnApplicationState
- * @param appname
- * an application name tag in the format defined by
- * {@link SliderUtils#createNameTag(String)}
- * @return a possibly empty list of Slider AMs
- * @throws YarnException
- * @throws IOException
- */
- public List<ApplicationReport> listDeployedInstances(String user,
- EnumSet<YarnApplicationState> appStates, String appname)
- throws YarnException, IOException {
- Preconditions.checkArgument(user != null, "Null User");
- Set<String> types = new HashSet<>(1);
- types.add(SliderKeys.APP_TYPE);
- Set<String> tags = null;
- if (appname != null) {
- tags = Collections.singleton(SliderUtils.createNameTag(appname));
- }
- List<ApplicationReport> allApps = getApplications(types, appStates, tags);
- List<ApplicationReport> results = new ArrayList<>();
- for (ApplicationReport report : allApps) {
- if (StringUtils.isEmpty(user) || user.equals(report.getUser())) {
- results.add(report);
- }
- }
- return results;
- }
-
- /**
- * Helper method to determine if a cluster application is running -or
- * is earlier in the lifecycle
- * @param app application report
- * @return true if the application is considered live
- */
- public boolean isApplicationLive(ApplicationReport app) {
- Preconditions.checkArgument(app != null, "Null app report");
-
- return app.getYarnApplicationState().ordinal() <= YarnApplicationState.RUNNING.ordinal();
- }
-
- /**
- * find all live instances of a specific app -if there is >1 in the cluster,
- * this returns them all. State should be running or less
- * @param user user
- * @param appname application name
- * @return the list of all matching application instances
- */
- public List<ApplicationReport> findAllLiveInstances(String user,
- String appname) throws YarnException, IOException {
- Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
- "Null/empty application name");
- return listDeployedInstances(user, SliderUtils.getAllLiveAppStates(),
- appname);
- }
-
- /**
- * Find a cluster in the instance list; biased towards live instances
- * @param instances list of instances
- * @param appname application name
- * @return the first found instance, else a failed/finished instance, or null
- * if there are none of those
- */
- public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances,
- String appname) {
- Preconditions.checkArgument(instances != null, "Null instances list");
- Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
- "Null/empty application name");
- // sort by most recent
- SliderUtils.sortApplicationsByMostRecent(instances);
- ApplicationReport found = null;
- for (ApplicationReport app : instances) {
- if (app.getName().equals(appname)) {
- if (isApplicationLive(app)) {
- return app;
- }
- // set the found value if not set
- found = found != null ? found : app;
- }
- }
- return found;
- }
-
- /**
- * Find an app in the instance list in the desired state
- * @param instances instance list
- * @param appname application name
- * @param desiredState yarn state desired
- * @return the match or null for none
- */
- public ApplicationReport findAppInInstanceList(List<ApplicationReport> instances,
- String appname,
- YarnApplicationState desiredState) {
- Preconditions.checkArgument(instances != null, "Null instances list");
- Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
- "Null/empty application name");
- Preconditions.checkArgument(desiredState != null, "Null desiredState");
- log.debug("Searching {} records for instance name {} in state '{}'",
- instances.size(), appname, desiredState);
- for (ApplicationReport app : instances) {
- if (app.getName().equals(appname)) {
-
- YarnApplicationState appstate =
- app.getYarnApplicationState();
- log.debug("app ID {} is in state {}", app.getApplicationId(), appstate);
- if (appstate.equals(desiredState)) {
- log.debug("match");
- return app;
- }
- }
- }
- // nothing found in desired state
- log.debug("No match");
- return null;
- }
-
- /**
- * List the nodes in the cluster, possibly filtering by node state or label.
- *
- * @param label label to filter by -or "" for any
- * @param live flag to request running nodes only
- * @return a possibly empty list of nodes in the cluster
- * @throws IOException IO problems
- * @throws YarnException YARN problems
- */
- public NodeInformationList listNodes(String label, boolean live)
- throws IOException, YarnException {
- Preconditions.checkArgument(label != null, "null label");
- NodeState[] states;
- if (live) {
- states = new NodeState[1];
- states[0] = NodeState.RUNNING;
- } else {
- states = new NodeState[0];
- }
- List<NodeReport> reports = getNodeReports(states);
- NodeInformationList results = new NodeInformationList(reports.size());
- for (NodeReport report : reports) {
- if (live && report.getNodeState() != NodeState.RUNNING) {
- continue;
- }
- if (!label.isEmpty() && !report.getNodeLabels().contains(label)) {
- continue;
- }
- // build node info from report
- NodeInformation info = new NodeInformation();
- info.hostname = report.getNodeId().getHost();
- info.healthReport = report.getHealthReport();
- info.httpAddress = report.getHttpAddress();
- info.labels = SliderUtils.extractNodeLabel(report);
- info.rackName = report.getRackName();
- info.state = report.getNodeState().toString();
- results.add(info);
- }
- return results;
- }
-
- /**
- * Monitor the submitted application for reaching the requested state.
- * Will also report if the app reaches a later state (failed, killed, etc)
- * Kill application if duration!= null & time expires.
- * @param appId Application Id of application to be monitored
- * @param duration how long to wait -must be more than 0
- * @param desiredState desired state.
- * @return the application report -null on a timeout
- * @throws YarnException
- * @throws IOException
- */
- public ApplicationReport monitorAppToState(
- ApplicationId appId, YarnApplicationState desiredState, Duration duration)
- throws YarnException, IOException {
-
- if (appId == null) {
- throw new BadCommandArgumentsException("null application ID");
- }
- if (duration.limit <= 0) {
- throw new BadCommandArgumentsException("Invalid monitoring duration");
- }
- log.debug("Waiting {} millis for app to reach state {} ",
- duration.limit,
- desiredState);
- duration.start();
- try {
- while (true) {
- // Get application report for the appId we are interested in
-
- ApplicationReport r = getApplicationReport(appId);
-
- log.debug("queried status is\n{}",
- new SliderUtils.OnDemandReportStringifier(r));
-
- YarnApplicationState state = r.getYarnApplicationState();
- if (state.ordinal() >= desiredState.ordinal()) {
- log.debug("App in desired state (or higher) :{}", state);
- return r;
- }
- if (duration.getLimitExceeded()) {
- log.debug(
- "Wait limit of {} millis to get to state {}, exceeded; app " +
- "status\n {}",
- duration.limit,
- desiredState,
- new SliderUtils.OnDemandReportStringifier(r));
- return null;
- }
-
- // sleep 1s.
- try {
- Thread.sleep(1000);
- } catch (InterruptedException ignored) {
- log.debug("Thread sleep in monitoring loop interrupted");
- }
- }
- } finally {
- duration.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
deleted file mode 100644
index 84c65b3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.slider.common.params.ActionTokensArgs;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.NotFoundException;
-import static org.apache.slider.core.launch.CredentialUtils.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-
-public class TokensOperation {
-
- private static final Logger log = LoggerFactory.getLogger(TokensOperation.class);
- public static final String E_INSECURE
- = "Cluster is not secure -tokens cannot be acquired";
- public static final String E_MISSING_SOURCE_FILE = "Missing source file: ";
- public static final String E_NO_KEYTAB = "No keytab: ";
-
- public int actionTokens(ActionTokensArgs args, FileSystem fs,
- Configuration conf,
- YarnClientImpl yarnClient)
- throws IOException, YarnException {
- Credentials credentials;
- String footnote = "";
- UserGroupInformation user = UserGroupInformation.getCurrentUser();
- boolean isSecure = UserGroupInformation.isSecurityEnabled();
- if (args.keytab != null) {
- File keytab = args.keytab;
- if (!keytab.isFile()) {
- throw new NotFoundException(E_NO_KEYTAB + keytab.getAbsolutePath());
- }
- String principal = args.principal;
- log.info("Logging in as {} from keytab {}", principal, keytab);
- user = UserGroupInformation.loginUserFromKeytabAndReturnUGI(
- principal, keytab.getCanonicalPath());
- }
- Credentials userCredentials = user.getCredentials();
- File output = args.output;
- if (output != null) {
- if (!isSecure) {
- throw new BadClusterStateException(E_INSECURE);
- }
- credentials = new Credentials(userCredentials);
- // filesystem
- addRMRenewableFSDelegationTokens(conf, fs, credentials);
- addRMDelegationToken(yarnClient, credentials);
- if (maybeAddTimelineToken(conf, credentials) != null) {
- log.debug("Added timeline token");
- }
- saveTokens(output, credentials);
- String filename = output.getCanonicalPath();
- footnote = String.format(
- "%d tokens saved to %s%n" + "To use these in the environment:%n"
- + "export %s=%s", credentials.numberOfTokens(), filename,
- UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, filename);
- } else if (args.source != null) {
- File source = args.source;
- log.info("Reading credentials from file {}", source);
- if (!source.isFile()) {
- throw new NotFoundException( E_MISSING_SOURCE_FILE + source.getAbsolutePath());
- }
- credentials = Credentials.readTokenStorageFile(args.source, conf);
- } else {
- StringBuffer origin = new StringBuffer();
- File file = locateEnvCredentials(System.getenv(), conf,
- origin);
- if (file != null) {
- log.info("Credential Source {}", origin);
- } else {
- log.info("Credential source: logged in user");
- }
- credentials = userCredentials;
- }
- // list the tokens
- log.info("\n{}", dumpTokens(credentials, "\n"));
- if (!footnote.isEmpty()) {
- log.info(footnote);
- }
- return 0;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
deleted file mode 100644
index e89a660..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
+++ /dev/null
@@ -1,355 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client.ipc;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.SliderClusterProtocol;
-import org.apache.slider.api.StateValues;
-import org.apache.slider.api.proto.Messages;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.api.types.NodeInformationList;
-import org.apache.slider.api.types.PingInformation;
-import org.apache.slider.common.tools.Duration;
-import org.apache.slider.core.exceptions.NoSuchNodeException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.exceptions.WaitTimeoutException;
-import org.apache.slider.core.persist.JsonSerDeser;
-import org.codehaus.jackson.JsonParseException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import static org.apache.slider.api.types.RestTypeMarshalling.unmarshall;
-
-/**
- * Cluster operations at a slightly higher level than the RPC code
- */
-public class SliderClusterOperations {
- protected static final Logger
- log = LoggerFactory.getLogger(SliderClusterOperations.class);
-
- private final SliderClusterProtocol appMaster;
- private static final JsonSerDeser<Application> jsonSerDeser =
- new JsonSerDeser<Application>(Application.class);
- private static final Messages.EmptyPayloadProto EMPTY;
- static {
- EMPTY = Messages.EmptyPayloadProto.newBuilder().build();
- }
-
- public SliderClusterOperations(SliderClusterProtocol appMaster) {
- this.appMaster = appMaster;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("SliderClusterOperations{");
- sb.append("IPC binding=").append(appMaster);
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * Get a node from the AM
- * @param uuid uuid of node
- * @return deserialized node
- * @throws IOException IO problems
- * @throws NoSuchNodeException if the node isn't found
- */
- public ClusterNode getNode(String uuid)
- throws IOException, NoSuchNodeException, YarnException {
- Messages.GetNodeRequestProto req =
- Messages.GetNodeRequestProto.newBuilder().setUuid(uuid).build();
- Messages.GetNodeResponseProto node = appMaster.getNode(req);
- return ClusterNode.fromProtobuf(node.getClusterNode());
- }
-
- /**
- * Unmarshall a list of nodes from a protobud response
- * @param nodes node list
- * @return possibly empty list of cluster nodes
- * @throws IOException
- */
- public List<ClusterNode> convertNodeWireToClusterNodes(List<Messages.RoleInstanceState> nodes)
- throws IOException {
- List<ClusterNode> nodeList = new ArrayList<>(nodes.size());
- for (Messages.RoleInstanceState node : nodes) {
- nodeList.add(ClusterNode.fromProtobuf(node));
- }
- return nodeList;
- }
-
- /**
- * Echo text (debug action)
- * @param text text
- * @return the text, echoed back
- * @throws YarnException
- * @throws IOException
- */
- public String echo(String text) throws YarnException, IOException {
- Messages.EchoRequestProto.Builder builder =
- Messages.EchoRequestProto.newBuilder();
- builder.setText(text);
- Messages.EchoRequestProto req = builder.build();
- Messages.EchoResponseProto response = appMaster.echo(req);
- return response.getText();
- }
-
-
- /**
- * Connect to a live cluster and get its current state
- * @return its description
- */
- public Application getApplication() throws YarnException, IOException {
- Messages.GetJSONClusterStatusRequestProto req =
- Messages.GetJSONClusterStatusRequestProto.newBuilder().build();
- Messages.GetJSONClusterStatusResponseProto resp =
- appMaster.getJSONClusterStatus(req);
- String statusJson = resp.getClusterSpec();
- try {
- return jsonSerDeser.fromJson(statusJson);
- } catch (JsonParseException e) {
- log.error("Error when parsing app json file", e);
- throw e;
- }
- }
-
- /**
- * Kill a container
- * @param id container ID
- * @return a success flag
- * @throws YarnException
- * @throws IOException
- */
- public boolean killContainer(String id) throws
- YarnException,
- IOException {
- Messages.KillContainerRequestProto.Builder builder =
- Messages.KillContainerRequestProto.newBuilder();
- builder.setId(id);
- Messages.KillContainerRequestProto req = builder.build();
- Messages.KillContainerResponseProto response = appMaster.killContainer(req);
- return response.getSuccess();
- }
-
- /**
- * List all node UUIDs in a role
- * @param role role name or "" for all
- * @return an array of UUID strings
- * @throws IOException
- * @throws YarnException
- */
- public String[] listNodeUUIDsByRole(String role) throws IOException, YarnException {
- Collection<String> uuidList = innerListNodeUUIDSByRole(role);
- String[] uuids = new String[uuidList.size()];
- return uuidList.toArray(uuids);
- }
-
- public List<String> innerListNodeUUIDSByRole(String role) throws IOException, YarnException {
- Messages.ListNodeUUIDsByRoleRequestProto req =
- Messages.ListNodeUUIDsByRoleRequestProto
- .newBuilder()
- .setRole(role)
- .build();
- Messages.ListNodeUUIDsByRoleResponseProto resp = appMaster.listNodeUUIDsByRole(req);
- return resp.getUuidList();
- }
-
- /**
- * List all nodes in a role. This is a double round trip: once to list
- * the nodes in a role, another to get their details
- * @param role
- * @return an array of ContainerNode instances
- * @throws IOException
- * @throws YarnException
- */
- public List<ClusterNode> listClusterNodesInRole(String role)
- throws IOException, YarnException {
-
- Collection<String> uuidList = innerListNodeUUIDSByRole(role);
- Messages.GetClusterNodesRequestProto req =
- Messages.GetClusterNodesRequestProto
- .newBuilder()
- .addAllUuid(uuidList)
- .build();
- Messages.GetClusterNodesResponseProto resp = appMaster.getClusterNodes(req);
- return convertNodeWireToClusterNodes(resp.getClusterNodeList());
- }
-
- /**
- * Get the details on a list of uuids
- * @param uuids instance IDs
- * @return a possibly empty list of node details
- * @throws IOException
- * @throws YarnException
- */
- @VisibleForTesting
- public List<ClusterNode> listClusterNodes(String[] uuids)
- throws IOException, YarnException {
-
- Messages.GetClusterNodesRequestProto req =
- Messages.GetClusterNodesRequestProto
- .newBuilder()
- .addAllUuid(Arrays.asList(uuids))
- .build();
- Messages.GetClusterNodesResponseProto resp = appMaster.getClusterNodes(req);
- return convertNodeWireToClusterNodes(resp.getClusterNodeList());
- }
-
- /**
- * Wait for an instance of a named role to be live (or past it in the lifecycle)
- * @param role role to look for
- * @param timeout time to wait
- * @return the state. If still in CREATED, the cluster didn't come up
- * in the time period. If LIVE, all is well. If >LIVE, it has shut for a reason
- * @throws IOException IO
- * @throws SliderException Slider
- * @throws WaitTimeoutException if the wait timed out
- */
- @VisibleForTesting
- public int waitForRoleInstanceLive(String role, long timeout)
- throws WaitTimeoutException, IOException, YarnException {
- Duration duration = new Duration(timeout);
- duration.start();
- boolean live = false;
- int state = StateValues.STATE_CREATED;
-
- log.info("Waiting {} millis for a live node in role {}", timeout, role);
- try {
- while (!live) {
- // see if there is a node in that role yet
- List<String> uuids = innerListNodeUUIDSByRole(role);
- String[] containers = uuids.toArray(new String[uuids.size()]);
- int roleCount = containers.length;
- ClusterNode roleInstance = null;
- if (roleCount != 0) {
-
- // if there is, get the node
- roleInstance = getNode(containers[0]);
- if (roleInstance != null) {
- state = roleInstance.state;
- live = state >= StateValues.STATE_LIVE;
- }
- }
- if (!live) {
- if (duration.getLimitExceeded()) {
- throw new WaitTimeoutException(
- String.format("Timeout after %d millis" +
- " waiting for a live instance of type %s; " +
- "instances found %d %s",
- timeout, role, roleCount,
- (roleInstance != null
- ? (" instance -\n" + roleInstance.toString())
- : "")
- ));
- } else {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException ignored) {
- // ignored
- }
- }
- }
- }
- } finally {
- duration.close();
- }
- return state;
- }
-
- public void flex(Map<String, Long> componentCounts) throws IOException{
- Messages.FlexComponentsRequestProto.Builder builder =
- Messages.FlexComponentsRequestProto.newBuilder();
- for (Entry<String, Long> componentCount : componentCounts.entrySet()) {
- Messages.ComponentCountProto componentProto =
- Messages.ComponentCountProto.newBuilder()
- .setName(componentCount.getKey())
- .setNumberOfContainers(componentCount.getValue()).build();
- builder.addComponents(componentProto);
- }
- appMaster.flexComponents(builder.build());
- }
-
- /**
- * Commit (possibly delayed) AM suicide
- *
- * @param signal exit code
- * @param text text text to log
- * @param delay delay in millis
- * @throws YarnException
- * @throws IOException
- */
- public void amSuicide(String text, int signal, int delay)
- throws IOException {
- Messages.AMSuicideRequestProto.Builder builder =
- Messages.AMSuicideRequestProto.newBuilder();
- if (text != null) {
- builder.setText(text);
- }
- builder.setSignal(signal);
- builder.setDelay(delay);
- Messages.AMSuicideRequestProto req = builder.build();
- appMaster.amSuicide(req);
- }
-
- public List<ContainerInformation> getContainers() throws IOException {
- Messages.GetLiveContainersResponseProto response = appMaster
- .getLiveContainers(Messages.GetLiveContainersRequestProto.newBuilder()
- .build());
- return unmarshall(response);
- }
-
- public NodeInformationList getLiveNodes() throws IOException {
- Messages.GetLiveNodesResponseProto response =
- appMaster.getLiveNodes(Messages.GetLiveNodesRequestProto.newBuilder().build());
-
- int records = response.getNodesCount();
- NodeInformationList nil = new NodeInformationList(records);
- for (int i = 0; i < records; i++) {
- nil.add(unmarshall(response.getNodes(i)));
- }
- return nil;
- }
-
- public NodeInformation getLiveNode(String hostname) throws IOException {
- Messages.GetLiveNodeRequestProto.Builder builder =
- Messages.GetLiveNodeRequestProto.newBuilder();
- builder.setName(hostname);
- return unmarshall(appMaster.getLiveNode(builder.build()));
- }
-
- public PingInformation ping(String text) throws IOException {
- return null;
- }
-
- public void stop(String text) throws IOException {
- amSuicide(text, 3, 0);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/BaseRestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/BaseRestClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/BaseRestClient.java
deleted file mode 100644
index d936a22..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/BaseRestClient.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client.rest;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.GenericType;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.slider.core.exceptions.ExceptionConverter;
-import org.apache.slider.core.restclient.HttpVerb;
-import org.apache.slider.core.restclient.UgiJerseyBinding;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.core.MediaType;
-import java.io.IOException;
-import java.net.URI;
-
-
-/**
- * This is a base class for Jersey REST clients in Slider.
- * It supports the execution of operations —with
- * exceptions uprated to IOExceptions when needed.
- * <p>
- * Subclasses can use these operations to provide an API-like view
- * of the REST model
- */
-public class BaseRestClient {
- private static final Logger log =
- LoggerFactory.getLogger(BaseRestClient.class);
- private final Client client;
-
- public BaseRestClient(
- Client client) {
- Preconditions.checkNotNull(client, "null jersey client");
- this.client = client;
- }
-
- /**
- * Get the jersey client
- * @return jersey client
- */
- public Client getClient() {
- return client;
- }
-
- /**
- * Execute the operation. Failures are raised as IOException subclasses
- * @param method method to execute
- * @param resource resource to work against
- * @param c class to build
- * @param <T> type expected
- * @return an instance of the type T
- * @throws IOException on any failure
- */
- public <T> T exec(HttpVerb method, WebResource resource, Class<T> c)
- throws IOException {
- try {
- Preconditions.checkArgument(c != null);
- log.debug("{}} {}", method, resource.getURI());
- return resource.accept(MediaType.APPLICATION_JSON_TYPE)
- .method(method.getVerb(), c);
- } catch (ClientHandlerException ex) {
- throw ExceptionConverter.convertJerseyException(method.getVerb(),
- resource.getURI().toString(),
- ex);
- } catch (UniformInterfaceException ex) {
- throw UgiJerseyBinding.uprateFaults(method,
- resource.getURI().toString(),
- ex);
- }
- }
-
- /**
- * Execute the operation. Failures are raised as IOException subclasses
- * @param method method to execute
- * @param resource resource to work against
- * @param t type to work with
- * @param <T> type expected
- * @return an instance of the type T
- * @throws IOException on any failure
- */
- public <T> T exec(HttpVerb method, WebResource resource, GenericType<T> t)
- throws IOException {
- try {
- Preconditions.checkArgument(t != null);
- log.debug("{}} {}", method, resource.getURI());
- resource.accept(MediaType.APPLICATION_JSON_TYPE);
- return resource.method(method.getVerb(), t);
- } catch (ClientHandlerException ex) {
- throw ExceptionConverter.convertJerseyException(method.getVerb(),
- resource.getURI().toString(),
- ex);
- } catch (UniformInterfaceException ex) {
- throw UgiJerseyBinding.uprateFaults(method, resource.getURI().toString(),
- ex);
- }
- }
-
-
- /**
- * Execute the GET operation. Failures are raised as IOException subclasses
- * @param resource resource to work against
- * @param c class to build
- * @param <T> type expected
- * @return an instance of the type T
- * @throws IOException on any failure
- */
- public <T> T get(WebResource resource, Class<T> c) throws IOException {
- return exec(HttpVerb.GET, resource, c);
- }
-
- /**
- * Create a Web resource from the client.
- *
- * @param u the URI of the resource.
- * @return the Web resource.
- */
- public WebResource resource(URI u) {
- return client.resource(u);
- }
-
- /**
- * Create a Web resource from the client.
- *
- * @param u the URI of the resource.
- * @return the Web resource.
- */
-
- public WebResource resource(String url) {
- return client.resource(url);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/Constants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/Constants.java
deleted file mode 100644
index 0e3559a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/Constants.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common;
-
-public class Constants {
- public static final int CONNECT_TIMEOUT = 10000;
- public static final int RPC_TIMEOUT = 15000;
-
- public static final String HADOOP_JAAS_DEBUG = "HADOOP_JAAS_DEBUG";
- public static final String KRB5_CCNAME = "KRB5CCNAME";
- public static final String JAVA_SECURITY_KRB5_CONF
- = "java.security.krb5.conf";
- public static final String JAVA_SECURITY_KRB5_REALM
- = "java.security.krb5.realm";
- public static final String SUN_SECURITY_KRB5_DEBUG
- = "sun.security.krb5.debug";
- public static final String SUN_SECURITY_SPNEGO_DEBUG
- = "sun.security.spnego.debug";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
deleted file mode 100644
index 61c828e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common;
-
-/**
- * Keys shared across tests
- */
-public interface SliderXMLConfKeysForTesting {
-
- String KEY_TEST_THAW_WAIT_TIME = "slider.test.thaw.wait.seconds";
-
- int DEFAULT_THAW_WAIT_TIME_SECONDS = 60;
-
-
- String KEY_TEST_FREEZE_WAIT_TIME = "slider.test.freeze.wait.seconds";
-
- int DEFAULT_TEST_FREEZE_WAIT_TIME_SECONDS = 60;
-
- String KEY_TEST_TIMEOUT = "slider.test.timeout.seconds";
-
- int DEFAULT_TEST_TIMEOUT_SECONDS = 30 * 60;
-
- String KEY_ACCUMULO_LAUNCH_TIME =
- "slider.test.accumulo.launch.wait.seconds";
- int DEFAULT_ACCUMULO_LAUNCH_TIME_SECONDS = 60 * 3;
-
- String KEY_ACCUMULO_GO_LIVE_TIME =
- "slider.test.accumulo.live.wait.seconds";
- int DEFAULT_ACCUMULO_LIVE_TIME_SECONDS = 90;
-
- String KEY_TEST_AGENT_ENABLED = "slider.test.agent.enabled";
- String KEY_AGENTTESTS_QUEUE_LABELED_DEFINED = "slider.test.agent.labeled.queue.enabled";
- String KEY_AGENTTESTS_LABELS_RED_BLUE_DEFINED = "slider.test.agent.labels.defined";
- String KEY_AGENTTESTS_AM_FAILURES_ENABLED = "slider.test.agent.am.failures.enabled";
-
- int DEFAULT_AGENT_LAUNCH_TIME_SECONDS = 60 * 3;
-
- String KEY_TEST_AGENT_HOME = "slider.test.agent.home";
- String KEY_TEST_AGENT_TAR = "slider.test.agent.tar";
-
- String KEY_TEST_TEARDOWN_KILLALL = "slider.test.teardown.killall";
- boolean DEFAULT_TEARDOWN_KILLALL = true;
-
-
- /**
- * Key for amount of RAM to request
- */
- String KEY_TEST_YARN_RAM_REQUEST = "slider.test.yarn.ram";
- String DEFAULT_YARN_RAM_REQUEST = "192";
-
- /**
- * security related keys
- */
- String TEST_SECURITY_DIR = "/tmp/work/security";
-
- /**
- * Local path to AM keytab: {@value}
- */
- String KEY_TEST_AM_KEYTAB = "slider.test.am.keytab.local";
-
- /**
- * Is the test cluster windows? Default is: same as the local system.
- * {@value}
- */
- String KEY_TEST_WINDOWS_CLUSTER = "slider.test.windows.cluster";
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractArgsDelegate.java
deleted file mode 100644
index ec88ca1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractArgsDelegate.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import org.apache.hadoop.yarn.service.client.params.ArgOps;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-
-/**
- * Base class for all the delegates
- */
-public class AbstractArgsDelegate extends ArgOps implements Arguments {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
deleted file mode 100644
index 57c27e7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParametersDelegate;
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.ComponentArgsDelegate;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-
-import java.io.File;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Abstract Action to build things; shares args across build and
- * list
- */
-public abstract class AbstractClusterBuildingActionArgs
- extends AbstractActionArgs {
- @Parameter(names = {ARG_APPDEF},
- description = "Template application definition file in JSON format.")
- public File appDef;
-
- public File getAppDef() {
- return appDef;
- }
-
- @Parameter(names = {
- ARG_QUEUE }, description = "Queue to submit the application")
- public String queue;
-
- @Parameter(names = {
- ARG_LIFETIME }, description = "Lifetime of the application from the time of request")
- public long lifetime;
-
- @ParametersDelegate
- public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
-
- @ParametersDelegate
- public OptionArgsDelegate optionsDelegate =
- new OptionArgsDelegate();
-
-
- public Map<String, String> getOptionsMap() throws
- BadCommandArgumentsException {
- return optionsDelegate.getOptionsMap();
- }
-
- /**
- * Get the role heap mapping (may be empty, but never null).
- * @return role heap mapping
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, Map<String, String>> getCompOptionMap() throws
- BadCommandArgumentsException {
- return optionsDelegate.getCompOptionMap();
- }
-
- @VisibleForTesting
- public List<String> getComponentTuples() {
- return componentDelegate.getComponentTuples();
- }
-
- /**
- * Get the role mapping (may be empty, but never null).
- * @return role mapping
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, String> getComponentMap() throws
- BadCommandArgumentsException {
- return componentDelegate.getComponentMap();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java
deleted file mode 100644
index 04ec9e2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_AM_SUICIDE},
- commandDescription = SliderActions.DESCRIBE_ACTION_AM_SUICIDE)
-public class ActionAMSuicideArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_AM_SUICIDE;
- }
-
- @Parameter(names = {ARG_MESSAGE},
- description = "reason for the action")
- public String message = "";
-
- @Parameter(names = {ARG_EXITCODE},
- description = "exit code")
- public int exitcode = 1;
-
- @Parameter(names = {ARG_WAIT},
- description = "time for AM to wait before exiting")
- public int waittime = 1000;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java
deleted file mode 100644
index 8dfde36..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_CLIENT},
- commandDescription = SliderActions.DESCRIBE_ACTION_CLIENT)
-
-public class ActionClientArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_CLIENT;
- }
-
- @Parameter(names = {ARG_INSTALL},
- description = "Install client")
- public boolean install;
-
- @Parameter(names = {ARG_NAME},
- description = "The name of the application")
- public String name;
-
- @Parameter(names = {ARG_PACKAGE},
- description = "Path to app package")
- public String packageURI;
-
- @Parameter(names = {ARG_DEST},
- description = "The location where to install the client")
- public File installLocation;
-
- @Parameter(names = {ARG_CONFIG},
- description = "Client configuration")
- public File clientConfig;
-
- /**
- * Get the min #of params expected
- *
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java
deleted file mode 100644
index cb36961..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(
- commandNames = { SliderActions.ACTION_DIAGNOSTICS},
- commandDescription = SliderActions.DESCRIBE_ACTION_DIAGNOSTIC)
-public class ActionDiagnosticArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_DIAGNOSTICS;
- }
-
- @Parameter(names = {ARG_NAME},
- description = "the name of the running application")
- public String name;
-
- @Parameter(names = {ARG_CLIENT},
- description = "print configuration of the slider client")
- public boolean client = false;
-
- @Parameter(names = {ARG_APPLICATION},
- description = "print configuration of the running application")
- public boolean application;
-
- @Parameter(names = {ARG_VERBOSE},
- description = "print out information in details")
- public boolean verbose = false;
-
- @Parameter(names = {ARG_YARN},
- description = "print configuration of the YARN cluster")
- public boolean yarn = false;
-
- @Parameter(names = {ARG_CREDENTIALS},
- description = "print credentials of the current user")
- public boolean credentials = false;
-
- @Parameter(names = {ARG_ALL},
- description = "print all of the information above")
- public boolean all;
-
- @Parameter(names = {ARG_LEVEL},
- description = "diagnose each slider configuration one by one")
- public boolean level;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
deleted file mode 100644
index b075ce0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_EXISTS},
- commandDescription = SliderActions.DESCRIBE_ACTION_EXISTS)
-
-public class ActionExistsArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_EXISTS;
- }
-
- @Parameter(names = {ARG_LIVE},
- description = "verify that the application is running")
- public boolean live;
-
- @Parameter(names = {ARG_STATE},
- description = "verify that the application is in the specific YARN state")
- public String state = "";
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for any application report")
- public File out;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java
deleted file mode 100644
index b08e1cf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_STOP },
- commandDescription = SliderActions.DESCRIBE_ACTION_FREEZE)
-
-public class ActionFreezeArgs extends AbstractActionArgs implements
- WaitTimeAccessor {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_STOP;
- }
-
- public static final String FREEZE_COMMAND_ISSUED = "stop command issued";
- @ParametersDelegate
- public WaitArgsDelegate waitDelegate = new WaitArgsDelegate();
-
- @Override
- public int getWaittime() {
- return waitDelegate.getWaittime();
- }
-
- @Override
- public void setWaittime(int waittime) {
- waitDelegate.setWaittime(waittime);
- }
-
- @Parameter(names={ARG_MESSAGE},
- description = "reason for the operation")
- public String message = FREEZE_COMMAND_ISSUED;
-
- @Parameter(names = {ARG_FORCE},
- description = "force the operation")
- public boolean force;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java
deleted file mode 100644
index fc6eb4f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-/**
- * The Help command
- */
-@Parameters(commandNames = { SliderActions.ACTION_HELP},
- commandDescription = SliderActions.DESCRIBE_ACTION_HELP)
-public class ActionHelpArgs extends AbstractActionArgs {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_HELP;
- }
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-
- /**
- * This action does not need hadoop services
- * @return false
- */
- @Override
- public boolean getHadoopServicesRequired() {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKDiagArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKDiagArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKDiagArgs.java
deleted file mode 100644
index be370bb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKDiagArgs.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.UsageException;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-
-@Parameters(commandNames = { SliderActions.ACTION_KDIAG},
- commandDescription = SliderActions.DESCRIBE_ACTION_KDIAG)
-
-public class ActionKDiagArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_KDIAG;
- }
-
- @Parameter(names = {ARG_SERVICES}, variableArity = true,
- description =" list of services to check")
- public List<String> services = new ArrayList<>();
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for report")
- public File out;
-
- @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
- public File keytab;
-
- @Parameter(names = {ARG_KEYLEN}, description = "minimum key length")
- public int keylen = 256;
-
- @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
- public String principal;
-
- @Parameter(names = {ARG_SECURE}, description = "Is security required")
- public boolean secure = false;
-
- @Override
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public boolean getHadoopServicesRequired() {
- return false;
- }
-
- @Override
- public boolean disableSecureLogin() {
- return true;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
- if (keytab != null && SliderUtils.isUnset(principal)) {
- throw new UsageException("Missing argument " + ARG_PRINCIPAL);
- }
- if (keytab == null && SliderUtils.isSet(principal)) {
- throw new UsageException("Missing argument " + ARG_KEYTAB);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java
deleted file mode 100644
index 7a46c66..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_KEYTAB},
- commandDescription = SliderActions.DESCRIBE_ACTION_KEYTAB)
-
-public class ActionKeytabArgs extends AbstractActionArgs {
-
- public ActionKeytabArgs() {
- super();
- }
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_INSTALL_KEYTAB;
- }
-
- @Parameter(names = {ARG_KEYTABINSTALL},
- description = "Install the keytab")
- public boolean install;
-
- @Parameter(names = {ARG_KEYTABDELETE},
- description = "Delete the keytab")
- public boolean delete;
-
- @Parameter(names = {ARG_KEYTABLIST},
- description = "List of installed keytabs")
- public boolean list;
-
- @Parameter(names = {ARG_KEYTAB},
- description = "Path or name of the keytab")
- public String keytab;
-
- @Parameter(names = {ARG_FOLDER},
- description = "The name of the folder in which to store the keytab")
- public String folder;
-
- @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing keytab")
- public boolean overwrite = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 3;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKillContainerArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKillContainerArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKillContainerArgs.java
deleted file mode 100644
index e1e94bd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKillContainerArgs.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_KILL_CONTAINER},
- commandDescription = SliderActions.DESCRIBE_ACTION_KILL_CONTAINER)
-
-public class ActionKillContainerArgs extends AbstractActionArgs {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_KILL_CONTAINER;
- }
-
- @Parameter(names = {ARG_ID},
- description = "ID of the container")
- public String id;
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[82/86] [abbrv] hadoop git commit: YARN-7165. Miscellaneous fixes in
yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
deleted file mode 100644
index f56139a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
+++ /dev/null
@@ -1,606 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# YARN Simplified API layer for services
-
-## Overview
-Bringing a new service on YARN today is not a simple experience. The APIs of
-existing frameworks are either too low level (native YARN), require writing
-new code (for frameworks with programmatic APIs) or writing a complex spec
-(for declarative frameworks). In addition to building critical building blocks
-inside YARN (as part of other efforts at
-[YARN-4692](https://issues.apache.org/jira/browse/YARN-4692)), there is a need for
-simplifying the user facing story for building services. Experience of projects
-like Apache Slider running real-life services like HBase, Storm, Accumulo,
-Solr etc, gives us some very good insights on how simplified APIs for services
-should look like.
-
-To this end, we should look at a new simple-services API layer backed by REST
-interfaces. This API can be used to create and manage the lifecycle of YARN
-services. Services here can range from simple single-component service to
-complex multi-component assemblies needing orchestration.
-[YARN-4793](https://issues.apache.org/jira/browse/YARN-4793) tracks this
-effort.
-
-This document spotlights on this specification. In most of the cases, the
-application owner will not be forced to make any changes to their applications.
-This is primarily true if the application is packaged with containerization
-technologies like docker. Irrespective of how complex the application is,
-there will be hooks provided at appropriate layers to allow pluggable and
-customizable application behavior.
-
-
-### Version information
-Version: 1.0.0
-
-### License information
-License: Apache 2.0
-License URL: http://www.apache.org/licenses/LICENSE-2.0.html
-
-### URI scheme
-Host: host.mycompany.com
-
-BasePath: /ws/v1/
-
-Schemes: HTTP
-
-### Consumes
-
-* application/json
-
-
-### Produces
-
-* application/json
-
-
-## Paths
-### Create a service
-```
-POST /services
-```
-
-#### Description
-
-Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state.
-
-#### Parameters
-|Type|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|----|
-|BodyParameter|Service|Service request object|true|Service||
-
-
-#### Responses
-|HTTP Code|Description|Schema|
-|----|----|----|
-|202|The request to create a service is accepted|No Content|
-|400|Invalid service definition provided in the request body|No Content|
-|500|Failed to create a service|No Content|
-|default|Unexpected error|ServiceStatus|
-
-
-### (TBD) List of services running in the cluster.
-```
-GET /services
-```
-
-#### Description
-
-Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name.
-
-#### Responses
-|HTTP Code|Description|Schema|
-|----|----|----|
-|200|An array of services|Service array|
-|default|Unexpected error|ServiceStatus|
-
-
-### Get current version of the API server.
-```
-GET /services/version
-```
-
-#### Description
-
-Get current version of the API server.
-
-#### Responses
-|HTTP Code|Description|Schema|
-|----|----|----|
-|200|Successful request|No Content|
-
-
-### Update a service or upgrade the binary version of the components of a running service
-```
-PUT /services/{service_name}
-```
-
-#### Description
-
-Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service. The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts (TBD).
-
-#### Parameters
-|Type|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|----|
-|PathParameter|service_name|Service name|true|string||
-|BodyParameter|Service|The updated service definition. It can contain the updated lifetime of a service or the desired state (STOPPED/STARTED) of a service to initiate a start/stop operation against the specified service|true|Service||
-
-
-#### Responses
-|HTTP Code|Description|Schema|
-|----|----|----|
-|204|Update or upgrade was successful|No Content|
-|404|Service does not exist|No Content|
-|default|Unexpected error|ServiceStatus|
-
-
-### Destroy a service
-```
-DELETE /services/{service_name}
-```
-
-#### Description
-
-Destroy a service and release all resources. This API might have to return JSON data providing location of logs (TBD), etc.
-
-#### Parameters
-|Type|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|----|
-|PathParameter|service_name|Service name|true|string||
-
-
-#### Responses
-|HTTP Code|Description|Schema|
-|----|----|----|
-|204|Destroy was successful|No Content|
-|404|Service does not exist|No Content|
-|default|Unexpected error|ServiceStatus|
-
-
-### Get details of a service.
-```
-GET /services/{service_name}
-```
-
-#### Description
-
-Return the details (including containers) of a running service
-
-#### Parameters
-|Type|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|----|
-|PathParameter|service_name|Service name|true|string||
-
-
-#### Responses
-|HTTP Code|Description|Schema|
-|----|----|----|
-|200|a service object|object|
-|404|Service does not exist|No Content|
-|default|Unexpected error|ServiceStatus|
-
-
-### Flex a component's number of instances.
-```
-PUT /services/{service_name}/components/{component_name}
-```
-
-#### Description
-
-Set a component's desired number of instanes
-
-#### Parameters
-|Type|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|----|
-|PathParameter|service_name|Service name|true|string||
-|PathParameter|component_name|Component name|true|string||
-|BodyParameter|Component|The definition of a component which contains the updated number of instances.|true|Component||
-
-
-#### Responses
-|HTTP Code|Description|Schema|
-|----|----|----|
-|200|Flex was successful|No Content|
-|404|Service does not exist|No Content|
-|default|Unexpected error|ServiceStatus|
-
-
-## Definitions
-### Artifact
-
-Artifact of a service component. If not specified, component will just run the bare launch command and no artifact will be localized.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|id|Artifact id. Examples are package location uri for tarball based services, image name for docker, name of service, etc.|true|string||
-|type|Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For SERVICE type, the service specified will be read and its components will be added into this service. The original component with artifact type SERVICE will be removed (any properties specified in the original component will be ignored).|false|enum (DOCKER, TARBALL, SERVICE)|DOCKER|
-|uri|Artifact location to support multiple artifact stores (optional).|false|string||
-
-
-### Component
-
-One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|name|Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.|true|string||
-|dependencies|An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of a service should be represented as a DAG.|false|string array||
-|readiness_check|Readiness check for this component.|false|ReadinessCheck||
-|artifact|Artifact of the component (optional). If not specified, the service level global artifact takes effect.|false|Artifact||
-|launch_command|The custom launch command of this component (optional for DOCKER component, required otherwise). When specified at the component level, it overrides the value specified at the global level (if any).|false|string||
-|resource|Resource of this component (optional). If not specified, the service level global resource takes effect.|false|Resource||
-|number_of_containers|Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.|false|integer (int64)||
-|run_privileged_container|Run all containers of this component in privileged mode (YARN-4262).|false|boolean||
-|placement_policy|Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.|false|PlacementPolicy||
-|configuration|Config properties for this component.|false|Configuration||
-|quicklinks|A list of quicklink keys defined at the service level, and to be resolved by this component.|false|string array||
-
-
-### ConfigFile
-
-A config file that needs to be created and made available as a volume in a service component container.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|type|Config file in the standard format like xml, properties, json, yaml, template.|false|enum (XML, PROPERTIES, JSON, YAML, TEMPLATE, ENV, HADOOP_XML)||
-|dest_file|The path that this configuration file should be created as. If it is an absolute path, it will be mounted into the DOCKER container. Absolute paths are only allowed for DOCKER containers. If it is a relative path, only the file name should be provided, and the file will be created in the container local working directory under a folder named conf.|false|string||
-|src_file|This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.|false|string||
-|props|A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.|false|object||
-
-
-### Configuration
-
-Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|properties|A blob of key-value pairs of common service properties.|false|object||
-|env|A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.|false|object||
-|files|Array of list of files that needs to be created and made available as volumes in the service component containers.|false|ConfigFile array||
-
-
-### Container
-
-An instance of a running service container.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|id|Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002.|false|string||
-|launch_time|The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.|false|string (date)||
-|ip|IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.|false|string||
-|hostname|Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.|false|string||
-|bare_host|The bare node or host in which the container is running, e.g. cn008.example.com.|false|string||
-|state|State of the container of a service.|false|ContainerState||
-|component_name|Name of the component that this container instance belongs to.|false|string||
-|resource|Resource used for this container.|false|Resource||
-|artifact|Artifact used for this container.|false|Artifact||
-|privileged_container|Container running in privileged mode or not.|false|boolean||
-
-
-### ContainerState
-
-The current state of the container of a service.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|state|enum of the state of the container|false|enum (INIT, STARTED, READY)||
-
-
-### PlacementPolicy
-
-Placement policy of an instance of a service. This feature is in the works in YARN-6592.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|label|Assigns a service to a named partition of the cluster where the service desires to run (optional). If not specified all services are submitted to a default label of the service owner. One or more labels can be setup for each service owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.|false|string||
-
-
-### ReadinessCheck
-
-A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|type|E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).|true|enum (HTTP, PORT)||
-|props|A blob of key value pairs that will be used to configure the check.|false|object||
-|artifact|Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET|false|Artifact||
-
-
-### Resource
-
-Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|profile|Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.|false|string||
-|cpus|Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).|false|integer (int32)||
-|memory|Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.|false|string||
-
-
-### Service
-
-a service resource has the following attributes.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|name|A unique service name. If Registry DNS is enabled, the max length is 63 characters.|true|string||
-|id|A unique service id.|false|string||
-|artifact|Artifact of single-component service.|false|Artifact||
-|resource|Resource of single-component service or the global default for multi-component services. Mandatory if it is a single-component service and if cpus and memory are not specified at the Service level.|false|Resource||
-|launch_command|The custom launch command of a service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section.|false|string||
-|launch_time|The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.|false|string (date)||
-|number_of_containers|Number of containers for each component in the service. Each component can further override this service-level global default.|false|integer (int64)||
-|number_of_running_containers|In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.|false|integer (int64)||
-|lifetime|Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.|false|integer (int64)||
-|placement_policy|(TBD) Advanced scheduling and placement policies. If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.|false|PlacementPolicy||
-|components|Components of a service.|false|Component array||
-|configuration|Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.|false|Configuration||
-|containers|Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.|false|Container array||
-|state|State of the service. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started service.|false|ServiceState||
-|quicklinks|A blob of key-value pairs of quicklinks to be exported for a service.|false|object||
-|queue|The YARN queue that this service should be submitted to.|false|string||
-
-
-### ServiceState
-
-The current state of a service.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|state|enum of the state of the service|false|enum (ACCEPTED, STARTED, READY, STOPPED, FAILED)||
-
-
-### ServiceStatus
-
-The current status of a submitted service, returned as a response to the GET API.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|diagnostics|Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state.|false|string||
-|state|Service state.|false|ServiceState||
-|code|An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.|false|integer (int32)||
-
-
-
-## Examples
-
-### Create a simple single-component service with most attribute values as defaults
-POST URL - http://localhost:9191/ws/v1/services
-
-##### POST Request JSON
-```json
-{
- "name": "hello-world",
- "components" :
- [
- {
- "name": "hello",
- "number_of_containers": 1,
- "artifact": {
- "id": "nginx:latest",
- "type": "DOCKER"
- },
- "launch_command": "./start_nginx.sh",
- "resource": {
- "cpus": 1,
- "memory": "256"
- }
- }
- ]
-}
-```
-
-##### GET Response JSON
-GET URL - http://localhost:9191/ws/v1/services/hello-world
-
-Note, lifetime value of -1 means unlimited lifetime.
-
-```json
-{
- "name": "hello-world",
- "id": "application_1503963985568_0002",
- "lifetime": -1,
- "components": [
- {
- "name": "hello",
- "dependencies": [],
- "resource": {
- "cpus": 1,
- "memory": "256"
- },
- "configuration": {
- "properties": {},
- "env": {},
- "files": []
- },
- "quicklinks": [],
- "containers": [
- {
- "id": "container_e03_1503963985568_0002_01_000001",
- "ip": "10.22.8.143",
- "hostname": "myhost.local",
- "state": "READY",
- "launch_time": 1504051512412,
- "bare_host": "10.22.8.143",
- "component_name": "hello-0"
- },
- {
- "id": "container_e03_1503963985568_0002_01_000002",
- "ip": "10.22.8.143",
- "hostname": "myhost.local",
- "state": "READY",
- "launch_time": 1504051536450,
- "bare_host": "10.22.8.143",
- "component_name": "hello-1"
- }
- ],
- "launch_command": "./start_nginx.sh",
- "number_of_containers": 1,
- "run_privileged_container": false
- }
- ],
- "configuration": {
- "properties": {},
- "env": {},
- "files": []
- },
- "quicklinks": {}
-}
-
-```
-### Update to modify the lifetime of a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
-
-##### PUT Request JSON
-
-Note, irrespective of what the current lifetime value is, this update request will set the lifetime of the service to be 3600 seconds (1 hour) from the time the request is submitted. Hence, if a a service has remaining lifetime of 5 mins (say) and would like to extend it to an hour OR if an application has remaining lifetime of 5 hours (say) and would like to reduce it down to an hour, then for both scenarios you need to submit the same request below.
-
-```json
-{
- "lifetime": 3600
-}
-```
-### Stop a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
-
-##### PUT Request JSON
-```json
-{
- "state": "STOPPED"
-}
-```
-
-### Start a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
-
-##### PUT Request JSON
-```json
-{
- "state": "STARTED"
-}
-```
-
-### Update to flex up/down the no of containers (instances) of a component of a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello
-
-##### PUT Request JSON
-```json
-{
- "name": "hello",
- "number_of_containers": 3
-}
-```
-
-### Destroy a service
-DELETE URL - http://localhost:9191/ws/v1/services/hello-world
-
-***
-
-### Create a complicated service - HBase
-POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
-
-##### POST Request JSON
-
-```json
-{
- "name": "hbase-app-1",
- "lifetime": "3600",
- "components": [
- {
- "name": "hbasemaster",
- "number_of_containers": 1,
- "artifact": {
- "id": "hbase:latest",
- "type": "DOCKER"
- },
- "launch_command": "/usr/hdp/current/hbase-master/bin/hbase master start",
- "resource": {
- "cpus": 1,
- "memory": "2048"
- },
- "configuration": {
- "env": {
- "HBASE_LOG_DIR": "<LOG_DIR>"
- },
- "files": [
- {
- "type": "XML",
- "dest_file": "/etc/hadoop/conf/core-site.xml",
- "props": {
- "fs.defaultFS": "${CLUSTER_FS_URI}"
- }
- },
- {
- "type": "XML",
- "dest_file": "/etc/hbase/conf/hbase-site.xml",
- "props": {
- "hbase.cluster.distributed": "true",
- "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
- "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
- "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
- "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
- "hbase.master.info.port": "16010"
- }
- }
- ]
- }
- },
- {
- "name": "regionserver",
- "number_of_containers": 3,
- "unique_component_support": "true",
- "artifact": {
- "id": "hbase:latest",
- "type": "DOCKER"
- },
- "launch_command": "/usr/hdp/current/hbase-regionserver/bin/hbase regionserver start",
- "resource": {
- "cpus": 1,
- "memory": "2048"
- },
- "configuration": {
- "env": {
- "HBASE_LOG_DIR": "<LOG_DIR>"
- },
- "files": [
- {
- "type": "XML",
- "dest_file": "/etc/hadoop/conf/core-site.xml",
- "props": {
- "fs.defaultFS": "${CLUSTER_FS_URI}"
- }
- },
- {
- "type": "XML",
- "dest_file": "/etc/hbase/conf/hbase-site.xml",
- "props": {
- "hbase.cluster.distributed": "true",
- "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
- "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
- "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
- "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
- "hbase.master.info.port": "16010",
- "hbase.regionserver.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}"
- }
- }
- ]
- }
- }
- ],
- "quicklinks": {
- "HBase Master Status UI": "http://hbasemaster0.${SERVICE_NAME}.${USER}.${DOMAIN}:16010/master-status",
- "Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/"
- }
-}
-```
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
deleted file mode 100644
index a927118..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
+++ /dev/null
@@ -1,144 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# YARN DNS Server
-
-<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
-
-## Introduction
-
-The YARN DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions:
-
-1. **Exposing existing service-discovery information via DNS** - Information provided in
-the current YARN service registry’s records will be converted into DNS entries, thus
-allowing users to discover information about YARN applications using standard DNS
-client mechanisms (for e.g. a DNS SRV Record specifying the hostname and port
-number for services).
-2. **Enabling Container to IP mappings** - Enables discovery of the IPs of containers via
-standard DNS lookups. Given the availability of the records via DNS, container
-name-based communication will be facilitated (e.g. ‘curl
-http://myContainer.myDomain.com/endpoint’).
-
-## Service Properties
-
-The existing YARN Service Registry is leveraged as the source of information for the DNS Service.
-
-The following core functions are supported by the DNS-Server:
-
-### Functional properties
-
-1. Supports creation of DNS records for end-points of the deployed YARN applications
-2. Record names remain unchanged during restart of containers and/or applications
-3. Supports reverse lookups (name based on IP).
-4. Supports security using the standards defined by The Domain Name System Security
-Extensions (DNSSEC)
-5. Highly available
-6. Scalable - The service provides the responsiveness (e.g. low-latency) required to
-respond to DNS queries (timeouts yield attempts to invoke other configured name
-servers).
-
-### Deployment properties
-
-1. Supports integration with existing DNS assets (e.g. a corporate DNS server) by acting as
-a DNS server for a Hadoop cluster zone/domain. The server is not intended to act as a
-primary DNS server and does not forward requests to other servers.
-2. The DNS Server exposes a port that can receive both TCP and UDP requests per
-DNS standards. The default port for DNS protocols is in a restricted, administrative port
-range (53), so the port is configurable for deployments in which the service may
-not be managed via an administrative account.
-
-## DNS Record Name Structure
-
-The DNS names of generated records are composed from the following elements (labels). Note that these elements must be compatible with DNS conventions (see “Preferred Name Syntax” in RFC 1035):
-
-* **domain** - the name of the cluster DNS domain. This name is provided as a
-configuration property. In addition, it is this name that is configured at a parent DNS
-server as the zone name for the defined yDNS zone (the zone for which the parent DNS
-server will forward requests to yDNS). E.g. yarncluster.com
-* **username** - the name of the application deployer. This name is the simple short-name (for
-e.g. the primary component of the Kerberos principal) associated with the user launching
-the application. As the username is one of the elements of DNS names, it is expected
-that this also confirms DNS name conventions (RFC 1035 linked above), so special translation is performed for names with special characters like hyphens and spaces.
-* **application name** - the name of the deployed YARN application. This name is inferred
-from the YARN registry path to the application's node. Application name, rather thn application id, was chosen as a way of making it easy for users to refer to human-readable DNS
-names. This obviously mandates certain uniqueness properties on application names.
-* **container id** - the YARN assigned ID to a container (e.g.
-container_e3741_1454001598828_01_000004)
-* **component name** - the name assigned to the deployed component (for e.g. a master
-component). A component is a distributed element of an application or service that is
-launched in a YARN container (e.g. an HBase master). One can imagine multiple
-components within an application. A component name is not yet a first class concept in
-YARN, but is a very useful one that we are introducing here for the sake of yDNS
-entries. Many frameworks like MapReduce, Slider already have component names
-(though, as mentioned, they are not yet supported in YARN in a first class fashion).
-* **api** - the api designation for the exposed endpoint
-
-### Notes about DNS Names
-
-* In most instances, the DNS names can be easily distinguished by the number of
-elements/labels that compose the name. The cluster’s domain name is always the last
-element. After that element is parsed out, reading from right to left, the first element
-maps to the application user and so on. Wherever it is not easily distinguishable, naming conventions are used to disambiguate the name using a prefix such as
-“container” or suffix such as “api”. For example, an endpoint published as a
-management endpoint will be referenced with the name *management-api.griduser.yarncluster.com*.
-* Unique application name (per user) is not currently supported/guaranteed by YARN, but
-it is supported by frameworks such as Apache Slider. The yDNS service currently
-leverages the last element of the ZK path entry for the application as an
-application name. These application names have to be unique for a given user.
-
-## DNS Server Functionality
-
-The primary functions of the DNS service are illustrated in the following diagram:
-
-
-
-
-### DNS record creation
-The following figure illustrates at slightly greater detail the DNS record creation and registration sequence (NOTE: service record updates would follow a similar sequence of steps,
-distinguished only by the different event type):
-
-
-
-### DNS record removal
-Similarly, record removal follows a similar sequence
-
-
-
-(NOTE: The DNS Zone requires a record as an argument for the deletion method, thus
-requiring similar parsing logic to identify the specific records that should be removed).
-
-### DNS Service initialization
-* The DNS service initializes both UDP and TCP listeners on a configured port. As
-noted above, the default port of 53 is in a restricted range that is only accessible to an
-account with administrative privileges.
-* Subsequently, the DNS service listens for inbound DNS requests. Those requests are
-standard DNS requests from users or other DNS servers (for example, DNS servers that have the
-YARN DNS service configured as a forwarder).
-
-## Configuration
-The YARN DNS server reads its configuration properties from the yarn-site.xml file. The following are the DNS associated configuration properties:
-
-| Name | Description |
-| ------------ | ------------- |
-| hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. |
-| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. |
-| hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. |
-| hadoop.registry.dns.bind-port | The port number for the DNS listener. The default port is 53. However, since that port falls in a administrator-only range, typical deployments may need to specify an alternate port. |
-| hadoop.registry.dns.dnssec.enabled | Indicates whether the DNSSEC support is enabled. Default is false. |
-| hadoop.registry.dns.public-key | The base64 representation of the server’s public key. Leveraged for creating the DNSKEY Record provided for DNSSEC client requests. |
-| hadoop.registry.dns.private-key-file | The path to the standard DNSSEC private key file. Must only be readable by the DNS launching identity. See [dnssec-keygen](https://ftp.isc.org/isc/bind/cur/9.9/doc/arm/man.dnssec-keygen.html) documentation. |
-| hadoop.registry.dns-ttl | The default TTL value to associate with DNS records. The default value is set to 1 (a value of 0 has undefined behavior). A typical value should be approximate to the time it takes YARN to restart a failed container. |
-| hadoop.registry.dns.zone-subnet | An indicator of the IP range associated with the cluster containers. The setting is utilized for the generation of the reverse zone name. |
-| hadoop.registry.dns.zone-mask | The network mask associated with the zone IP range. If specified, it is utilized to ascertain the IP range possible and come up with an appropriate reverse zone name. |
-| hadoop.registry.dns.zones-dir | A directory containing zone configuration files to read during zone initialization. This directory can contain zone master files named *zone-name.zone*. See [here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file documentation.|
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
deleted file mode 100644
index e6a4e91..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
+++ /dev/null
@@ -1,107 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# Introduction: YARN Native Services
-
-## Overview
-YARN Native Services provides first class framework support and APIs to host long running services natively in YARN. In addition to launching services, the new APIs support performing lifecycle management operations, such as flex service components up/down, manage lifetime, upgrade the service to a newer version, and stop/restart/delete the service.
-
-The native services capabilities are built on the existing low-level resource management API provided by YARN that can support any type of application. Other application frameworks like Hadoop MapReduce already expose higher level APIs that users can leverage to run applications on top of YARN. With the advent of containerization technologies like Docker, providing first class support and APIs for long running services at the framework level made sense.
-
-Relying on a framework has the advantage of exposing a simpler usage model to the user by enabling service configuration and launch through specification (without writing new code), as well as hiding complex low-level details including state management and fault-tolerance etc. Users/operators of existing services typically like to avoid modifying an existing service to be aware of YARN. With first class support capable of running a single Docker image as well as complex assemblies comprised of multiple Docker images, there is no need for service owners to be aware of YARN. Developers of new services do not have to worry about YARN internals and only need to focus on containerization of their service(s).
-
-## First class support for services
-In order to natively provide first class support for long running services, several new features and improvements have been made at the framework level.
-
-### Incorporate Apache Slider into Apache YARN
-Apache Slider, which existed as a separate incubator project has been merged into YARN to kick start the first class support. Apache Slider is a universal Application Master (AM) which had several key features built in - fault tolerance of service containers and AM, work-preserving AM restarts, service logs management, service management like flex up/down, stop/start, and rolling upgrade to newer service versions, etc. Of course lot more work has been done on top of what Apache Slider brought in, details of which follow.
-
-### Native Services API
-A significant effort has gone into simplifying the user facing story for building services. In the past, bringing a new service to YARN was not a pleasant experience. The APIs of existing frameworks are either too low-level (native YARN), require writing new code (for frameworks with programmatic APIs) or require writing a complex spec (for declarative frameworks).
-
-The new REST APIs are very simple to use. The REST layer acts as a single point of entry for creation and lifecycle management of YARN services. Services here can range from simple single-component apps to the most complex, multi-component applications needing special orchestration needs.
-
-Plan is to make this a unified REST based entry point for other important features like resource-profile management ([YARN-3926](https://issues.apache.org/jira/browse/YARN-4793)), package-definitions' lifecycle-management and service-discovery ([YARN-913](https://issues.apache.org/jira/browse/YARN-913)/[YARN-4757](https://issues.apache.org/jira/browse/YARN-4757)).
-
-### Native Services Discovery
-The new discovery solution exposes the registry information through a more generic and widely used mechanism: DNS. Service Discovery via DNS uses the well-known DNS interfaces to browse the network for services. Having the registry information exposed via DNS simplifies the life of services.
-
-The previous read mechanisms of YARN Service Registry were limited to a registry specific (java) API and a REST interface. In practice, this made it very difficult for wiring up existing clients and services. For e.g., dynamic configuration of dependent endpoints of a service was not easy to implement using the registry-read mechanisms, **without** code-changes to existing services. These are solved by the DNS based service discovery.
-
-### Scheduling
-[YARN-6592](https://issues.apache.org/jira/browse/YARN-6592) covers a host of scheduling features that are useful for short-running applications and services alike. Below, are a few very important YARN core features that help schedule services better. Without these, running services on YARN is a hassle.
-
-* Affinity (TBD)
-* Anti-affinity (TBD)
-* Gang scheduling (TBD)
-* Malleable container sizes ([YARN-1197](https://issues.apache.org/jira/browse/YARN-1197))
-
-### Resource Profiles
-YARN always had support for memory as a resource, inheriting it from Hadoop-(1.x)’s MapReduce platform. Later support for CPU as a resource ([YARN-2](https://issues.apache.org/jira/browse/YARN-2)/[YARN-3](https://issues.apache.org/jira/browse/YARN-3)) was added. Multiple efforts added support for various other resource-types in YARN such as disk ([YARN-2139](https://issues.apache.org/jira/browse/YARN-2139)), and network ([YARN-2140](https://issues.apache.org/jira/browse/YARN-2140)), specifically benefiting long running services.
-
-In many systems outside of YARN, users are already accustomed to specifying their desired ‘box’ of requirements where each box comes with a predefined amount of each resources. Admins would define various available box-sizes (small, medium, large etc) and users would pick the ones they desire and everybody is happy. In [YARN-3926](https://issues.apache.org/jira/browse/YARN-3926), YARN introduces Resource Profiles which extends the YARN resource model for easier resource-type management and profiles. This helps in two ways - the system can schedule applications better and it can perform intelligent over-subscription of resources where applicable.
-
-Resource profiles are all the more important for services since -
-* Similar to short running apps, you don’t have to fiddle with varying resource-requirements for each container type
-* Services usually end up planning for peak usages, leaving a lot of possibility of barren utilization
-
-### Special handling of preemption and container reservations
-TBD.
-
-Preemption and reservation of long running containers have different implications from regular ones. Preemption of resources in YARN today works by killing of containers. For long-lived services this is unacceptable. Also, scheduler should avoid allocating long running containers on borrowed resources. [YARN-4724](https://issues.apache.org/jira/browse/YARN-4724) will address some of these special recognition of service containers.
-
-### Container auto-restarts
-If a service container dies, expiring container's allocation and releasing the allocation is undesirable in many cases. Long running containers may exit for various reasons, crash and need to restart but forcing them to go through the complete scheduling cycle, resource localization, etc. is both unnecessary and expensive.
-
-Services can enable app-specific policies to prevent NodeManagers to automatically restart containers. [YARN-3998](https://issues.apache.org/jira/browse/YARN-3998) implements a retry-policy to let NM re-launch a service container when it fails.
-
-### Container allocation re-use for application upgrades
-TBD.
-
-Auto-restart of containers will support upgrade of service containers without reclaiming the resources first. During an upgrade, with multitude of other applications running in the system, giving up and getting back resources allocated to the service is hard to manage. Node-Labels help this cause but are not straight-forward to use to address the app-specific use-cases. The umbrella [YARN-4726](https://issues.apache.org/jira/browse/YARN-4726) along with [YARN-5620](https://issues.apache.org/jira/browse/YARN-5620) and [YARN-4470](https://issues.apache.org/jira/browse/YARN-4470) will take care of this.
-
-### Dynamic Configurations
-Most production-level services require dynamic configurations to manage and simplify their lifecycle. Container’s resource size, local/work dirs and log-dirs are the most basic information services need. Service's endpoint details (host/port), their inter-component dependencies, health-check endpoints, etc. are all critical to the success of today's real-life services.
-
-### Resource re-localization for reconfiguration/upgrades
-TBD
-
-### Service Registry
-TBD
-
-### Service persistent storage and volume support
-TBD
-
-### Packaging
-TBD
-
-### Container image registry (private, public and hybrid)
-TBD
-
-### Container image management and APIs
-TBD
-
-### Container image storage
-TBD
-
-### Monitoring
-TBD
-
-### Metrics
-TBD
-
-### Service Logs
-TBD
-
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
new file mode 100644
index 0000000..7b62c36
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
@@ -0,0 +1,77 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Concepts
+This document describes some key concepts and features that makes YARN as a first-class platform in order to natively support long running services on YARN.
+
+### Service Framework (ApplicationMaster) on YARN
+A container orchestration framework is implemented to help deploying services on YARN. In a nutshell, the framework is an ApplicationMaster that
+requests containers from ResourceManager based on service definition provided by the user and launch the containers across the cluster adhering to placement policies.
+It also does all the heavy lifting work such as resolving the service definition and configurations, managing component life cycles such as automatically restarting
+failed containers, monitoring components' healthiness and readiness, ensuring dependency start order across components, flexing up/down components,
+upgrading components etc. The end goal of the framework is to make sure the service is up and running as the state that user desired.
+
+
+### A Restful API-Server for deploying/managing services on YARN
+A restful API server is developed to allow users to deploy/manage their services on YARN via a simple JSON spec. This avoids users
+from dealing with the low-level APIs, writing complex code to bring their services onto YARN. The REST layer acts as a unified REST based entry for
+creation and lifecycle management of YARN services. Services here can range from simple single-component apps to the most complex,
+multi-component applications needing special orchestration needs. Please refer to this [API doc](YarnServiceAPI.md) for detailed API documentations.
+
+The API-server is stateless, which means users can simply spin up multiple instances, and have a load balancer fronting them to
+support HA, distribute the load etc.
+
+### Service Discovery
+A DNS server is implemented to enable discovering services on YARN via the standard mechanism: DNS lookup.
+The DNS server essentially exposes the information in YARN service registry by translating them into DNS records such as A record and SRV record.
+Clients can discover the IPs of containers via standard DNS lookup.
+The previous read mechanisms of YARN Service Registry were limited to a registry specific (java) API and a REST interface and are difficult
+to wireup existing clients and services. The DNS based service discovery eliminates this gap. Please refer to this [DNS doc](ServiceDiscovery.md)
+for more details.
+
+### Scheduling
+
+A host of scheduling features are being developed to support long running services.
+
+* Affinity and anti-affinity scheduling across containers ([YARN-6592](https://issues.apache.org/jira/browse/YARN-6592)).
+* Container resizing ([YARN-1197](https://issues.apache.org/jira/browse/YARN-1197))
+* Special handling of container preemption/reservation for services
+
+### Container auto-restarts
+
+[YARN-3998](https://issues.apache.org/jira/browse/YARN-3998) implements a retry-policy to let NM re-launch a service container when it fails.
+The service REST API provides users a way to enable NodeManager to automatically restart the container if it fails.
+The advantage is that it avoids the entire cycle of releasing the failed containers, re-asking new containers, re-do resource localizations and so on, which
+greatly minimizes container downtime.
+
+
+### Container in-place upgrade
+
+[YARN-4726](https://issues.apache.org/jira/browse/YARN-4726) aims to support upgrading containers in-place, that is, without losing the container allocations.
+It opens up a few APIs in NodeManager to allow ApplicationMasters to upgrade their containers via a simple API call.
+Under the hood, NodeManager does below steps:
+* Downloading the new resources such as jars, docker container images, new configurations.
+* Stop the old container.
+* Start the new container with the newly downloaded resources.
+
+At the time of writing this document, core changes are done but the feature is not usable end-to-end.
+
+### Resource Profiles
+
+In [YARN-3926](https://issues.apache.org/jira/browse/YARN-3926), YARN introduces Resource Profiles which extends the YARN resource model for easier
+resource-type management and profiles.
+It primarily solves two problems:
+* Make it easy to support new resource types such as network bandwith([YARN-2140](https://issues.apache.org/jira/browse/YARN-2140)), disks([YARN-2139](https://issues.apache.org/jira/browse/YARN-2139)).
+ Under the hood, it unifies the scheduler codebase to essentially parameterize the resource types.
+* User can specify the container resource requirement by a profile name, rather than fiddling with varying resource-requirements for each resource type.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
new file mode 100644
index 0000000..3dd4d8c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
@@ -0,0 +1,58 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# YARN Service
+## Overview
+Yarn Service framework provides first class support and APIs to host long running services natively in YARN.
+In a nutshell, it serves as a container orchestration platform for managing containerized services on YARN. It supports both docker container
+and traditional process based containers in YARN.
+
+The responsibility of this framework includes performing configuration resolutions and mounts,
+lifecycle management such as stop/start/delete the service, flexing service components up/down, rolling upgrades services on YARN, monitoring services' healthiness and readiness and more.
+
+The yarn-service framework primarily includes below components:
+
+* A core framework (ApplicationMaster) running on YARN to serve as a container orchestrator, being responsible for all service lifecycle managements.
+* A restful API-server to for users to interact with YARN to deploy/manage their services via a simple JSON spec.
+* A DNS server backed by YARN service registry to enable discovering services on YARN by the standard DNS lookup.
+
+## Why should I try YARN Service framework?
+
+YARN Service framework makes it easy to bring existing services onto YARN.
+It hides all the complex low-level details of application management and relieves
+users from forced into writing new code. Developers of new services do not have
+to worry about YARN internals and only need to focus on containerization of their
+service(s).
+
+Further, another huge win of this feature is that now you can enable both
+traditional batch processing jobs and long running services in a single platform!
+The benefits of combining these workloads are two-fold:
+
+* Greatly simplify the cluster operations as you have only a single cluster to deal with.
+* Making both batch jobs and services share a cluster can greatly improve resource utilization.
+
+## How do I get started?
+
+*`This feature is in alpha state`* and so APIs, command lines are subject to change. We will continue to update the documents over time.
+
+[QuickStart](QuickStart.md) shows a quick tutorial that walks you through simple steps to deploy a service on YARN.
+
+## How do I get my hands dirty?
+
+* [Concepts](Concepts.md): Describes the internals of the framework and some features in YARN core to support running services on YARN.
+* [Service REST API](YarnServiceAPI.md): The API doc for deploying/managing services on YARN.
+* [Service Discovery](ServiceDiscovery.md): Deep dives into the YARN DNS internals
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
new file mode 100644
index 0000000..327566b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -0,0 +1,218 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Quick Start
+
+This document describes how to deploy services on YARN using the YARN Service framework.
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
+## Start HDFS and YARN components
+
+ Start all the hadoop components HDFS, YARN as usual.
+
+
+## Example service
+Below is a simple service definition that launches sleep containers on YARN by writing a simple spec file and without writing any code.
+
+```
+{
+ "name": "sleeper-service",
+ "components" :
+ [
+ {
+ "name": "sleeper",
+ "number_of_containers": 1,
+ "launch_command": "sleep 900000",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ }
+ }
+ ]
+}
+```
+
+For launching docker based services using YARN Service framework, please refer to [API doc](YarnServiceAPI.md).
+
+## Manage services on YARN via CLI
+Below steps walk you through deploying a services on YARN using CLI.
+Refer to [Yarn Commands](../YarnCommands.md) for the full list of commands and options.
+### Deploy a service
+```
+yarn service create --file ${PATH_TO_SERVICE_DEF_FILE}
+```
+Params:
+- SERVICE_NAME: The name of the service. Note that this needs to be unique across all running services.
+- PATH_TO_SERVICE_DEF: The path to the service definition file in JSON format.
+
+For example:
+```
+yarn service create --file /path/to/local/sleeper.json
+```
+
+### Flex a component of a service
+Increase or decrease the number of containers for a component.
+```
+yarn service flex ${SERVICE_NAME} --component ${COMPONENT_NAME} ${NUMBER_OF_CONTAINERS}
+```
+For example, for a service named `sleeper-service`:
+
+Set the `sleeper` component to `2` containers (absolute number).
+
+```
+yarn service flex sleeper-service --component sleeper 2
+```
+
+### Stop a service
+Stopping a service will stop all containers of the service and the ApplicationMaster, but does not delete the state of a service, such as the service root folder on hdfs.
+```
+yarn service stop ${SERVICE_NAME}
+```
+
+### Restart a stopped service
+Restarting a stopped service is easy - just call start!
+```
+yarn service start ${SERVICE_NAME}
+```
+
+### Destroy a service
+In addition to stopping a service, it also deletes the service root folder on hdfs and the records in YARN Service Registry.
+```
+yarn service destroy ${SERVICE_NAME}
+```
+
+## Manage services on YARN via REST API
+Below steps walk you through deploying services on YARN via REST API.
+ Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
+### Start API-Server for deploying services on YARN
+API server is the service that sits in front of YARN ResourceManager and lets users submit their API specs via HTTP.
+```
+ yarn --daemon start apiserver
+ ```
+The above command starts the API Server on the localhost at port 9191 by default.
+
+### Deploy a service
+POST the aforementioned example service definition to the api-server endpoint:
+```
+POST http://localhost:9191/ws/v1/services
+```
+
+### Get a service status
+```
+GET http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+```
+
+### Flex a component of a service
+```
+PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}/components/${COMPONENT_NAME}
+```
+`PUT` Request body:
+```
+{
+ "name": "${COMPONENT_NAME}",
+ "number_of_containers": ${COUNT}
+}
+```
+For example:
+```
+{
+ "name": "sleeper",
+ "number_of_containers": 2
+}
+```
+
+### Stop a service
+Stopping a service will stop all containers of the service and the ApplicationMaster, but does not delete the state of a service, such as the service root folder on hdfs.
+
+```
+PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+```
+
+`PUT` Request body:
+```
+{
+ "name": "${SERVICE_NAME}",
+ "state": "STOPPED"
+}
+```
+
+### Restart a stopped service
+Restarting a stopped service is easy.
+
+```
+PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+```
+
+`PUT` Request body:
+```
+{
+ "name": "${SERVICE_NAME}",
+ "state": "STARTED"
+}
+```
+### Destroy a service
+In addition to stopping a service, it also deletes the service root folder on hdfs and the records in YARN Service Registry.
+```
+DELETE http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+```
+
+## Services UI with YARN UI2 and Timeline Service v2
+A new `service` tab is added in the YARN UI2 specially to show YARN Services in a first class manner.
+The services framework posts the data into TimelineService and the `service` UI reads data from TimelineService to render its content.
+
+### Enable Timeline Service v2
+Please refer to [TimeLineService v2 doc](../TimelineServiceV2.md) for how to enable Timeline Service v2.
+
+### Enable new YARN UI
+
+Set below config in `yarn-site.xml` and start ResourceManager.
+If you are building from source code, make sure you use `-Pyarn-ui` in the `mvn` command - this will generate the war file for the new YARN UI.
+```
+ <property>
+ <description>To enable RM web ui2 application.</description>
+ <name>yarn.webapp.ui2.enable</name>
+ <value>true</value>
+ </property>
+```
+
+## Service Discovery with YARN DNS
+YARN Service framework comes with a DNS server (backed by YARN Service Registry) which enables DNS based discovery of services deployed on YARN.
+That is, user can simply access their services in a well-defined naming format as below:
+
+```
+${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}
+```
+For example, in a cluster whose domain name is `yarncluster` (as defined by the `hadoop.registry.dns.domain-name` in `yarn-site.xml`), a service named `hbase` deployed by user `dev`
+with two components `hbasemaster` and `regionserver` can be accessed as below:
+
+This URL points to the usual hbase master UI
+```
+http://hbasemaster-0.hbase.dev.yarncluster:16010/master-status
+```
+
+
+Note that YARN service framework assigns COMPONENT_INSTANCE_NAME for each container in a sequence of monotonically increasing integers. For example, `hbasemaster-0` gets
+assigned `0` since it is the first and only instance for the `hbasemaster` component. In case of `regionserver` component, it can have multiple containers
+ and so be named as such: `regionserver-0`, `regionserver-1`, `regionserver-2` ... etc
+
+`Disclaimer`: The DNS implementation is still experimental. It should not be used as a fully-functional corporate DNS.
+
+### Start the DNS server
+By default, the DNS runs on non-privileged port `5353`.
+If it is configured to use the standard privileged port `53`, the DNS server needs to be run as root:
+```
+sudo su - -c "yarn org.apache.hadoop.registry.server.dns.RegistryDNSServer > /${HADOOP_LOG_FOLDER}/registryDNS.log 2>&1 &" root
+```
+Please refer to [YARN DNS doc](ServicesDiscovery.md) for the full list of configurations.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
new file mode 100644
index 0000000..6318a07
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
@@ -0,0 +1,150 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# YARN DNS Server
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
+## Introduction
+
+The YARN DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions:
+
+1. **Exposing existing service-discovery information via DNS** - Information provided in
+the current YARN service registry’s records will be converted into DNS entries, thus
+allowing users to discover information about YARN applications using standard DNS
+client mechanisms (for e.g. a DNS SRV Record specifying the hostname and port
+number for services).
+2. **Enabling Container to IP mappings** - Enables discovery of the IPs of containers via
+standard DNS lookups. Given the availability of the records via DNS, container
+name-based communication will be facilitated (e.g. ‘curl
+http://myContainer.myDomain.com/endpoint’).
+
+## Service Properties
+
+The existing YARN Service Registry is leveraged as the source of information for the DNS Service.
+
+The following core functions are supported by the DNS-Server:
+
+### Functional properties
+
+1. Supports creation of DNS records for end-points of the deployed YARN applications
+2. Record names remain unchanged during restart of containers and/or applications
+3. Supports reverse lookups (name based on IP). Note, this works only for Docker containers.
+4. Supports security using the standards defined by The Domain Name System Security
+Extensions (DNSSEC)
+5. Highly available
+6. Scalable - The service provides the responsiveness (e.g. low-latency) required to
+respond to DNS queries (timeouts yield attempts to invoke other configured name
+servers).
+
+### Deployment properties
+
+1. Supports integration with existing DNS assets (e.g. a corporate DNS server) by acting as
+a DNS server for a Hadoop cluster zone/domain. The server is not intended to act as a
+primary DNS server and does not forward requests to other servers.
+2. The DNS Server exposes a port that can receive both TCP and UDP requests per
+DNS standards. The default port for DNS protocols is in a restricted, administrative port
+range (5353), so the port is configurable for deployments in which the service may
+not be managed via an administrative account.
+
+## DNS Record Name Structure
+
+The DNS names of generated records are composed from the following elements (labels). Note that these elements must be compatible with DNS conventions (see “Preferred Name Syntax” in RFC 1035):
+
+* **domain** - the name of the cluster DNS domain. This name is provided as a
+configuration property. In addition, it is this name that is configured at a parent DNS
+server as the zone name for the defined yDNS zone (the zone for which the parent DNS
+server will forward requests to yDNS). E.g. yarncluster.com
+* **username** - the name of the application deployer. This name is the simple short-name (for
+e.g. the primary component of the Kerberos principal) associated with the user launching
+the application. As the username is one of the elements of DNS names, it is expected
+that this also confirms DNS name conventions (RFC 1035 linked above), so special translation is performed for names with special characters like hyphens and spaces.
+* **application name** - the name of the deployed YARN application. This name is inferred
+from the YARN registry path to the application's node. Application name, rather thn application id, was chosen as a way of making it easy for users to refer to human-readable DNS
+names. This obviously mandates certain uniqueness properties on application names.
+* **container id** - the YARN assigned ID to a container (e.g.
+container_e3741_1454001598828_01_000004)
+* **component name** - the name assigned to the deployed component (for e.g. a master
+component). A component is a distributed element of an application or service that is
+launched in a YARN container (e.g. an HBase master). One can imagine multiple
+components within an application. A component name is not yet a first class concept in
+YARN, but is a very useful one that we are introducing here for the sake of yDNS
+entries. Many frameworks like MapReduce, Slider already have component names
+(though, as mentioned, they are not yet supported in YARN in a first class fashion).
+* **api** - the api designation for the exposed endpoint
+
+### Notes about DNS Names
+
+* In most instances, the DNS names can be easily distinguished by the number of
+elements/labels that compose the name. The cluster’s domain name is always the last
+element. After that element is parsed out, reading from right to left, the first element
+maps to the application user and so on. Wherever it is not easily distinguishable, naming conventions are used to disambiguate the name using a prefix such as
+“container” or suffix such as “api”. For example, an endpoint published as a
+management endpoint will be referenced with the name *management-api.griduser.yarncluster.com*.
+* Unique application name (per user) is not currently supported/guaranteed by YARN, but
+it is supported by frameworks such as Apache Slider. The yDNS service currently
+leverages the last element of the ZK path entry for the application as an
+application name. These application names have to be unique for a given user.
+
+## DNS Server Functionality
+
+The primary functions of the DNS service are illustrated in the following diagram:
+
+
+
+### DNS record creation
+The following figure illustrates at slightly greater detail the DNS record creation and registration sequence (NOTE: service record updates would follow a similar sequence of steps,
+distinguished only by the different event type):
+
+
+
+### DNS record removal
+Similarly, record removal follows a similar sequence
+
+
+
+(NOTE: The DNS Zone requires a record as an argument for the deletion method, thus
+requiring similar parsing logic to identify the specific records that should be removed).
+
+### DNS Service initialization
+* The DNS service initializes both UDP and TCP listeners on a configured port. As
+noted above, the default port of 5353 is in a restricted range that is only accessible to an
+account with administrative privileges.
+* Subsequently, the DNS service listens for inbound DNS requests. Those requests are
+standard DNS requests from users or other DNS servers (for example, DNS servers that have the
+YARN DNS service configured as a forwarder).
+
+## Start the DNS Server
+By default, the DNS runs on non-privileged port `5353`.
+If it is configured to use the standard privileged port `53`, the DNS server needs to be run as root:
+```
+sudo su - -c "yarn org.apache.hadoop.registry.server.dns.RegistryDNSServer > /${HADOOP_LOG_FOLDER}/registryDNS.log 2>&1 &" root
+```
+
+## Configuration
+The YARN DNS server reads its configuration properties from the yarn-site.xml file. The following are the DNS associated configuration properties:
+
+| Name | Description |
+| ------------ | ------------- |
+| hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. |
+| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. |
+| hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. |
+| hadoop.registry.dns.bind-port | The port number for the DNS listener. The default port is 5353. However, since that port falls in a administrator-only range, typical deployments may need to specify an alternate port. |
+| hadoop.registry.dns.dnssec.enabled | Indicates whether the DNSSEC support is enabled. Default is false. |
+| hadoop.registry.dns.public-key | The base64 representation of the server’s public key. Leveraged for creating the DNSKEY Record provided for DNSSEC client requests. |
+| hadoop.registry.dns.private-key-file | The path to the standard DNSSEC private key file. Must only be readable by the DNS launching identity. See [dnssec-keygen](https://ftp.isc.org/isc/bind/cur/9.9/doc/arm/man.dnssec-keygen.html) documentation. |
+| hadoop.registry.dns-ttl | The default TTL value to associate with DNS records. The default value is set to 1 (a value of 0 has undefined behavior). A typical value should be approximate to the time it takes YARN to restart a failed container. |
+| hadoop.registry.dns.zone-subnet | An indicator of the IP range associated with the cluster containers. The setting is utilized for the generation of the reverse zone name. |
+| hadoop.registry.dns.zone-mask | The network mask associated with the zone IP range. If specified, it is utilized to ascertain the IP range possible and come up with an appropriate reverse zone name. |
+| hadoop.registry.dns.zones-dir | A directory containing zone configuration files to read during zone initialization. This directory can contain zone master files named *zone-name.zone*. See [here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file documentation.|
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[69/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
new file mode 100644
index 0000000..fb2fd16
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -0,0 +1,654 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.UpdatedContainer;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.TimelineV2Client;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.service.api.ServiceApiConstants;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.service.component.ComponentEvent;
+import org.apache.hadoop.yarn.service.component.ComponentEventType;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
+import org.apache.hadoop.yarn.service.provider.ProviderUtils;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
+import org.apache.hadoop.yarn.util.BoundedAppender;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.text.MessageFormat;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
+import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.*;
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
+
+/**
+ *
+ */
+public class ServiceScheduler extends CompositeService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceScheduler.class);
+ private Service app;
+
+ // component_name -> component
+ private final Map<String, Component> componentsByName =
+ new ConcurrentHashMap<>();
+
+ // id - > component
+ protected final Map<Long, Component> componentsById =
+ new ConcurrentHashMap<>();
+
+ private final Map<ContainerId, ComponentInstance> liveInstances =
+ new ConcurrentHashMap<>();
+
+ private ServiceMetrics serviceMetrics;
+
+ private ServiceTimelinePublisher serviceTimelinePublisher;
+
+ // Global diagnostics that will be reported to RM on eRxit.
+ // The unit the number of characters. This will be limited to 64 * 1024
+ // characters.
+ private BoundedAppender diagnostics = new BoundedAppender(64 * 1024);
+
+ // A cache for loading config files from remote such as hdfs
+ public LoadingCache<ConfigFile, Object> configFileCache = null;
+
+ public ScheduledExecutorService executorService;
+ public Map<String, String> globalTokens = new HashMap<>();
+
+ private AMRMClientAsync<AMRMClient.ContainerRequest> amRMClient;
+ private NMClientAsync nmClient;
+ private AsyncDispatcher dispatcher;
+ AsyncDispatcher compInstanceDispatcher;
+ private YarnRegistryViewForProviders yarnRegistryOperations;
+ private ServiceContext context;
+ private ContainerLaunchService containerLaunchService;
+
+ public ServiceScheduler(ServiceContext context) {
+ super(context.service.getName());
+ this.context = context;
+ }
+
+ public void buildInstance(ServiceContext context, Configuration configuration)
+ throws YarnException {
+ app = context.service;
+ executorService = Executors.newScheduledThreadPool(10);
+ RegistryOperations registryClient = RegistryOperationsFactory
+ .createInstance("ServiceScheduler", configuration);
+ addIfService(registryClient);
+ yarnRegistryOperations =
+ createYarnRegistryOperations(context, registryClient);
+
+ // register metrics
+ serviceMetrics = ServiceMetrics
+ .register(app.getName(), "Metrics for service");
+ serviceMetrics.tag("type", "Metrics type [component or service]", "service");
+ serviceMetrics.tag("appId", "Service id for service", app.getId());
+
+ amRMClient = createAMRMClient();
+ addIfService(amRMClient);
+
+ nmClient = createNMClient();
+ addIfService(nmClient);
+
+ dispatcher = new AsyncDispatcher("Component dispatcher");
+ dispatcher.register(ComponentEventType.class,
+ new ComponentEventHandler());
+ dispatcher.setDrainEventsOnStop();
+ addIfService(dispatcher);
+
+ compInstanceDispatcher =
+ new AsyncDispatcher("CompInstance dispatcher");
+ compInstanceDispatcher.register(ComponentInstanceEventType.class,
+ new ComponentInstanceEventHandler());
+ addIfService(compInstanceDispatcher);
+ containerLaunchService = new ContainerLaunchService(context.fs);
+ addService(containerLaunchService);
+
+ if (YarnConfiguration.timelineServiceV2Enabled(configuration)) {
+ TimelineV2Client timelineClient = TimelineV2Client
+ .createTimelineClient(context.attemptId.getApplicationId());
+ amRMClient.registerTimelineV2Client(timelineClient);
+ serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
+ addService(serviceTimelinePublisher);
+ DefaultMetricsSystem.instance().register("ServiceMetricsSink",
+ "For processing metrics to ATS",
+ new ServiceMetricsSink(serviceTimelinePublisher));
+ LOG.info("Timeline v2 is enabled.");
+ }
+
+ initGlobalTokensForSubstitute(context);
+ //substitute quicklinks
+ ProviderUtils.substituteMapWithTokens(app.getQuicklinks(), globalTokens);
+ createConfigFileCache(context.fs.getFileSystem());
+
+ createAllComponents();
+ }
+
+ protected YarnRegistryViewForProviders createYarnRegistryOperations(
+ ServiceContext context, RegistryOperations registryClient) {
+ return new YarnRegistryViewForProviders(registryClient,
+ RegistryUtils.currentUser(), YarnServiceConstants.APP_TYPE, app.getName(),
+ context.attemptId);
+ }
+
+ protected NMClientAsync createNMClient() {
+ return NMClientAsync.createNMClientAsync(new NMClientCallback());
+ }
+
+ protected AMRMClientAsync<AMRMClient.ContainerRequest> createAMRMClient() {
+ return AMRMClientAsync
+ .createAMRMClientAsync(1000, new AMRMClientCallback());
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ try {
+ buildInstance(context, conf);
+ } catch (YarnException e) {
+ throw new YarnRuntimeException(e);
+ }
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void serviceStop() throws Exception {
+ LOG.info("Stopping service scheduler");
+
+ if (executorService != null) {
+ executorService.shutdownNow();
+ }
+
+ DefaultMetricsSystem.shutdown();
+ if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ serviceTimelinePublisher
+ .serviceAttemptUnregistered(context, diagnostics.toString());
+ }
+ // Cleanup each component instance. no need to release containers as
+ // they will be automatically released by RM
+ for (ComponentInstance instance : liveInstances.values()) {
+ instance.cleanupRegistryAndCompHdfsDir();
+ }
+ String msg = diagnostics.toString()
+ + "Navigate to the failed component for more details.";
+ amRMClient
+ .unregisterApplicationMaster(FinalApplicationStatus.ENDED, msg, "");
+ LOG.info("Service " + app.getName()
+ + " unregistered with RM, with attemptId = " + context.attemptId
+ + ", diagnostics = " + diagnostics);
+ super.serviceStop();
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ super.serviceStart();
+ InetSocketAddress bindAddress = context.clientAMService.getBindAddress();
+ RegisterApplicationMasterResponse response = amRMClient
+ .registerApplicationMaster(bindAddress.getHostName(),
+ bindAddress.getPort(), "N/A");
+ if (response.getClientToAMTokenMasterKey() != null
+ && response.getClientToAMTokenMasterKey().remaining() != 0) {
+ context.secretManager
+ .setMasterKey(response.getClientToAMTokenMasterKey().array());
+ }
+ registerServiceInstance(context.attemptId, app);
+
+ //TODO handle containers recover
+ }
+
+ private void recover() {
+
+ }
+
+ private void initGlobalTokensForSubstitute(ServiceContext context) {
+ // ZK
+ globalTokens.put(ServiceApiConstants.CLUSTER_ZK_QUORUM, getConfig()
+ .getTrimmed(KEY_REGISTRY_ZK_QUORUM, DEFAULT_REGISTRY_ZK_QUORUM));
+ String user = null;
+ try {
+ user = UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ LOG.error("Failed to get user.", e);
+ }
+ globalTokens
+ .put(SERVICE_ZK_PATH, ServiceRegistryUtils.mkClusterPath(user, app.getName()));
+
+ globalTokens.put(ServiceApiConstants.USER, user);
+ String dnsDomain = getConfig().getTrimmed(KEY_DNS_DOMAIN);
+ if (dnsDomain != null && !dnsDomain.isEmpty()) {
+ globalTokens.put(ServiceApiConstants.DOMAIN, dnsDomain);
+ }
+ // HDFS
+ String clusterFs = getConfig().getTrimmed(FS_DEFAULT_NAME_KEY);
+ if (clusterFs != null && !clusterFs.isEmpty()) {
+ globalTokens.put(ServiceApiConstants.CLUSTER_FS_URI, clusterFs);
+ globalTokens.put(ServiceApiConstants.CLUSTER_FS_HOST,
+ URI.create(clusterFs).getHost());
+ }
+ globalTokens.put(SERVICE_HDFS_DIR, context.serviceHdfsDir);
+ // service name
+ globalTokens.put(SERVICE_NAME_LC, app.getName().toLowerCase());
+ globalTokens.put(SERVICE_NAME, app.getName());
+ }
+
+ private void createConfigFileCache(final FileSystem fileSystem) {
+ this.configFileCache =
+ CacheBuilder.newBuilder().expireAfterAccess(10, TimeUnit.MINUTES)
+ .build(new CacheLoader<ConfigFile, Object>() {
+ @Override public Object load(ConfigFile key) throws Exception {
+ switch (key.getType()) {
+ case HADOOP_XML:
+ try (FSDataInputStream input = fileSystem
+ .open(new Path(key.getSrcFile()))) {
+ org.apache.hadoop.conf.Configuration confRead =
+ new org.apache.hadoop.conf.Configuration(false);
+ confRead.addResource(input);
+ Map<String, String> map = new HashMap<>(confRead.size());
+ for (Map.Entry<String, String> entry : confRead) {
+ map.put(entry.getKey(), entry.getValue());
+ }
+ return map;
+ }
+ case TEMPLATE:
+ try (FSDataInputStream fileInput = fileSystem
+ .open(new Path(key.getSrcFile()))) {
+ return IOUtils.toString(fileInput);
+ }
+ default:
+ return null;
+ }
+ }
+ });
+ context.configCache = configFileCache;
+ }
+
+ private void registerServiceInstance(ApplicationAttemptId attemptId,
+ Service service) throws IOException {
+ LOG.info("Registering " + attemptId + ", " + service.getName()
+ + " into registry");
+ ServiceRecord serviceRecord = new ServiceRecord();
+ serviceRecord.set(YarnRegistryAttributes.YARN_ID,
+ attemptId.getApplicationId().toString());
+ serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
+ PersistencePolicies.APPLICATION);
+ serviceRecord.description = "Yarn Service Master";
+
+ serviceRecord.addExternalEndpoint(RegistryTypeUtils
+ .ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc",
+ context.clientAMService.getBindAddress()));
+
+ // set any provided attributes
+ setUserProvidedServiceRecordAttributes(service.getConfiguration(),
+ serviceRecord);
+
+ executorService.submit(new Runnable() {
+ @Override public void run() {
+ try {
+ yarnRegistryOperations.registerSelf(serviceRecord, true);
+ LOG.info("Registered service under {}; absolute path {}",
+ yarnRegistryOperations.getSelfRegistrationPath(),
+ yarnRegistryOperations.getAbsoluteSelfRegistrationPath());
+ boolean isFirstAttempt = 1 == attemptId.getAttemptId();
+ // delete the children in case there are any and this is an AM startup.
+ // just to make sure everything underneath is purged
+ if (isFirstAttempt) {
+ yarnRegistryOperations.deleteChildren(
+ yarnRegistryOperations.getSelfRegistrationPath(), true);
+ }
+ } catch (IOException e) {
+ LOG.error(
+ "Failed to register app " + app.getName() + " in registry");
+ }
+ }
+ });
+ if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ serviceTimelinePublisher.serviceAttemptRegistered(app, getConfig());
+ }
+ }
+
+ private void setUserProvidedServiceRecordAttributes(
+ org.apache.hadoop.yarn.service.api.records.Configuration conf, ServiceRecord record) {
+ String prefix = "service.record.attribute";
+ for (Map.Entry<String, String> entry : conf.getProperties().entrySet()) {
+ if (entry.getKey().startsWith(prefix)) {
+ String key = entry.getKey().substring(prefix.length() + 1);
+ record.set(key, entry.getValue().trim());
+ }
+ }
+ }
+
+ private void createAllComponents() {
+ long allocateId = 0;
+
+ // sort components by dependencies
+ Collection<org.apache.hadoop.yarn.service.api.records.Component> sortedComponents =
+ ServiceApiUtil.sortByDependencies(app.getComponents());
+
+ for (org.apache.hadoop.yarn.service.api.records.Component compSpec : sortedComponents) {
+ Component component = new Component(compSpec, allocateId, context);
+ componentsById.put(allocateId, component);
+ componentsByName.put(component.getName(), component);
+ allocateId++;
+
+ // Trigger the component without dependencies
+ if (component.areDependenciesReady()) {
+ ComponentEvent event = new ComponentEvent(compSpec.getName(), FLEX)
+ .setDesired(compSpec.getNumberOfContainers());
+ component.handle(event);
+ }
+ }
+ }
+
+ private final class ComponentEventHandler
+ implements EventHandler<ComponentEvent> {
+ @Override
+ public void handle(ComponentEvent event) {
+ Component component = componentsByName.get(event.getName());
+
+ if (component == null) {
+ LOG.error("No component exists for " + event.getName());
+ return;
+ }
+ try {
+ component.handle(event);
+ } catch (Throwable t) {
+ LOG.error(MessageFormat
+ .format("[COMPONENT {0}]: Error in handling event type {1}",
+ component.getName(), event.getType()), t);
+ }
+ }
+ }
+
+ private final class ComponentInstanceEventHandler
+ implements EventHandler<ComponentInstanceEvent> {
+ @Override
+ public void handle(ComponentInstanceEvent event) {
+ ComponentInstance instance =
+ liveInstances.get(event.getContainerId());
+ if (instance == null) {
+ LOG.error("No component instance exists for " + event.getContainerId());
+ return;
+ }
+ try {
+ instance.handle(event);
+ } catch (Throwable t) {
+ LOG.error(instance.getCompInstanceId() +
+ ": Error in handling event type " + event.getType(), t);
+ }
+ }
+ }
+
+ class AMRMClientCallback extends AMRMClientAsync.AbstractCallbackHandler {
+
+ @Override
+ public void onContainersAllocated(List<Container> containers) {
+ LOG.info(containers.size() + " containers allocated. ");
+ for (Container container : containers) {
+ Component comp = componentsById.get(container.getAllocationRequestId());
+ ComponentEvent event =
+ new ComponentEvent(comp.getName(), CONTAINER_ALLOCATED)
+ .setContainer(container);
+ dispatcher.getEventHandler().handle(event);
+ LOG.info("[COMPONENT {}]: {} outstanding container requests.",
+ comp.getName(),
+ amRMClient.getMatchingRequests(container.getAllocationRequestId()).size());
+ // remove the corresponding request
+ Collection<AMRMClient.ContainerRequest> collection = amRMClient
+ .getMatchingRequests(container.getAllocationRequestId());
+ if (collection.iterator().hasNext()) {
+ AMRMClient.ContainerRequest request = collection.iterator().next();
+ amRMClient.removeContainerRequest(request);
+ }
+
+ }
+ }
+
+ @Override
+ public void onContainersCompleted(List<ContainerStatus> statuses) {
+ for (ContainerStatus status : statuses) {
+ ContainerId containerId = status.getContainerId();
+ ComponentInstance instance = liveInstances.get(status.getContainerId());
+ if (instance == null) {
+ LOG.error(
+ "Container {} Completed. No component instance exists. exitStatus={}. diagnostics={} ",
+ containerId, status.getExitStatus(), status.getDiagnostics());
+ return;
+ }
+ ComponentEvent event =
+ new ComponentEvent(instance.getCompName(), CONTAINER_COMPLETED)
+ .setStatus(status).setInstance(instance);
+ dispatcher.getEventHandler().handle(event);
+ }
+ }
+
+ @Override
+ public void onContainersUpdated(List<UpdatedContainer> containers) {
+ }
+
+ @Override public void onShutdownRequest() {
+ //Was used for non-work-preserving restart in YARN, should be deprecated.
+ }
+
+ @Override public void onNodesUpdated(List<NodeReport> updatedNodes) {
+ StringBuilder str = new StringBuilder();
+ str.append("Nodes updated info: ").append(System.lineSeparator());
+ for (NodeReport report : updatedNodes) {
+ str.append(report.getNodeId()).append(", state = ")
+ .append(report.getNodeState()).append(", healthDiagnostics = ")
+ .append(report.getHealthReport()).append(System.lineSeparator());
+ }
+ LOG.warn(str.toString());
+ }
+
+ @Override public float getProgress() {
+ // get running containers over desired containers
+ long total = 0;
+ for (org.apache.hadoop.yarn.service.api.records.Component component : app
+ .getComponents()) {
+ total += component.getNumberOfContainers();
+ }
+ // Probably due to user flexed down to 0
+ if (total == 0) {
+ return 100;
+ }
+ return Math.max((float) liveInstances.size() / total * 100, 100);
+ }
+
+ @Override public void onError(Throwable e) {
+ LOG.error("Error in AMRMClient callback handler ", e);
+ }
+ }
+
+
+ private class NMClientCallback extends NMClientAsync.AbstractCallbackHandler {
+
+ @Override public void onContainerStarted(ContainerId containerId,
+ Map<String, ByteBuffer> allServiceResponse) {
+ ComponentInstance instance = liveInstances.get(containerId);
+ if (instance == null) {
+ LOG.error("No component instance exists for " + containerId);
+ return;
+ }
+ ComponentEvent event =
+ new ComponentEvent(instance.getCompName(), CONTAINER_STARTED)
+ .setInstance(instance);
+ dispatcher.getEventHandler().handle(event);
+ }
+
+ @Override public void onContainerStatusReceived(ContainerId containerId,
+ ContainerStatus containerStatus) {
+
+ }
+
+ @Override public void onContainerStopped(ContainerId containerId) {
+
+ }
+
+ @Override
+ public void onStartContainerError(ContainerId containerId, Throwable t) {
+ ComponentInstance instance = liveInstances.get(containerId);
+ if (instance == null) {
+ LOG.error("No component instance exists for " + containerId);
+ return;
+ }
+ amRMClient.releaseAssignedContainer(containerId);
+ // After container released, it'll get CONTAINER_COMPLETED event from RM
+ // automatically which will trigger stopping COMPONENT INSTANCE
+ }
+
+ @Override public void onContainerResourceIncreased(ContainerId containerId,
+ Resource resource) {
+
+ }
+
+ @Override public void onGetContainerStatusError(ContainerId containerId,
+ Throwable t) {
+
+ }
+
+ @Override
+ public void onIncreaseContainerResourceError(ContainerId containerId,
+ Throwable t) {
+
+ }
+
+ @Override
+ public void onStopContainerError(ContainerId containerId, Throwable t) {
+
+ }
+ }
+
+ public ServiceMetrics getServiceMetrics() {
+ return serviceMetrics;
+ }
+
+ public AMRMClientAsync<AMRMClient.ContainerRequest> getAmRMClient() {
+ return amRMClient;
+ }
+
+ public NMClientAsync getNmClient() {
+ return nmClient;
+ }
+
+ public void addLiveCompInstance(ContainerId containerId,
+ ComponentInstance instance) {
+ liveInstances.put(containerId, instance);
+ }
+
+ public void removeLiveCompInstance(ContainerId containerId) {
+ liveInstances.remove(containerId);
+ }
+
+ public AsyncDispatcher getCompInstanceDispatcher() {
+ return compInstanceDispatcher;
+ }
+
+ public YarnRegistryViewForProviders getYarnRegistryOperations() {
+ return yarnRegistryOperations;
+ }
+
+ public ServiceTimelinePublisher getServiceTimelinePublisher() {
+ return serviceTimelinePublisher;
+ }
+
+ public Map<ContainerId, ComponentInstance> getLiveInstances() {
+ return liveInstances;
+ }
+
+ public ContainerLaunchService getContainerLaunchService() {
+ return containerLaunchService;
+ }
+
+ public ServiceContext getContext() {
+ return context;
+ }
+
+ public Map<String, Component> getAllComponents() {
+ return componentsByName;
+ }
+
+ public Service getApp() {
+ return app;
+ }
+
+ public AsyncDispatcher getDispatcher() {
+ return dispatcher;
+ }
+
+ public BoundedAppender getDiagnostics() {
+ return diagnostics;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
new file mode 100644
index 0000000..0bfb220
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
+
+/**
+ * This class defines constants that can be used in input spec for
+ * variable substitutions
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public interface ServiceApiConstants {
+
+ // Constants for service
+ String SERVICE_NAME = $("SERVICE_NAME");
+
+ String SERVICE_NAME_LC = $("SERVICE_NAME.lc");
+
+ String USER = $("USER");
+
+ String DOMAIN = $("DOMAIN");
+
+ // Constants for component
+ String COMPONENT_NAME = $("COMPONENT_NAME");
+
+ String COMPONENT_NAME_LC = $("COMPONENT_NAME.lc");
+
+ String COMPONENT_INSTANCE_NAME = $("COMPONENT_INSTANCE_NAME");
+
+ // Constants for component instance
+ String COMPONENT_ID = $("COMPONENT_ID");
+
+ String CONTAINER_ID = $("CONTAINER_ID");
+
+ // Constants for default cluster ZK
+ String CLUSTER_ZK_QUORUM = $("CLUSTER_ZK_QUORUM");
+
+ // URI for the default cluster fs
+ String CLUSTER_FS_URI = $("CLUSTER_FS_URI");
+
+ // the host component of the cluster fs UI
+ String CLUSTER_FS_HOST = $("CLUSTER_FS_HOST");
+
+ // Path in zookeeper for a specific service
+ String SERVICE_ZK_PATH = $("SERVICE_ZK_PATH");
+
+ // Constants for service specific hdfs dir
+ String SERVICE_HDFS_DIR = $("SERVICE_HDFS_DIR");
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
new file mode 100644
index 0000000..4328c6f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Artifact of an service component.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "Artifact of an service component")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Artifact implements Serializable {
+ private static final long serialVersionUID = 3608929500111099035L;
+
+ private String id = null;
+
+ public enum TypeEnum {
+ DOCKER("DOCKER"), TARBALL("TARBALL"), SERVICE("SERVICE");
+
+ private String value;
+
+ TypeEnum(String value) {
+ this.value = value;
+ }
+
+ @Override
+ @JsonValue
+ public String toString() {
+ return value;
+ }
+ }
+
+ private TypeEnum type = TypeEnum.DOCKER;
+ private String uri = null;
+
+ /**
+ * Artifact id. Examples are package location uri for tarball based services,
+ * image name for docker, etc.
+ **/
+ public Artifact id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", required = true, value = "Artifact id. Examples are package location uri for tarball based services, image name for docker, etc.")
+ @JsonProperty("id")
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
+ * Artifact type, like docker, tarball, etc. (optional).
+ **/
+ public Artifact type(TypeEnum type) {
+ this.type = type;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact type, like docker, tarball, etc. (optional).")
+ @JsonProperty("type")
+ public TypeEnum getType() {
+ return type;
+ }
+
+ public void setType(TypeEnum type) {
+ this.type = type;
+ }
+
+ /**
+ * Artifact location to support multiple artifact stores (optional).
+ **/
+ public Artifact uri(String uri) {
+ this.uri = uri;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact location to support multiple artifact stores (optional).")
+ @JsonProperty("uri")
+ public String getUri() {
+ return uri;
+ }
+
+ public void setUri(String uri) {
+ this.uri = uri;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Artifact artifact = (Artifact) o;
+ return Objects.equals(this.id, artifact.id)
+ && Objects.equals(this.type, artifact.type)
+ && Objects.equals(this.uri, artifact.uri);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, type, uri);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Artifact {\n");
+
+ sb.append(" id: ").append(toIndentedString(id)).append("\n");
+ sb.append(" type: ").append(toIndentedString(type)).append("\n");
+ sb.append(" uri: ").append(toIndentedString(uri)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
new file mode 100644
index 0000000..7ac86d4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.io.Serializable;
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class BaseResource implements Serializable {
+ private static final long serialVersionUID = 1492603053176889431L;
+
+ private String uri;
+
+ /**
+ * Resource location for a service, e.g.
+ * /ws/v1/services/helloworld
+ *
+ **/
+ public String getUri() {
+ return uri;
+ }
+
+ public void setUri(String uri) {
+ this.uri = uri;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("BaseResource [uri=");
+ builder.append(uri);
+ builder.append("]");
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
new file mode 100644
index 0000000..b0a8e82
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
@@ -0,0 +1,412 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * One or more components of the service. If the service is HBase say,
+ * then the component can be a simple role like master or regionserver. If the
+ * service is a complex business webapp then a component can be other
+ * services say Kafka or Storm. Thereby it opens up the support for complex
+ * and nested services.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Component implements Serializable {
+ private static final long serialVersionUID = -8430058381509087805L;
+
+ private String name = null;
+ private List<String> dependencies = new ArrayList<String>();
+ private ReadinessCheck readinessCheck = null;
+ private Artifact artifact = null;
+ private String launchCommand = null;
+ private Resource resource = null;
+ private Long numberOfContainers = null;
+ private Boolean runPrivilegedContainer = false;
+ private PlacementPolicy placementPolicy = null;
+ private Configuration configuration = new Configuration();
+ private List<String> quicklinks = new ArrayList<String>();
+ private List<Container> containers =
+ Collections.synchronizedList(new ArrayList<Container>());
+
+ /**
+ * Name of the service component (mandatory).
+ **/
+ public Component name(String name) {
+ this.name = name;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", required = true, value = "Name of the service component (mandatory).")
+ @JsonProperty("name")
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * An array of service components which should be in READY state (as
+ * defined by readiness check), before this component can be started. The
+ * dependencies across all components of a service should be represented
+ * as a DAG.
+ **/
+ public Component dependencies(List<String> dependencies) {
+ this.dependencies = dependencies;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an service should be represented as a DAG.")
+ @JsonProperty("dependencies")
+ public List<String> getDependencies() {
+ return dependencies;
+ }
+
+ public void setDependencies(List<String> dependencies) {
+ this.dependencies = dependencies;
+ }
+
+ /**
+ * Readiness check for this component.
+ **/
+ public Component readinessCheck(ReadinessCheck readinessCheck) {
+ this.readinessCheck = readinessCheck;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Readiness check for this component.")
+ @JsonProperty("readiness_check")
+ public ReadinessCheck getReadinessCheck() {
+ return readinessCheck;
+ }
+
+ @XmlElement(name = "readiness_check")
+ public void setReadinessCheck(ReadinessCheck readinessCheck) {
+ this.readinessCheck = readinessCheck;
+ }
+
+ /**
+ * Artifact of the component (optional). If not specified, the service
+ * level global artifact takes effect.
+ **/
+ public Component artifact(Artifact artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact of the component (optional). If not specified, the service level global artifact takes effect.")
+ @JsonProperty("artifact")
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void setArtifact(Artifact artifact) {
+ this.artifact = artifact;
+ }
+
+ /**
+ * The custom launch command of this component (optional). When specified at
+ * the component level, it overrides the value specified at the global level
+ * (if any).
+ **/
+ public Component launchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The custom launch command of this component (optional). When specified at the component level, it overrides the value specified at the global level (if any).")
+ @JsonProperty("launch_command")
+ public String getLaunchCommand() {
+ return launchCommand;
+ }
+
+ @XmlElement(name = "launch_command")
+ public void setLaunchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ }
+
+ /**
+ * Resource of this component (optional). If not specified, the service
+ * level global resource takes effect.
+ **/
+ public Component resource(Resource resource) {
+ this.resource = resource;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Resource of this component (optional). If not specified, the service level global resource takes effect.")
+ @JsonProperty("resource")
+ public Resource getResource() {
+ return resource;
+ }
+
+ public void setResource(Resource resource) {
+ this.resource = resource;
+ }
+
+ /**
+ * Number of containers for this component (optional). If not specified,
+ * the service level global number_of_containers takes effect.
+ **/
+ public Component numberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.")
+ @JsonProperty("number_of_containers")
+ public Long getNumberOfContainers() {
+ return numberOfContainers;
+ }
+
+ @XmlElement(name = "number_of_containers")
+ public void setNumberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ }
+
+ @ApiModelProperty(example = "null", value = "Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.")
+ @JsonProperty("containers")
+ public List<Container> getContainers() {
+ return containers;
+ }
+
+ public void setContainers(List<Container> containers) {
+ this.containers = containers;
+ }
+
+ public void addContainer(Container container) {
+ this.containers.add(container);
+ }
+
+ public void removeContainer(Container container) {
+ containers.remove(container);
+ }
+ public Container getContainer(String id) {
+ for (Container container : containers) {
+ if (container.getId().equals(id)) {
+ return container;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Run all containers of this component in privileged mode (YARN-4262).
+ **/
+ public Component runPrivilegedContainer(Boolean runPrivilegedContainer) {
+ this.runPrivilegedContainer = runPrivilegedContainer;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Run all containers of this component in privileged mode (YARN-4262).")
+ @JsonProperty("run_privileged_container")
+ public Boolean getRunPrivilegedContainer() {
+ return runPrivilegedContainer;
+ }
+
+ @XmlElement(name = "run_privileged_container")
+ public void setRunPrivilegedContainer(Boolean runPrivilegedContainer) {
+ this.runPrivilegedContainer = runPrivilegedContainer;
+ }
+
+ /**
+ * Advanced scheduling and placement policies for all containers of this
+ * component (optional). If not specified, the service level placement_policy
+ * takes effect. Refer to the description at the global level for more
+ * details.
+ **/
+ public Component placementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.")
+ @JsonProperty("placement_policy")
+ public PlacementPolicy getPlacementPolicy() {
+ return placementPolicy;
+ }
+
+ @XmlElement(name = "placement_policy")
+ public void setPlacementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ }
+
+ /**
+ * Config properties for this component.
+ **/
+ public Component configuration(Configuration configuration) {
+ this.configuration = configuration;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Config properties for this component.")
+ @JsonProperty("configuration")
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public void setConfiguration(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ /**
+ * A list of quicklink keys defined at the service level, and to be
+ * resolved by this component.
+ **/
+ public Component quicklinks(List<String> quicklinks) {
+ this.quicklinks = quicklinks;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A list of quicklink keys defined at the service level, and to be resolved by this component.")
+ @JsonProperty("quicklinks")
+ public List<String> getQuicklinks() {
+ return quicklinks;
+ }
+
+ public void setQuicklinks(List<String> quicklinks) {
+ this.quicklinks = quicklinks;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Component component = (Component) o;
+ return Objects.equals(this.name, component.name)
+ && Objects.equals(this.dependencies, component.dependencies)
+ && Objects.equals(this.readinessCheck, component.readinessCheck)
+ && Objects.equals(this.artifact, component.artifact)
+ && Objects.equals(this.launchCommand, component.launchCommand)
+ && Objects.equals(this.resource, component.resource)
+ && Objects.equals(this.numberOfContainers, component.numberOfContainers)
+ && Objects.equals(this.runPrivilegedContainer,
+ component.runPrivilegedContainer)
+ && Objects.equals(this.placementPolicy, component.placementPolicy)
+ && Objects.equals(this.configuration, component.configuration)
+ && Objects.equals(this.quicklinks, component.quicklinks);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, dependencies, readinessCheck, artifact,
+ launchCommand, resource, numberOfContainers,
+ runPrivilegedContainer, placementPolicy, configuration, quicklinks);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Component {\n");
+
+ sb.append(" name: ").append(toIndentedString(name)).append("\n");
+ sb.append(" dependencies: ").append(toIndentedString(dependencies))
+ .append("\n");
+ sb.append(" readinessCheck: ").append(toIndentedString(readinessCheck))
+ .append("\n");
+ sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
+ sb.append(" launchCommand: ").append(toIndentedString(launchCommand))
+ .append("\n");
+ sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
+ sb.append(" numberOfContainers: ")
+ .append(toIndentedString(numberOfContainers)).append("\n");
+ sb.append(" containers: ").append(toIndentedString(containers))
+ .append("\n");
+ sb.append(" runPrivilegedContainer: ")
+ .append(toIndentedString(runPrivilegedContainer)).append("\n");
+ sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy))
+ .append("\n");
+ sb.append(" configuration: ").append(toIndentedString(configuration))
+ .append("\n");
+ sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
+ .append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+
+ /**
+ * Merge from another component into this component without overwriting.
+ */
+ public void mergeFrom(Component that) {
+ if (this.getArtifact() == null) {
+ this.setArtifact(that.getArtifact());
+ }
+ if (this.getResource() == null) {
+ this.setResource(that.getResource());
+ }
+ if (this.getNumberOfContainers() == null) {
+ this.setNumberOfContainers(that.getNumberOfContainers());
+ }
+ if (this.getLaunchCommand() == null) {
+ this.setLaunchCommand(that.getLaunchCommand());
+ }
+ this.getConfiguration().mergeFrom(that.getConfiguration());
+ if (this.getQuicklinks() == null) {
+ this.setQuicklinks(that.getQuicklinks());
+ }
+ if (this.getRunPrivilegedContainer() == null) {
+ this.setRunPrivilegedContainer(that.getRunPrivilegedContainer());
+ }
+ if (this.getDependencies() == null) {
+ this.setDependencies(that.getDependencies());
+ }
+ if (this.getPlacementPolicy() == null) {
+ this.setPlacementPolicy(that.getPlacementPolicy());
+ }
+ if (this.getReadinessCheck() == null) {
+ this.setReadinessCheck(that.getReadinessCheck());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
new file mode 100644
index 0000000..c1502c7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * A config file that needs to be created and made available as a volume in an
+ * service component container.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "A config file that needs to be created and made available as a volume in an service component container.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ConfigFile implements Serializable {
+ private static final long serialVersionUID = -7009402089417704612L;
+
+ public enum TypeEnum {
+ XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
+ "TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),;
+
+ private String value;
+
+ TypeEnum(String value) {
+ this.value = value;
+ }
+
+ @Override
+ @JsonValue
+ public String toString() {
+ return value;
+ }
+ }
+
+ private TypeEnum type = null;
+ private String destFile = null;
+ private String srcFile = null;
+ private Map<String, String> props = new HashMap<>();
+
+ public ConfigFile copy() {
+ ConfigFile copy = new ConfigFile();
+ copy.setType(this.getType());
+ copy.setSrcFile(this.getSrcFile());
+ copy.setDestFile(this.getDestFile());
+ if (this.getProps() != null && !this.getProps().isEmpty()) {
+ copy.getProps().putAll(this.getProps());
+ }
+ return copy;
+ }
+
+ /**
+ * Config file in the standard format like xml, properties, json, yaml,
+ * template.
+ **/
+ public ConfigFile type(TypeEnum type) {
+ this.type = type;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Config file in the standard format like xml, properties, json, yaml, template.")
+ @JsonProperty("type")
+ public TypeEnum getType() {
+ return type;
+ }
+
+ public void setType(TypeEnum type) {
+ this.type = type;
+ }
+
+ /**
+ * The absolute path that this configuration file should be mounted as, in the
+ * service container.
+ **/
+ public ConfigFile destFile(String destFile) {
+ this.destFile = destFile;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The absolute path that this configuration file should be mounted as, in the service container.")
+ @JsonProperty("dest_file")
+ public String getDestFile() {
+ return destFile;
+ }
+
+ @XmlElement(name = "dest_file")
+ public void setDestFile(String destFile) {
+ this.destFile = destFile;
+ }
+
+ /**
+ * This provides the source location of the configuration file, the content
+ * of which is dumped to dest_file post property substitutions, in the format
+ * as specified in type. Typically the src_file would point to a source
+ * controlled network accessible file maintained by tools like puppet, chef,
+ * or hdfs etc. Currently, only hdfs is supported.
+ **/
+ public ConfigFile srcFile(String srcFile) {
+ this.srcFile = srcFile;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "This provides the source location of the configuration file, "
+ + "the content of which is dumped to dest_file post property substitutions, in the format as specified in type. "
+ + "Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.")
+ @JsonProperty("src_file")
+ public String getSrcFile() {
+ return srcFile;
+ }
+
+ @XmlElement(name = "src_file")
+ public void setSrcFile(String srcFile) {
+ this.srcFile = srcFile;
+ }
+
+ /**
+ A blob of key value pairs that will be dumped in the dest_file in the format
+ as specified in type. If src_file is specified, src_file content are dumped
+ in the dest_file and these properties will overwrite, if any, existing
+ properties in src_file or be added as new properties in src_file.
+ **/
+ public ConfigFile props(Map<String, String> props) {
+ this.props = props;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be dumped in the dest_file in the format as specified in type."
+ + " If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any,"
+ + " existing properties in src_file or be added as new properties in src_file.")
+ @JsonProperty("props")
+ public Map<String, String> getProps() {
+ return props;
+ }
+
+ public void setProps(Map<String, String> props) {
+ this.props = props;
+ }
+
+ public long getLong(String name, long defaultValue) {
+ if (name == null) {
+ return defaultValue;
+ }
+ String value = props.get(name.trim());
+ return Long.parseLong(value);
+ }
+
+ public boolean getBoolean(String name, boolean defaultValue) {
+ if (name == null) {
+ return defaultValue;
+ }
+ return Boolean.valueOf(props.get(name.trim()));
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ConfigFile configFile = (ConfigFile) o;
+ return Objects.equals(this.type, configFile.type)
+ && Objects.equals(this.destFile, configFile.destFile)
+ && Objects.equals(this.srcFile, configFile.srcFile);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(type, destFile, srcFile, props);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class ConfigFile {\n");
+
+ sb.append(" type: ").append(toIndentedString(type)).append("\n");
+ sb.append(" destFile: ").append(toIndentedString(destFile)).append("\n");
+ sb.append(" srcFile: ").append(toIndentedString(srcFile)).append("\n");
+ sb.append(" props: ").append(toIndentedString(props)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
new file mode 100644
index 0000000..e10305a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.util.Locale;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public enum ConfigFormat {
+
+ JSON("json"),
+ PROPERTIES("properties"),
+ XML("xml"),
+ HADOOP_XML("hadoop_xml"),
+ ENV("env"),
+ TEMPLATE("template"),
+ YAML("yaml"),
+ ;
+ ConfigFormat(String suffix) {
+ this.suffix = suffix;
+ }
+
+ private final String suffix;
+
+ public String getSuffix() {
+ return suffix;
+ }
+
+
+ @Override
+ public String toString() {
+ return suffix;
+ }
+
+ /**
+ * Get a matching format or null
+ * @param type
+ * @return the format
+ */
+ public static ConfigFormat resolve(String type) {
+ for (ConfigFormat format: values()) {
+ if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) {
+ return format;
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
new file mode 100644
index 0000000..2f8ca96
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Set of configuration properties that can be injected into the service
+ * components via envs, files and custom pluggable helper docker containers.
+ * Files of several standard formats like xml, properties, json, yaml and
+ * templates will be supported.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Configuration implements Serializable {
+ private static final long serialVersionUID = -4330788704981074466L;
+
+ private Map<String, String> properties = new HashMap<String, String>();
+ private Map<String, String> env = new HashMap<String, String>();
+ private List<ConfigFile> files = new ArrayList<ConfigFile>();
+
+ /**
+ * A blob of key-value pairs of common service properties.
+ **/
+ public Configuration properties(Map<String, String> properties) {
+ this.properties = properties;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key-value pairs of common service properties.")
+ @JsonProperty("properties")
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Map<String, String> properties) {
+ this.properties = properties;
+ }
+
+ /**
+ * A blob of key-value pairs which will be appended to the default system
+ * properties and handed off to the service at start time. All placeholder
+ * references to properties will be substituted before injection.
+ **/
+ public Configuration env(Map<String, String> env) {
+ this.env = env;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.")
+ @JsonProperty("env")
+ public Map<String, String> getEnv() {
+ return env;
+ }
+
+ public void setEnv(Map<String, String> env) {
+ this.env = env;
+ }
+
+ /**
+ * Array of list of files that needs to be created and made available as
+ * volumes in the service component containers.
+ **/
+ public Configuration files(List<ConfigFile> files) {
+ this.files = files;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Array of list of files that needs to be created and made available as volumes in the service component containers.")
+ @JsonProperty("files")
+ public List<ConfigFile> getFiles() {
+ return files;
+ }
+
+ public void setFiles(List<ConfigFile> files) {
+ this.files = files;
+ }
+
+ public long getPropertyLong(String name, long defaultValue) {
+ String value = getProperty(name);
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ }
+ return Long.parseLong(value);
+ }
+
+ public int getPropertyInt(String name, int defaultValue) {
+ String value = getProperty(name);
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ }
+ return Integer.parseInt(value);
+ }
+
+ public boolean getPropertyBool(String name, boolean defaultValue) {
+ String value = getProperty(name);
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ }
+ return Boolean.parseBoolean(value);
+ }
+
+ public String getProperty(String name, String defaultValue) {
+ String value = getProperty(name);
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ }
+ return value;
+ }
+
+ public void setProperty(String name, String value) {
+ properties.put(name, value);
+ }
+
+ public String getProperty(String name) {
+ return properties.get(name.trim());
+ }
+
+ public String getEnv(String name) {
+ return env.get(name.trim());
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Configuration configuration = (Configuration) o;
+ return Objects.equals(this.properties, configuration.properties)
+ && Objects.equals(this.env, configuration.env)
+ && Objects.equals(this.files, configuration.files);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(properties, env, files);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Configuration {\n");
+
+ sb.append(" properties: ").append(toIndentedString(properties))
+ .append("\n");
+ sb.append(" env: ").append(toIndentedString(env)).append("\n");
+ sb.append(" files: ").append(toIndentedString(files)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+
+ /**
+ * Merge all properties and envs from that configuration to this configration.
+ * For ConfigFiles, all properties and envs of that ConfigFile are merged into
+ * this ConfigFile.
+ */
+ public synchronized void mergeFrom(Configuration that) {
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
+ .getProperties());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
+
+ Map<String, ConfigFile> thatMap = new HashMap<>();
+ for (ConfigFile file : that.getFiles()) {
+ thatMap.put(file.getDestFile(), file.copy());
+ }
+ for (ConfigFile thisFile : files) {
+ if(thatMap.containsKey(thisFile.getDestFile())) {
+ ConfigFile thatFile = thatMap.get(thisFile.getDestFile());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProps(),
+ thatFile.getProps());
+ thatMap.remove(thisFile.getDestFile());
+ }
+ }
+ // add remaining new files from that Configration
+ for (ConfigFile thatFile : thatMap.values()) {
+ files.add(thatFile.copy());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
new file mode 100644
index 0000000..cf8cd79
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Date;
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * An instance of a running service container.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "An instance of a running service container")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Container extends BaseResource {
+ private static final long serialVersionUID = -8955788064529288L;
+
+ private String id = null;
+ private Date launchTime = null;
+ private String ip = null;
+ private String hostname = null;
+ private String bareHost = null;
+ private ContainerState state = null;
+ private String componentName = null;
+ private Resource resource = null;
+ private Artifact artifact = null;
+ private Boolean privilegedContainer = null;
+
+ /**
+ * Unique container id of a running service, e.g.
+ * container_e3751_1458061340047_0008_01_000002.
+ **/
+ public Container id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002.")
+ @JsonProperty("id")
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
+ * The time when the container was created, e.g. 2016-03-16T01:01:49.000Z.
+ * This will most likely be different from cluster launch time.
+ **/
+ public Container launchTime(Date launchTime) {
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.")
+ @JsonProperty("launch_time")
+ public Date getLaunchTime() {
+ return launchTime == null ? null : (Date) launchTime.clone();
+ }
+
+ @XmlElement(name = "launch_time")
+ public void setLaunchTime(Date launchTime) {
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
+ }
+
+ /**
+ * IP address of a running container, e.g. 172.31.42.141. The IP address and
+ * hostname attribute values are dependent on the cluster/docker network setup
+ * as per YARN-4007.
+ **/
+ public Container ip(String ip) {
+ this.ip = ip;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.")
+ @JsonProperty("ip")
+ public String getIp() {
+ return ip;
+ }
+
+ public void setIp(String ip) {
+ this.ip = ip;
+ }
+
+ /**
+ * Fully qualified hostname of a running container, e.g.
+ * ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and
+ * hostname attribute values are dependent on the cluster/docker network setup
+ * as per YARN-4007.
+ **/
+ public Container hostname(String hostname) {
+ this.hostname = hostname;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.")
+ @JsonProperty("hostname")
+ public String getHostname() {
+ return hostname;
+ }
+
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
+
+ /**
+ * The bare node or host in which the container is running, e.g.
+ * cn008.example.com.
+ **/
+ public Container bareHost(String bareHost) {
+ this.bareHost = bareHost;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The bare node or host in which the container is running, e.g. cn008.example.com.")
+ @JsonProperty("bare_host")
+ public String getBareHost() {
+ return bareHost;
+ }
+
+ @XmlElement(name = "bare_host")
+ public void setBareHost(String bareHost) {
+ this.bareHost = bareHost;
+ }
+
+ /**
+ * State of the container of an service.
+ **/
+ public Container state(ContainerState state) {
+ this.state = state;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "State of the container of an service.")
+ @JsonProperty("state")
+ public ContainerState getState() {
+ return state;
+ }
+
+ public void setState(ContainerState state) {
+ this.state = state;
+ }
+
+ /**
+ * Name of the component that this container instance belongs to.
+ **/
+ public Container componentName(String componentName) {
+ this.componentName = componentName;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Name of the component that this container instance belongs to.")
+ @JsonProperty("component_name")
+ public String getComponentName() {
+ return componentName;
+ }
+
+ @XmlElement(name = "component_name")
+ public void setComponentName(String componentName) {
+ this.componentName = componentName;
+ }
+
+ /**
+ * Resource used for this container.
+ **/
+ public Container resource(Resource resource) {
+ this.resource = resource;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Resource used for this container.")
+ @JsonProperty("resource")
+ public Resource getResource() {
+ return resource;
+ }
+
+ public void setResource(Resource resource) {
+ this.resource = resource;
+ }
+
+ /**
+ * Artifact used for this container.
+ **/
+ public Container artifact(Artifact artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact used for this container.")
+ @JsonProperty("artifact")
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void setArtifact(Artifact artifact) {
+ this.artifact = artifact;
+ }
+
+ /**
+ * Container running in privileged mode or not.
+ **/
+ public Container privilegedContainer(Boolean privilegedContainer) {
+ this.privilegedContainer = privilegedContainer;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Container running in privileged mode or not.")
+ @JsonProperty("privileged_container")
+ public Boolean getPrivilegedContainer() {
+ return privilegedContainer;
+ }
+
+ public void setPrivilegedContainer(Boolean privilegedContainer) {
+ this.privilegedContainer = privilegedContainer;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Container container = (Container) o;
+ return Objects.equals(this.id, container.id);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Container {\n");
+
+ sb.append(" id: ").append(toIndentedString(id)).append("\n");
+ sb.append(" launchTime: ").append(toIndentedString(launchTime))
+ .append("\n");
+ sb.append(" ip: ").append(toIndentedString(ip)).append("\n");
+ sb.append(" hostname: ").append(toIndentedString(hostname)).append("\n");
+ sb.append(" bareHost: ").append(toIndentedString(bareHost)).append("\n");
+ sb.append(" state: ").append(toIndentedString(state)).append("\n");
+ sb.append(" componentName: ").append(toIndentedString(componentName))
+ .append("\n");
+ sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
+ sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
+ sb.append(" privilegedContainer: ")
+ .append(toIndentedString(privilegedContainer)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
new file mode 100644
index 0000000..bf09ff2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * The current state of the container of an application.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public enum ContainerState {
+ RUNNING_BUT_UNREADY, READY, STOPPED
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[55/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
deleted file mode 100644
index 1049698..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ /dev/null
@@ -1,872 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.RetryNTimes;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
-import org.apache.hadoop.yarn.service.ServiceMaster;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.client.params.CommonArgs;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.provider.ProviderUtils;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.hadoop.yarn.util.Times;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
-import org.apache.hadoop.yarn.service.containerlaunch.JavaCommandLineBuilder;
-import org.apache.hadoop.yarn.service.utils.ZookeeperUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.text.MessageFormat;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_CREATE;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_FLEX;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_QUEUE;
-import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
-
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class ServiceClient extends CompositeService
- implements SliderExitCodes, YarnServiceConstants {
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceClient.class);
- private SliderFileSystem fs;
- private YarnClient yarnClient;
- // Avoid looking up applicationId from fs all the time.
- private Map<String, ApplicationId> cachedAppIds = new ConcurrentHashMap<>();
- private Map<String, ClientAMProtocol> cachedAMProxies = new ConcurrentHashMap<>();
-
- private RegistryOperations registryClient;
- private CuratorFramework curatorClient;
- private YarnRPC rpc;
-
- private static EnumSet<YarnApplicationState> terminatedStates =
- EnumSet.of(FINISHED, FAILED, KILLED);
- private static EnumSet<YarnApplicationState> liveStates =
- EnumSet.of(NEW, NEW_SAVING, SUBMITTED, RUNNING);
-
- public ServiceClient() {
- super(ServiceClient.class.getName());
- }
-
- @Override protected void serviceInit(Configuration configuration)
- throws Exception {
- fs = new SliderFileSystem(configuration);
- yarnClient = YarnClient.createYarnClient();
- rpc = YarnRPC.create(configuration);
- addService(yarnClient);
- super.serviceInit(configuration);
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (registryClient != null) {
- registryClient.stop();
- }
- super.serviceStop();
- }
-
- private Application loadAppJsonFromLocalFS(
- AbstractClusterBuildingActionArgs args) throws IOException {
- File file = args.getAppDef();
- Path filePath = new Path(file.getAbsolutePath());
- LOG.info("Loading app json from: " + filePath);
- Application application = ServiceApiUtil.jsonSerDeser
- .load(FileSystem.getLocal(getConfig()), filePath);
- if (args.lifetime > 0) {
- application.setLifetime(args.lifetime);
- }
- application.setName(args.getClusterName());
- return application;
- }
-
- public int actionBuild(AbstractClusterBuildingActionArgs args)
- throws IOException, YarnException {
- return actionBuild(loadAppJsonFromLocalFS(args));
- }
-
- public int actionBuild(Application application)
- throws YarnException, IOException {
- Path appDir = checkAppNotExistOnHdfs(application);
- ServiceApiUtil.validateAndResolveApplication(application, fs, getConfig());
- createDirAndPersistApp(appDir, application);
- return EXIT_SUCCESS;
- }
-
- public int actionCreate(AbstractClusterBuildingActionArgs args)
- throws IOException, YarnException {
- actionCreate(loadAppJsonFromLocalFS(args));
- return EXIT_SUCCESS;
- }
-
- public ApplicationId actionCreate(Application application)
- throws IOException, YarnException {
- String appName = application.getName();
- validateClusterName(appName);
- ServiceApiUtil.validateAndResolveApplication(application, fs, getConfig());
- verifyNoLiveAppInRM(appName, "create");
- Path appDir = checkAppNotExistOnHdfs(application);
-
- // Write the definition first and then submit - AM will read the definition
- createDirAndPersistApp(appDir, application);
- ApplicationId appId = submitApp(application);
- cachedAppIds.put(appName, appId);
- application.setId(appId.toString());
- // update app definition with appId
- persistAppDef(appDir, application);
- return appId;
- }
-
- // Called by ServiceCLI
- protected int actionFlexByCLI(ClientArgs args)
- throws YarnException, IOException {
- ActionFlexArgs flexArgs = args.getActionFlexArgs();
- Map<String, Long> componentCounts =
- new HashMap<>(flexArgs.getComponentMap().size());
- Application persistedApp =
- ServiceApiUtil.loadApplication(fs, flexArgs.getClusterName());
- if (!StringUtils.isEmpty(persistedApp.getId())) {
- cachedAppIds.put(persistedApp.getName(),
- ApplicationId.fromString(persistedApp.getId()));
- }
- for (Map.Entry<String, String> entry : flexArgs.getComponentMap()
- .entrySet()) {
- String compName = entry.getKey();
- ServiceApiUtil.validateCompName(compName);
- Component component = persistedApp.getComponent(compName);
- if (component == null) {
- throw new IllegalArgumentException(entry.getKey() + " does not exist !");
- }
- long numberOfContainers =
- parseNumberOfContainers(component, entry.getValue());
- componentCounts.put(compName, numberOfContainers);
- }
- // throw usage exception if no changes proposed
- if (componentCounts.size() == 0) {
- actionHelp(ACTION_FLEX, args);
- }
- flexComponents(args.getClusterName(), componentCounts, persistedApp);
- return EXIT_SUCCESS;
- }
-
- // Parse the number of containers requested by user, e.g.
- // +5 means add 5 additional containers
- // -5 means reduce 5 containers, if it goes to negative, sets it to 0
- // 5 means sets it to 5 containers.
- private long parseNumberOfContainers(Component component, String newNumber) {
-
- long orig = component.getNumberOfContainers();
- if (newNumber.startsWith("+")) {
- return orig + Long.parseLong(newNumber.substring(1));
- } else if (newNumber.startsWith("-")) {
- long ret = orig - Long.parseLong(newNumber.substring(1));
- if (ret < 0) {
- LOG.warn(MessageFormat.format(
- "[COMPONENT {}]: component count goes to negative ({}{} = {}), reset it to 0.",
- component.getName(), orig, newNumber, ret));
- ret = 0;
- }
- return ret;
- } else {
- return Long.parseLong(newNumber);
- }
- }
-
- // Called by Rest Service
- public Map<String, Long> flexByRestService(String appName,
- Map<String, Long> componentCounts) throws YarnException, IOException {
- // load app definition
- Application persistedApp = ServiceApiUtil.loadApplication(fs, appName);
- cachedAppIds.put(persistedApp.getName(),
- ApplicationId.fromString(persistedApp.getId()));
- return flexComponents(appName, componentCounts, persistedApp);
- }
-
- private Map<String, Long> flexComponents(String appName,
- Map<String, Long> componentCounts, Application persistedApp)
- throws YarnException, IOException {
- validateClusterName(appName);
-
- Map<String, Long> original = new HashMap<>(componentCounts.size());
-
- ComponentCountProto.Builder countBuilder = ComponentCountProto.newBuilder();
- FlexComponentsRequestProto.Builder requestBuilder =
- FlexComponentsRequestProto.newBuilder();
-
- for (Component persistedComp : persistedApp.getComponents()) {
- String name = persistedComp.getName();
- if (componentCounts.containsKey(persistedComp.getName())) {
- original.put(name, persistedComp.getNumberOfContainers());
- persistedComp.setNumberOfContainers(componentCounts.get(name));
-
- // build the request
- countBuilder.setName(persistedComp.getName())
- .setNumberOfContainers(persistedComp.getNumberOfContainers());
- requestBuilder.addComponents(countBuilder.build());
- }
- }
- if (original.size() < componentCounts.size()) {
- componentCounts.keySet().removeAll(original.keySet());
- throw new YarnException("Components " + componentCounts.keySet()
- + " do not exist in app definition.");
- }
- ServiceApiUtil.jsonSerDeser
- .save(fs.getFileSystem(), ServiceApiUtil.getAppJsonPath(fs, appName),
- persistedApp, true);
- ClientAMProtocol proxy = getAMProxy(appName);
- if (proxy == null) {
- String message = appName + " is not running";
- LOG.error(message);
- throw new YarnException(message);
- }
- proxy.flexComponents(requestBuilder.build());
- for (Map.Entry<String, Long> entry : original.entrySet()) {
- LOG.info("[COMPONENT {}]: number of containers changed from {} to {}",
- entry.getKey(), entry.getValue(),
- componentCounts.get(entry.getKey()));
- }
- return original;
- }
-
- public int actionStop(String appName, boolean waitForAppStopped)
- throws YarnException, IOException {
- validateClusterName(appName);
- getAppId(appName);
- ApplicationId currentAppId = cachedAppIds.get(appName);
- ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
- if (terminatedStates.contains(report.getYarnApplicationState())) {
- LOG.info("Application {} is already in a terminated state {}", appName,
- report.getYarnApplicationState());
- return EXIT_SUCCESS;
- }
- LOG.info("Stopping application {}, with appId = {}", appName, currentAppId);
- try {
- ClientAMProtocol proxy = getAMProxy(appName, report);
- cachedAppIds.remove(appName);
- cachedAMProxies.remove(appName);
- if (proxy != null) {
- // try to stop the app gracefully.
- StopRequestProto request = StopRequestProto.newBuilder().build();
- proxy.stop(request);
- LOG.info("Application " + appName + " is being gracefully stopped...");
- } else {
- yarnClient.killApplication(currentAppId,
- appName + " is forcefully killed by user!");
- LOG.info("Forcefully kill the application: " + appName);
- return EXIT_SUCCESS;
- }
-
- if (!waitForAppStopped) {
- return EXIT_SUCCESS;
- }
- // Wait until the app is killed.
- long startTime = System.currentTimeMillis();
- int pollCount = 0;
- while (true) {
- Thread.sleep(2000);
- report = yarnClient.getApplicationReport(currentAppId);
- if (terminatedStates.contains(report.getYarnApplicationState())) {
- LOG.info("Application " + appName + " is stopped.");
- break;
- }
- // Forcefully kill after 10 seconds.
- if ((System.currentTimeMillis() - startTime) > 10000) {
- LOG.info("Stop operation timeout stopping, forcefully kill the app "
- + appName);
- yarnClient.killApplication(currentAppId,
- "Forcefully kill the app by user");
- break;
- }
- if (++pollCount % 10 == 0) {
- LOG.info("Waiting for application " + appName + " to be stopped.");
- }
- }
- } catch (IOException | YarnException | InterruptedException e) {
- LOG.info("Failed to stop " + appName
- + " gracefully, forcefully kill the app.");
- yarnClient.killApplication(currentAppId, "Forcefully kill the app");
- }
- return EXIT_SUCCESS;
- }
-
- public int actionDestroy(String appName) throws Exception {
- validateClusterName(appName);
- verifyNoLiveAppInRM(appName, "Destroy");
- Path appDir = fs.buildClusterDirPath(appName);
- FileSystem fileSystem = fs.getFileSystem();
- // remove from the appId cache
- cachedAppIds.remove(appName);
- cachedAMProxies.remove(appName);
- if (fileSystem.exists(appDir)) {
- if (fileSystem.delete(appDir, true)) {
- LOG.info("Successfully deleted application dir for " + appName + ": "
- + appDir);
- } else {
- String message =
- "Failed to delete application + " + appName + " at: " + appDir;
- LOG.info(message);
- throw new YarnException(message);
- }
- }
- deleteZKNode(appName);
- String registryPath = ServiceRegistryUtils.registryPathForInstance(appName);
- try {
- getRegistryClient().delete(registryPath, true);
- } catch (IOException e) {
- LOG.warn("Error deleting registry entry {}", registryPath, e);
- }
- LOG.info("Destroyed cluster {}", appName);
- return EXIT_SUCCESS;
- }
-
- private synchronized RegistryOperations getRegistryClient()
- throws SliderException, IOException {
-
- if (registryClient == null) {
- registryClient =
- RegistryOperationsFactory.createInstance("ServiceClient", getConfig());
- registryClient.init(getConfig());
- registryClient.start();
- }
- return registryClient;
- }
-
- private void deleteZKNode(String clusterName) throws Exception {
- CuratorFramework curatorFramework = getCuratorClient();
- String user = RegistryUtils.currentUser();
- String zkPath = ServiceRegistryUtils.mkClusterPath(user, clusterName);
- if (curatorFramework.checkExists().forPath(zkPath) != null) {
- curatorFramework.delete().deletingChildrenIfNeeded().forPath(zkPath);
- LOG.info("Deleted zookeeper path: " + zkPath);
- }
- }
-
- private synchronized CuratorFramework getCuratorClient()
- throws BadConfigException {
- String registryQuorum =
- getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
-
- // though if neither is set: trouble
- if (SliderUtils.isUnset(registryQuorum)) {
- throw new BadConfigException(
- "No Zookeeper quorum provided in the" + " configuration property "
- + RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
- }
- ZookeeperUtils.splitToHostsAndPortsStrictly(registryQuorum);
-
- if (curatorClient == null) {
- curatorClient =
- CuratorFrameworkFactory.builder().connectString(registryQuorum)
- .sessionTimeoutMs(10000).retryPolicy(new RetryNTimes(5, 2000))
- .build();
- curatorClient.start();
- }
- return curatorClient;
- }
-
- private int actionHelp(String actionName, CommonArgs args)
- throws YarnException, IOException {
- throw new UsageException(CommonArgs.usage(args, actionName));
- }
-
- private void verifyNoLiveAppInRM(String appname, String action)
- throws IOException, YarnException {
- Set<String> types = new HashSet<>(1);
- types.add(YarnServiceConstants.APP_TYPE);
- Set<String> tags = null;
- if (appname != null) {
- tags = Collections.singleton(SliderUtils.createNameTag(appname));
- }
- GetApplicationsRequest request = GetApplicationsRequest.newInstance();
- request.setApplicationTypes(types);
- request.setApplicationTags(tags);
- request.setApplicationStates(liveStates);
- List<ApplicationReport> reports = yarnClient.getApplications(request);
- if (!reports.isEmpty()) {
- throw new YarnException(
- "Failed to " + action + " application, as " + appname
- + " already exists.");
- }
- }
-
- private ApplicationId submitApp(Application app)
- throws IOException, YarnException {
- String appName = app.getName();
- Configuration conf = getConfig();
- Path appRootDir = fs.buildClusterDirPath(app.getName());
-
- YarnClientApplication yarnApp = yarnClient.createApplication();
- ApplicationSubmissionContext submissionContext =
- yarnApp.getApplicationSubmissionContext();
- ServiceApiUtil.validateCompResourceSize(
- yarnApp.getNewApplicationResponse().getMaximumResourceCapability(),
- app);
-
- submissionContext.setKeepContainersAcrossApplicationAttempts(true);
- if (app.getLifetime() > 0) {
- Map<ApplicationTimeoutType, Long> appTimeout = new HashMap<>();
- appTimeout.put(ApplicationTimeoutType.LIFETIME, app.getLifetime());
- submissionContext.setApplicationTimeouts(appTimeout);
- }
- submissionContext.setMaxAppAttempts(conf.getInt(
- YarnServiceConf.AM_RESTART_MAX, 2));
-
- Map<String, LocalResource> localResources = new HashMap<>();
-
- // copy local slideram-log4j.properties to hdfs and add to localResources
- boolean hasAMLog4j =
- addAMLog4jResource(appName, conf, localResources);
- // copy jars to hdfs and add to localResources
- addJarResource(appName, localResources);
- // add keytab if in secure env
- addKeytabResourceIfSecure(fs, localResources, conf, appName);
- if (LOG.isDebugEnabled()) {
- printLocalResources(localResources);
- }
- Map<String, String> env = addAMEnv(conf);
-
- // create AM CLI
- String cmdStr =
- buildCommandLine(appName, conf, appRootDir, hasAMLog4j);
- submissionContext.setResource(Resource.newInstance(YarnServiceConf
- .getLong(YarnServiceConf.AM_RESOURCE_MEM, YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM,
- app.getConfiguration(), conf), 1));
- String queue = app.getQueue();
- if (StringUtils.isEmpty(queue)) {
- queue = conf.get(YARN_QUEUE, "default");
- }
- submissionContext.setQueue(queue);
- submissionContext.setApplicationName(appName);
- submissionContext.setApplicationType(YarnServiceConstants.APP_TYPE);
- Set<String> appTags =
- AbstractClientProvider.createApplicationTags(appName, null, null);
- if (!appTags.isEmpty()) {
- submissionContext.setApplicationTags(appTags);
- }
- ContainerLaunchContext amLaunchContext =
- Records.newRecord(ContainerLaunchContext.class);
- amLaunchContext.setCommands(Collections.singletonList(cmdStr));
- amLaunchContext.setEnvironment(env);
- amLaunchContext.setLocalResources(localResources);
- submissionContext.setAMContainerSpec(amLaunchContext);
- yarnClient.submitApplication(submissionContext);
- return submissionContext.getApplicationId();
- }
-
- private void printLocalResources(Map<String, LocalResource> map) {
- LOG.debug("Added LocalResource for localization: ");
- StringBuilder builder = new StringBuilder();
- for (Map.Entry<String, LocalResource> entry : map.entrySet()) {
- builder.append(entry.getKey()).append(" -> ")
- .append(entry.getValue().getResource().getFile())
- .append(System.lineSeparator());
- }
- LOG.debug(builder.toString());
- }
-
- private String buildCommandLine(String appName, Configuration conf,
- Path appRootDir, boolean hasSliderAMLog4j) throws BadConfigException {
- JavaCommandLineBuilder CLI = new JavaCommandLineBuilder();
- CLI.forceIPv4().headless();
- //TODO CLI.setJVMHeap
- //TODO CLI.addJVMOPTS
- if (hasSliderAMLog4j) {
- CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, YARN_SERVICE_LOG4J_FILENAME);
- CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
- }
- CLI.add(ServiceMaster.class.getCanonicalName());
- CLI.add(ACTION_CREATE, appName);
- //TODO debugAM CLI.add(Arguments.ARG_DEBUG)
- CLI.add(Arguments.ARG_CLUSTER_URI, new Path(appRootDir, appName + ".json"));
- // pass the registry binding
- CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
- RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
- CLI.addMandatoryConfOption(conf, RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
-
- // write out the path output
- CLI.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
- String cmdStr = CLI.build();
- LOG.info("AM launch command: {}", cmdStr);
- return cmdStr;
- }
-
- private Map<String, String> addAMEnv(Configuration conf) throws IOException {
- Map<String, String> env = new HashMap<>();
- ClasspathConstructor classpath =
- buildClasspath(YarnServiceConstants.SUBMITTED_CONF_DIR, "lib", fs, getConfig()
- .getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false));
- env.put("CLASSPATH", classpath.buildClasspath());
- env.put("LANG", "en_US.UTF-8");
- env.put("LC_ALL", "en_US.UTF-8");
- env.put("LANGUAGE", "en_US.UTF-8");
- String jaas = System.getenv("HADOOP_JAAS_DEBUG");
- if (jaas != null) {
- env.put("HADOOP_JAAS_DEBUG", jaas);
- }
- if (!UserGroupInformation.isSecurityEnabled()) {
- String userName = UserGroupInformation.getCurrentUser().getUserName();
- LOG.info("Run as user " + userName);
- // HADOOP_USER_NAME env is used by UserGroupInformation when log in
- // This env makes AM run as this user
- env.put("HADOOP_USER_NAME", userName);
- }
- LOG.info("AM env: \n{}", stringifyMap(env));
- return env;
- }
-
- protected Path addJarResource(String appName,
- Map<String, LocalResource> localResources)
- throws IOException, SliderException {
- Path libPath = fs.buildClusterDirPath(appName);
- ProviderUtils
- .addProviderJar(localResources, ServiceMaster.class, SERVICE_CORE_JAR, fs,
- libPath, "lib", false);
- Path dependencyLibTarGzip = fs.getDependencyTarGzip();
- if (fs.isFile(dependencyLibTarGzip)) {
- LOG.info("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
- + dependencyLibTarGzip);
- SliderUtils.putAmTarGzipAndUpdate(localResources, fs);
- } else {
- String[] libs = SliderUtils.getLibDirs();
- for (String libDirProp : libs) {
- ProviderUtils.addAllDependencyJars(localResources, fs, libPath, "lib",
- libDirProp);
- }
- }
- return libPath;
- }
-
- private boolean addAMLog4jResource(String appName, Configuration conf,
- Map<String, LocalResource> localResources)
- throws IOException, BadClusterStateException {
- boolean hasAMLog4j = false;
- String hadoopConfDir =
- System.getenv(ApplicationConstants.Environment.HADOOP_CONF_DIR.name());
- if (hadoopConfDir != null) {
- File localFile =
- new File(hadoopConfDir, YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
- if (localFile.exists()) {
- Path localFilePath = createLocalPath(localFile);
- Path appDirPath = fs.buildClusterDirPath(appName);
- Path remoteConfPath =
- new Path(appDirPath, YarnServiceConstants.SUBMITTED_CONF_DIR);
- Path remoteFilePath =
- new Path(remoteConfPath, YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
- copy(conf, localFilePath, remoteFilePath);
- LocalResource localResource =
- fs.createAmResource(remoteConfPath, LocalResourceType.FILE);
- localResources.put(localFilePath.getName(), localResource);
- hasAMLog4j = true;
- } else {
- LOG.warn("AM log4j property file doesn't exist: " + localFile);
- }
- }
- return hasAMLog4j;
- }
-
- public int actionStart(String appName) throws YarnException, IOException {
- validateClusterName(appName);
- Path appDir = checkAppExistOnHdfs(appName);
- Application application = ServiceApiUtil.loadApplication(fs, appName);
- ServiceApiUtil.validateAndResolveApplication(application, fs, getConfig());
- // see if it is actually running and bail out;
- verifyNoLiveAppInRM(appName, "thaw");
- ApplicationId appId = submitApp(application);
- application.setId(appId.toString());
- // write app definition on to hdfs
- createDirAndPersistApp(appDir, application);
- return 0;
- }
-
- private Path checkAppNotExistOnHdfs(Application application)
- throws IOException, SliderException {
- Path appDir = fs.buildClusterDirPath(application.getName());
- fs.verifyDirectoryNonexistent(
- new Path(appDir, application.getName() + ".json"));
- return appDir;
- }
-
- private Path checkAppExistOnHdfs(String appName)
- throws IOException, SliderException {
- Path appDir = fs.buildClusterDirPath(appName);
- fs.verifyPathExists(new Path(appDir, appName + ".json"));
- return appDir;
- }
-
- private void createDirAndPersistApp(Path appDir, Application application)
- throws IOException, SliderException {
- FsPermission appDirPermission = new FsPermission("750");
- fs.createWithPermissions(appDir, appDirPermission);
- persistAppDef(appDir, application);
- }
-
- private void persistAppDef(Path appDir, Application application)
- throws IOException {
- Path appJson = new Path(appDir, application.getName() + ".json");
- ServiceApiUtil.jsonSerDeser
- .save(fs.getFileSystem(), appJson, application, true);
- LOG.info(
- "Persisted application " + application.getName() + " at " + appJson);
- }
-
- private void addKeytabResourceIfSecure(SliderFileSystem fileSystem,
- Map<String, LocalResource> localResource, Configuration conf,
- String appName) throws IOException, BadConfigException {
- if (!UserGroupInformation.isSecurityEnabled()) {
- return;
- }
- String keytabPreInstalledOnHost =
- conf.get(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
- if (StringUtils.isEmpty(keytabPreInstalledOnHost)) {
- String amKeytabName =
- conf.get(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
- String keytabDir = conf.get(YarnServiceConf.KEY_HDFS_KEYTAB_DIR);
- Path keytabPath =
- fileSystem.buildKeytabPath(keytabDir, amKeytabName, appName);
- if (fileSystem.getFileSystem().exists(keytabPath)) {
- LocalResource keytabRes =
- fileSystem.createAmResource(keytabPath, LocalResourceType.FILE);
- localResource
- .put(YarnServiceConstants.KEYTAB_DIR + "/" + amKeytabName, keytabRes);
- LOG.info("Adding AM keytab on hdfs: " + keytabPath);
- } else {
- LOG.warn("No keytab file was found at {}.", keytabPath);
- if (conf.getBoolean(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) {
- throw new BadConfigException("No keytab file was found at %s.",
- keytabPath);
- } else {
- LOG.warn("The AM will be "
- + "started without a kerberos authenticated identity. "
- + "The application is therefore not guaranteed to remain "
- + "operational beyond 24 hours.");
- }
- }
- }
- }
-
- public String updateLifetime(String appName, long lifetime)
- throws YarnException, IOException {
- getAppId(appName);
- ApplicationId currentAppId = cachedAppIds.get(appName);
- ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
- if (report == null) {
- throw new YarnException("Application not found for " + appName);
- }
- ApplicationId appId = report.getApplicationId();
- LOG.info("Updating lifetime of an application: appName = " + appName
- + ", appId = " + appId + ", lifetime = " + lifetime);
- Map<ApplicationTimeoutType, String> map = new HashMap<>();
- String newTimeout =
- Times.formatISO8601(System.currentTimeMillis() + lifetime * 1000);
- map.put(ApplicationTimeoutType.LIFETIME, newTimeout);
- UpdateApplicationTimeoutsRequest request =
- UpdateApplicationTimeoutsRequest.newInstance(appId, map);
- yarnClient.updateApplicationTimeouts(request);
- LOG.info(
- "Successfully updated lifetime for an application: appName = " + appName
- + ", appId = " + appId + ". New expiry time in ISO8601 format is "
- + newTimeout);
- return newTimeout;
- }
-
- public Application getStatus(String appName)
- throws IOException, YarnException {
- validateClusterName(appName);
- ApplicationId currentAppId = getAppId(appName);
- ApplicationReport appReport = yarnClient.getApplicationReport(currentAppId);
- ClientAMProtocol amProxy = getAMProxy(appName, appReport);
- Application appSpec;
- if (amProxy != null) {
- GetStatusResponseProto response =
- amProxy.getStatus(GetStatusRequestProto.newBuilder().build());
- appSpec = ServiceApiUtil.jsonSerDeser.fromJson(response.getStatus());
- } else {
- appSpec = new Application();
- appSpec.setName(appName);
- }
- ApplicationTimeout lifetime =
- appReport.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME);
- if (lifetime != null) {
- appSpec.setLifetime(lifetime.getRemainingTime());
- }
- return appSpec;
- }
-
- public YarnClient getYarnClient() {
- return this.yarnClient;
- }
-
- public int actionDependency(ActionDependencyArgs args)
- throws IOException, YarnException {
- String currentUser = RegistryUtils.currentUser();
- LOG.info("Running command as user {}", currentUser);
-
- Path dependencyLibTarGzip = fs.getDependencyTarGzip();
-
- // Check if dependency has already been uploaded, in which case log
- // appropriately and exit success (unless overwrite has been requested)
- if (fs.isFile(dependencyLibTarGzip) && !args.overwrite) {
- System.out.println(String.format(
- "Dependency libs are already uploaded to %s. Use %s "
- + "if you want to re-upload", dependencyLibTarGzip.toUri(),
- Arguments.ARG_OVERWRITE));
- return EXIT_SUCCESS;
- }
-
- String[] libDirs = SliderUtils.getLibDirs();
- if (libDirs.length > 0) {
- File tempLibTarGzipFile = File.createTempFile(
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
- // copy all jars
- tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter());
-
- LOG.info("Version Info: " + VersionInfo.getBuildVersion());
- fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip,
- new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS));
- return EXIT_SUCCESS;
- } else {
- return EXIT_FALSE;
- }
- }
-
- // Get AMProxy with the appReport provided
- protected ClientAMProtocol getAMProxy(String appName, ApplicationReport report)
- throws IOException {
- if (!cachedAMProxies.containsKey(appName) && !StringUtils
- .isEmpty(report.getHost())) {
- insertAMProxy(appName, report.getHost(), report.getRpcPort());
- }
- return cachedAMProxies.get(appName);
- }
-
- // Get AMProxy without appReport provided - it'll getAppReport from RM
- protected ClientAMProtocol getAMProxy(String appName)
- throws IOException, YarnException {
- ApplicationId currentAppId = getAppId(appName);
-
- if (cachedAMProxies.containsKey(appName)) {
- return cachedAMProxies.get(appName);
- } else {
- ApplicationReport appReport =
- yarnClient.getApplicationReport(currentAppId);
- String host = appReport.getHost();
- int port = appReport.getRpcPort();
- if (!StringUtils.isEmpty(host)) {
- return insertAMProxy(appName, host, port);
- }
- return null;
- }
- }
-
- private ClientAMProtocol insertAMProxy(String appName, String host, int port)
- throws IOException {
- InetSocketAddress address =
- NetUtils.createSocketAddrForHost(host, port);
- ClientAMProtocol amProxy =
- ClientAMProxy.createProxy(getConfig(), ClientAMProtocol.class,
- UserGroupInformation.getCurrentUser(), rpc, address);
- cachedAMProxies.put(appName, amProxy);
- return amProxy;
- }
-
- private synchronized ApplicationId getAppId(String appName)
- throws IOException, YarnException {
- if (cachedAppIds.containsKey(appName)) {
- return cachedAppIds.get(appName);
- }
- Application persistedApp = ServiceApiUtil.loadApplication(fs, appName);
- if (persistedApp == null) {
- throw new YarnException("Application " + appName
- + " doesn't exist on hdfs. Please check if the app exists in RM");
- }
- ApplicationId currentAppId = ApplicationId.fromString(persistedApp.getId());
- cachedAppIds.put(appName, currentAppId);
- return currentAppId;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
deleted file mode 100644
index ea3bb0a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Base args for all actions
- */
-public abstract class AbstractActionArgs extends ArgOps implements Arguments {
- protected static final Logger log =
- LoggerFactory.getLogger(AbstractActionArgs.class);
-
-
- protected AbstractActionArgs() {
- }
-
- /**
- * URI/binding to the filesystem
- */
- @Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG},
- description = "Filesystem Binding")
- public String filesystemBinding;
-
- @Parameter(names = {ARG_BASE_PATH},
- description = "Service base path on the filesystem",
- converter = PathArgumentConverter.class)
- public Path basePath;
-
- /**
- * This is the default parameter
- */
- @Parameter
- public final List<String> parameters = new ArrayList<>();
-
- /**
- * get the name: relies on arg 1 being the cluster name in all operations
- * @return the name argument, null if there is none
- */
- public String getClusterName() {
- return (parameters.isEmpty()) ? null : parameters.get(0);
- }
-
- /**
- -D name=value
-
- Define an configuration option which overrides any options in
- the configuration XML files of the image or in the image configuration
- directory. The values will be persisted.
- Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
-
- */
-
- @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions")
- public final List<String> definitions = new ArrayList<>();
-
- /**
- * System properties
- */
- @Parameter(names = {ARG_SYSPROP}, arity = 1,
- description = "system properties in the form name value" +
- " These are set after the JVM is started.")
- public final List<String> sysprops = new ArrayList<>(0);
-
-
- @Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER},
- description = "Binding (usually hostname:port) of the YARN resource manager")
- public String manager;
-
-
- @Parameter(names = ARG_DEBUG, description = "Debug mode")
- public boolean debug = false;
-
- @Parameter(names = {ARG_HELP}, description = "Help", help = true)
- public boolean help = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 1;
- }
-
- /**
- * Get the name of the action
- * @return the action name
- */
- public abstract String getActionName() ;
-
- /**
- * Get the max #of params expected
- * @return the number of params in the {@link #parameters} field;
- */
- public int getMaxParams() {
- return getMinParams();
- }
-
- public void validate() throws BadCommandArgumentsException, UsageException {
-
- int minArgs = getMinParams();
- int actionArgSize = parameters.size();
- if (minArgs > actionArgSize) {
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName() +
- ", Expected minimum " + minArgs + " but got " + actionArgSize);
- }
- int maxArgs = getMaxParams();
- if (maxArgs == -1) {
- maxArgs = minArgs;
- }
- if (actionArgSize > maxArgs) {
- String message = String.format("%s for action %s: limit is %d but saw %d: ",
- ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
- getActionName(), maxArgs,
- actionArgSize);
-
- log.error(message);
- int index = 1;
- StringBuilder buf = new StringBuilder(message);
- for (String actionArg : parameters) {
- log.error("[{}] \"{}\"", index++, actionArg);
- buf.append(" \"").append(actionArg).append("\" ");
- }
- throw new BadCommandArgumentsException(buf.toString());
- }
- }
-
- @Override
- public String toString() {
- return super.toString() + ": " + getActionName();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
deleted file mode 100644
index 457e357..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.yarn.service.client.params.ArgOps;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-
-/**
- * Base class for all the delegates
- */
-public class AbstractArgsDelegate extends ArgOps implements Arguments {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
deleted file mode 100644
index 017286f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParametersDelegate;
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-import java.io.File;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Abstract Action to build things; shares args across build and
- * list
- */
-public abstract class AbstractClusterBuildingActionArgs
- extends AbstractActionArgs {
- @Parameter(names = {ARG_APPDEF},
- description = "Template application definition file in JSON format.")
- public File appDef;
-
- public File getAppDef() {
- return appDef;
- }
-
- @Parameter(names = {
- ARG_QUEUE }, description = "Queue to submit the application")
- public String queue;
-
- @Parameter(names = {
- ARG_LIFETIME }, description = "Lifetime of the application from the time of request")
- public long lifetime;
-
- @ParametersDelegate
- public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
-
- @ParametersDelegate
- public OptionArgsDelegate optionsDelegate =
- new OptionArgsDelegate();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
deleted file mode 100644
index c2ff545..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_BUILD},
- commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
-
-public class ActionBuildArgs extends AbstractClusterBuildingActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_BUILD;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
deleted file mode 100644
index 0097b4e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_CLIENT},
- commandDescription = SliderActions.DESCRIBE_ACTION_CLIENT)
-
-public class ActionClientArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_CLIENT;
- }
-
- @Parameter(names = {ARG_INSTALL},
- description = "Install client")
- public boolean install;
-
- @Parameter(names = {ARG_NAME},
- description = "The name of the application")
- public String name;
-
- @Parameter(names = {ARG_PACKAGE},
- description = "Path to app package")
- public String packageURI;
-
- @Parameter(names = {ARG_DEST},
- description = "The location where to install the client")
- public File installLocation;
-
- @Parameter(names = {ARG_CONFIG},
- description = "Client configuration")
- public File clientConfig;
-
- /**
- * Get the min #of params expected
- *
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
deleted file mode 100644
index eecffb6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_CREATE},
- commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
-
-public class ActionCreateArgs extends AbstractClusterBuildingActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_CREATE;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
deleted file mode 100644
index 51e07c9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_DEPENDENCY },
- commandDescription = SliderActions.DESCRIBE_ACTION_DEPENDENCY)
-public class ActionDependencyArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_DEPENDENCY;
- }
-
- @Parameter(names = { ARG_UPLOAD },
- description = "Upload AM and agent libraries to HDFS for this client")
- public boolean upload;
-
- @Parameter(names = { ARG_OVERWRITE },
- description = "Overwrite current uploaded dependency libs")
- public boolean overwrite = false;
-
- /**
- * Get the min #of params expected
- *
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
-
- if (!upload) {
- throw new UsageException("Option " + ARG_UPLOAD + " is mandatory");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
deleted file mode 100644
index 8c41c04..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_DESTROY},
- commandDescription = SliderActions.DESCRIBE_ACTION_DESTROY)
-
-public class ActionDestroyArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_DESTROY;
- }
-
- @Parameter(names = {ARG_FORCE},
- description = "force the operation")
- public boolean force;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
deleted file mode 100644
index ba3c5a9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_EXISTS},
- commandDescription = SliderActions.DESCRIBE_ACTION_EXISTS)
-
-public class ActionExistsArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_EXISTS;
- }
-
- @Parameter(names = {ARG_LIVE},
- description = "verify that the application is running")
- public boolean live;
-
- @Parameter(names = {ARG_STATE},
- description = "verify that the application is in the specific YARN state")
- public String state = "";
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for any application report")
- public File out;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
deleted file mode 100644
index b7acf58..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-import java.util.List;
-import java.util.Map;
-
-@Parameters(commandNames = { SliderActions.ACTION_FLEX},
- commandDescription = SliderActions.DESCRIBE_ACTION_FLEX)
-
-public class ActionFlexArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_FLEX;
- }
-
- @ParametersDelegate
- public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
-
- /**
- * Get the component mapping (may be empty, but never null)
- * @return mapping
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, String> getComponentMap() throws
- BadCommandArgumentsException {
- return componentDelegate.getComponentMap();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
deleted file mode 100644
index aecf0eb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-
-@Parameters(commandNames = { SliderActions.ACTION_STOP },
- commandDescription = SliderActions.DESCRIBE_ACTION_FREEZE)
-
-public class ActionFreezeArgs extends AbstractActionArgs implements
- WaitTimeAccessor {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_STOP;
- }
-
- public static final String FREEZE_COMMAND_ISSUED = "stop command issued";
- @ParametersDelegate
- public WaitArgsDelegate waitDelegate = new WaitArgsDelegate();
-
- @Override
- public int getWaittime() {
- return waitDelegate.getWaittime();
- }
-
- @Override
- public void setWaittime(int waittime) {
- waitDelegate.setWaittime(waittime);
- }
-
- @Parameter(names={ARG_MESSAGE},
- description = "reason for the operation")
- public String message = FREEZE_COMMAND_ISSUED;
-
- @Parameter(names = {ARG_FORCE},
- description = "force the operation")
- public boolean force;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
deleted file mode 100644
index 51aa88a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-/**
- * The Help command
- */
-@Parameters(commandNames = { SliderActions.ACTION_HELP},
- commandDescription = SliderActions.DESCRIBE_ACTION_HELP)
-public class ActionHelpArgs extends AbstractActionArgs {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_HELP;
- }
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
deleted file mode 100644
index 061121e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-
-@Parameters(commandNames = { SliderActions.ACTION_KDIAG},
- commandDescription = SliderActions.DESCRIBE_ACTION_KDIAG)
-
-public class ActionKDiagArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_KDIAG;
- }
-
- @Parameter(names = {ARG_SERVICES}, variableArity = true,
- description =" list of services to check")
- public List<String> services = new ArrayList<>();
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for report")
- public File out;
-
- @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
- public File keytab;
-
- @Parameter(names = {ARG_KEYLEN}, description = "minimum key length")
- public int keylen = 256;
-
- @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
- public String principal;
-
- @Parameter(names = {ARG_SECURE}, description = "Is security required")
- public boolean secure = false;
-
- @Override
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
- if (keytab != null && SliderUtils.isUnset(principal)) {
- throw new UsageException("Missing argument " + ARG_PRINCIPAL);
- }
- if (keytab == null && SliderUtils.isSet(principal)) {
- throw new UsageException("Missing argument " + ARG_KEYTAB);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
deleted file mode 100644
index 7e51457..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_KEYTAB},
- commandDescription = SliderActions.DESCRIBE_ACTION_KEYTAB)
-
-public class ActionKeytabArgs extends AbstractActionArgs {
-
- public ActionKeytabArgs() {
- super();
- }
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_INSTALL_KEYTAB;
- }
-
- @Parameter(names = {ARG_KEYTABINSTALL},
- description = "Install the keytab")
- public boolean install;
-
- @Parameter(names = {ARG_KEYTABDELETE},
- description = "Delete the keytab")
- public boolean delete;
-
- @Parameter(names = {ARG_KEYTABLIST},
- description = "List of installed keytabs")
- public boolean list;
-
- @Parameter(names = {ARG_KEYTAB},
- description = "Path or name of the keytab")
- public String keytab;
-
- @Parameter(names = {ARG_FOLDER},
- description = "The name of the folder in which to store the keytab")
- public String folder;
-
- @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing keytab")
- public boolean overwrite = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 3;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
deleted file mode 100644
index 005c172..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_LIST},
- commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
-
-public class ActionListArgs extends AbstractActionArgs {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_LIST;
- }
-
- @Parameter(names = {ARG_LIVE},
- description = "List only live application instances")
- public boolean live;
-
- @Parameter(names = {ARG_STATE},
- description = "list only applications in the specific YARN state")
- public String state = "";
-
- @Parameter(names = {ARG_VERBOSE},
- description = "print out information in details")
- public boolean verbose = false;
-
- @Parameter(names = {ARG_CONTAINERS},
- description = "List containers of an application instance")
- public boolean containers;
-
- @Parameter(names = {ARG_VERSION},
- description = "Filter containers by app version (used with " +
- ARG_CONTAINERS + ")")
- public String version;
-
- @Parameter(names = {ARG_COMPONENTS}, variableArity = true,
- description = "Filter containers by component names (used with " +
- ARG_CONTAINERS + ")")
- public Set<String> components = new HashSet<>(0);
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[65/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
new file mode 100644
index 0000000..cb7131e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -0,0 +1,494 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.apache.hadoop.yarn.service.ContainerFailureTracker;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.ServiceScheduler;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import org.apache.hadoop.yarn.service.ServiceMetrics;
+import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
+import org.apache.hadoop.yarn.state.MultipleArcTransition;
+import org.apache.hadoop.yarn.state.SingleArcTransition;
+import org.apache.hadoop.yarn.state.StateMachine;
+import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.Apps;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils;
+import org.apache.hadoop.yarn.service.monitor.probe.Probe;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.*;
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.START;
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.STOP;
+import static org.apache.hadoop.yarn.service.component.ComponentState.*;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_THRESHOLD;
+
+public class Component implements EventHandler<ComponentEvent> {
+ private static final Logger LOG = LoggerFactory.getLogger(Component.class);
+
+ private org.apache.hadoop.yarn.service.api.records.Component componentSpec;
+ private long allocateId;
+ private Priority priority;
+ private ServiceMetrics componentMetrics;
+ private ServiceScheduler scheduler;
+ private ServiceContext context;
+ private AMRMClientAsync<ContainerRequest> amrmClient;
+ private AtomicLong instanceIdCounter = new AtomicLong();
+ private Map<ComponentInstanceId, ComponentInstance> compInstances =
+ new ConcurrentHashMap<>();
+ // component instances to be assigned with a container
+ private List<ComponentInstance> pendingInstances = new LinkedList<>();
+ private ContainerFailureTracker failureTracker;
+ private Probe probe;
+ private final ReentrantReadWriteLock.ReadLock readLock;
+ private final ReentrantReadWriteLock.WriteLock writeLock;
+ public int maxContainerFailurePerComp;
+ // The number of containers failed since last reset. This excludes preempted,
+ // disk_failed containers etc. This will be reset to 0 periodically.
+ public AtomicInteger currentContainerFailure = new AtomicInteger(0);
+
+ private StateMachine<ComponentState, ComponentEventType, ComponentEvent>
+ stateMachine;
+ private AsyncDispatcher compInstanceDispatcher;
+ private static final StateMachineFactory<Component, ComponentState, ComponentEventType, ComponentEvent>
+ stateMachineFactory =
+ new StateMachineFactory<Component, ComponentState, ComponentEventType, ComponentEvent>(
+ INIT)
+ // INIT will only got to FLEXING
+ .addTransition(INIT, EnumSet.of(STABLE, FLEXING),
+ FLEX, new FlexComponentTransition())
+
+ // container allocated by RM
+ .addTransition(FLEXING, FLEXING, CONTAINER_ALLOCATED,
+ new ContainerAllocatedTransition())
+ // container launched on NM
+ .addTransition(FLEXING, EnumSet.of(STABLE, FLEXING),
+ CONTAINER_STARTED, new ContainerStartedTransition())
+ // container failed while flexing
+ .addTransition(FLEXING, FLEXING, CONTAINER_COMPLETED,
+ new ContainerCompletedTransition())
+ // Flex while previous flex is still in progress
+ .addTransition(FLEXING, EnumSet.of(FLEXING), FLEX,
+ new FlexComponentTransition())
+
+ // container failed while stable
+ .addTransition(STABLE, FLEXING, CONTAINER_COMPLETED,
+ new ContainerCompletedTransition())
+ // Ignore surplus container
+ .addTransition(STABLE, STABLE, CONTAINER_ALLOCATED,
+ new ContainerAllocatedTransition())
+ // Flex by user
+ // For flex up, go to FLEXING state
+ // For flex down, go to STABLE state
+ .addTransition(STABLE, EnumSet.of(STABLE, FLEXING),
+ FLEX, new FlexComponentTransition())
+ .installTopology();
+
+ public Component(
+ org.apache.hadoop.yarn.service.api.records.Component component,
+ long allocateId, ServiceContext context) {
+ this.allocateId = allocateId;
+ this.priority = Priority.newInstance((int) allocateId);
+ this.componentSpec = component;
+ componentMetrics = ServiceMetrics.register(component.getName(),
+ "Metrics for component " + component.getName());
+ componentMetrics
+ .tag("type", "Metrics type [component or service]", "component");
+ this.scheduler = context.scheduler;
+ this.context = context;
+ amrmClient = scheduler.getAmRMClient();
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ this.readLock = lock.readLock();
+ this.writeLock = lock.writeLock();
+ this.stateMachine = stateMachineFactory.make(this);
+ compInstanceDispatcher = scheduler.getCompInstanceDispatcher();
+ failureTracker =
+ new ContainerFailureTracker(context, this);
+ probe = MonitorUtils.getProbe(componentSpec.getReadinessCheck());
+ maxContainerFailurePerComp = componentSpec.getConfiguration()
+ .getPropertyInt(CONTAINER_FAILURE_THRESHOLD, 10);
+ createNumCompInstances(component.getNumberOfContainers());
+ }
+
+ private void createNumCompInstances(long count) {
+ for (int i = 0; i < count; i++) {
+ createOneCompInstance();
+ }
+ }
+
+ private void createOneCompInstance() {
+ ComponentInstanceId id =
+ new ComponentInstanceId(instanceIdCounter.getAndIncrement(),
+ componentSpec.getName());
+ ComponentInstance instance = new ComponentInstance(this, id);
+ compInstances.put(id, instance);
+ pendingInstances.add(instance);
+ }
+
+ private static class FlexComponentTransition implements
+ MultipleArcTransition<Component, ComponentEvent, ComponentState> {
+ // For flex up, go to FLEXING state
+ // For flex down, go to STABLE state
+ @Override
+ public ComponentState transition(Component component,
+ ComponentEvent event) {
+ component.setDesiredContainers((int)event.getDesired());
+ if (!component.areDependenciesReady()) {
+ LOG.info("[FLEX COMPONENT {}]: Flex deferred because dependencies not"
+ + " satisfied.", component.getName());
+ return component.getState();
+ }
+ if (component.getState() == INIT) {
+ // This happens on init
+ LOG.info("[INIT COMPONENT " + component.getName() + "]: " + event
+ .getDesired() + " instances.");
+ component.requestContainers(event.getDesired());
+ return FLEXING;
+ }
+ long before = component.getComponentSpec().getNumberOfContainers();
+ long delta = event.getDesired() - before;
+ component.getComponentSpec().setNumberOfContainers(event.getDesired());
+ if (delta > 0) {
+ // Scale up
+ LOG.info("[FLEX UP COMPONENT " + component.getName() + "]: scaling up from "
+ + before + " to " + event.getDesired());
+ component.requestContainers(delta);
+ component.createNumCompInstances(delta);
+ return FLEXING;
+ } else if (delta < 0){
+ delta = 0 - delta;
+ // scale down
+ LOG.info("[FLEX DOWN COMPONENT " + component.getName()
+ + "]: scaling down from " + before + " to " + event.getDesired());
+ List<ComponentInstance> list =
+ new ArrayList<>(component.compInstances.values());
+
+ // sort in Most recent -> oldest order, destroy most recent ones.
+ Collections.sort(list, Collections.reverseOrder());
+ for (int i = 0; i < delta; i++) {
+ ComponentInstance instance = list.get(i);
+ // remove the instance
+ component.compInstances.remove(instance.getCompInstanceId());
+ component.pendingInstances.remove(instance);
+ component.componentMetrics.containersFailed.incr();
+ component.componentMetrics.containersRunning.decr();
+ // decrement id counter
+ component.instanceIdCounter.decrementAndGet();
+ instance.destroy();
+ }
+ return STABLE;
+ } else {
+ LOG.info("[FLEX COMPONENT " + component.getName() + "]: already has " +
+ event.getDesired() + " instances, ignoring");
+ return STABLE;
+ }
+ }
+ }
+
+ private static class ContainerAllocatedTransition extends BaseTransition {
+ @Override
+ public void transition(Component component, ComponentEvent event) {
+ component.assignContainerToCompInstance(event.getContainer());
+ }
+ }
+
+ private static class ContainerStartedTransition implements
+ MultipleArcTransition<Component,ComponentEvent,ComponentState> {
+
+ @Override public ComponentState transition(Component component,
+ ComponentEvent event) {
+ component.compInstanceDispatcher.getEventHandler().handle(
+ new ComponentInstanceEvent(event.getInstance().getContainerId(),
+ START));
+ component.incRunningContainers();
+ return checkIfStable(component);
+ }
+ }
+
+ private static ComponentState checkIfStable(Component component) {
+ // if desired == running
+ if (component.componentMetrics.containersRunning.value() == component
+ .getComponentSpec().getNumberOfContainers()) {
+ return STABLE;
+ } else {
+ return FLEXING;
+ }
+ }
+
+ private static class ContainerCompletedTransition extends BaseTransition {
+ @Override
+ public void transition(Component component, ComponentEvent event) {
+ component.updateMetrics(event.getStatus());
+
+ // add back to pending list
+ component.pendingInstances.add(event.getInstance());
+ LOG.info(
+ "[COMPONENT {}]: {} completed, num pending comp instances increased to {}.",
+ component.getName(), event.getStatus().getContainerId(),
+ component.pendingInstances.size());
+ component.compInstanceDispatcher.getEventHandler().handle(
+ new ComponentInstanceEvent(event.getStatus().getContainerId(),
+ STOP).setStatus(event.getStatus()));
+ }
+ }
+
+ public ServiceMetrics getCompMetrics () {
+ return componentMetrics;
+ }
+
+ private void assignContainerToCompInstance(Container container) {
+ if (pendingInstances.size() == 0) {
+ LOG.info(
+ "[COMPONENT {}]: No pending component instance left, release surplus container {}",
+ getName(), container.getId());
+ scheduler.getAmRMClient().releaseAssignedContainer(container.getId());
+ componentMetrics.surplusContainers.incr();
+ scheduler.getServiceMetrics().surplusContainers.incr();
+ return;
+ }
+ ComponentInstance instance = pendingInstances.remove(0);
+ LOG.info(
+ "[COMPONENT {}]: {} allocated, num pending component instances reduced to {}",
+ getName(), container.getId(), pendingInstances.size());
+ instance.setContainer(container);
+ scheduler.addLiveCompInstance(container.getId(), instance);
+ LOG.info(
+ "[COMPONENT {}]: Assigned {} to component instance {} and launch on host {} ",
+ getName(), container.getId(), instance.getCompInstanceName(),
+ container.getNodeId());
+ scheduler.getContainerLaunchService()
+ .launchCompInstance(scheduler.getApp(), instance, container);
+ }
+
+ @SuppressWarnings({ "unchecked" })
+ public void requestContainers(long count) {
+ Resource resource = Resource
+ .newInstance(componentSpec.getResource().getMemoryMB(),
+ componentSpec.getResource().getCpus());
+
+ for (int i = 0; i < count; i++) {
+ //TODO Once YARN-5468 is done, use that for anti-affinity
+ ContainerRequest request =
+ ContainerRequest.newBuilder().capability(resource).priority(priority)
+ .allocationRequestId(allocateId).relaxLocality(true).build();
+ amrmClient.addContainerRequest(request);
+ }
+ }
+
+ private void setDesiredContainers(int n) {
+ int delta = n - scheduler.getServiceMetrics().containersDesired.value();
+ if (delta > 0) {
+ scheduler.getServiceMetrics().containersDesired.incr(delta);
+ } else {
+ scheduler.getServiceMetrics().containersDesired.decr(delta);
+ }
+ componentMetrics.containersDesired.set(n);
+ }
+
+
+
+ private void updateMetrics(ContainerStatus status) {
+ switch (status.getExitStatus()) {
+ case SUCCESS:
+ componentMetrics.containersSucceeded.incr();
+ scheduler.getServiceMetrics().containersSucceeded.incr();
+ return;
+ case PREEMPTED:
+ componentMetrics.containersPreempted.incr();
+ scheduler.getServiceMetrics().containersPreempted.incr();
+ break;
+ case DISKS_FAILED:
+ componentMetrics.containersDiskFailure.incr();
+ scheduler.getServiceMetrics().containersDiskFailure.incr();
+ break;
+ default:
+ break;
+ }
+
+ // containersFailed include preempted, disks_failed etc.
+ componentMetrics.containersFailed.incr();
+ scheduler.getServiceMetrics().containersFailed.incr();
+
+ // dec running container
+ decRunningContainers();
+
+ if (Apps.shouldCountTowardsNodeBlacklisting(status.getExitStatus())) {
+ String host = scheduler.getLiveInstances().get(status.getContainerId())
+ .getNodeId().getHost();
+ failureTracker.incNodeFailure(host);
+ currentContainerFailure.getAndIncrement() ;
+ }
+ }
+
+ public boolean areDependenciesReady() {
+ List<String> dependencies = componentSpec.getDependencies();
+ if (SliderUtils.isEmpty(dependencies)) {
+ return true;
+ }
+ for (String dependency : dependencies) {
+ Component dependentComponent =
+ scheduler.getAllComponents().get(dependency);
+ if (dependentComponent == null) {
+ LOG.error("Couldn't find dependency {} for {} (should never happen)",
+ dependency, getName());
+ continue;
+ }
+ if (dependentComponent.getNumReadyInstances() < dependentComponent
+ .getNumDesiredInstances()) {
+ LOG.info("[COMPONENT {}]: Dependency {} not satisfied, only {} of {}"
+ + " instances are ready.", getName(), dependency,
+ dependentComponent.getNumReadyInstances(),
+ dependentComponent.getNumDesiredInstances());
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private void incRunningContainers() {
+ componentMetrics.containersRunning.incr();
+ scheduler.getServiceMetrics().containersRunning.incr();
+ }
+
+ public void incContainersReady() {
+ componentMetrics.containersReady.incr();
+ }
+
+ public void decContainersReady() {
+ componentMetrics.containersReady.decr();
+ }
+
+ private void decRunningContainers() {
+ componentMetrics.containersRunning.decr();
+ scheduler.getServiceMetrics().containersRunning.decr();
+ }
+
+ public int getNumReadyInstances() {
+ return componentMetrics.containersReady.value();
+ }
+
+ public int getNumRunningInstances() {
+ return componentMetrics.containersRunning.value();
+ }
+
+ public int getNumDesiredInstances() {
+ return componentMetrics.containersDesired.value();
+ }
+
+ public Map<ComponentInstanceId, ComponentInstance> getAllComponentInstances() {
+ return compInstances;
+ }
+
+ public org.apache.hadoop.yarn.service.api.records.Component getComponentSpec() {
+ return this.componentSpec;
+ }
+
+ public void resetCompFailureCount() {
+ LOG.info("[COMPONENT {}]: Reset container failure count from {} to 0.",
+ getName(), currentContainerFailure.get());
+ currentContainerFailure.set(0);
+ failureTracker.resetContainerFailures();
+ }
+
+ public Probe getProbe() {
+ return probe;
+ }
+
+ public Priority getPriority() {
+ return priority;
+ }
+
+ public long getAllocateId() {
+ return allocateId;
+ }
+
+ public String getName () {
+ return componentSpec.getName();
+ }
+
+ public ComponentState getState() {
+ this.readLock.lock();
+
+ try {
+ return this.stateMachine.getCurrentState();
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+ public ServiceScheduler getScheduler() {
+ return scheduler;
+ }
+
+ @Override
+ public void handle(ComponentEvent event) {
+ try {
+ writeLock.lock();
+ ComponentState oldState = getState();
+ try {
+ stateMachine.doTransition(event.getType(), event);
+ } catch (InvalidStateTransitionException e) {
+ LOG.error(MessageFormat.format("[COMPONENT {0}]: Invalid event {1} at {2}",
+ componentSpec.getName(), event.getType(), oldState), e);
+ }
+ if (oldState != getState()) {
+ LOG.info("[COMPONENT {}] Transitioned from {} to {} on {} event.",
+ componentSpec.getName(), oldState, getState(), event.getType());
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private static class BaseTransition implements
+ SingleArcTransition<Component, ComponentEvent> {
+
+ @Override public void transition(Component component,
+ ComponentEvent event) {
+ }
+ }
+
+ public ServiceContext getContext() {
+ return context;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
new file mode 100644
index 0000000..d93dcf1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+
+public class ComponentEvent extends AbstractEvent<ComponentEventType> {
+ private long desired;
+ private final String name;
+ private final ComponentEventType type;
+ private Container container;
+ private ComponentInstance instance;
+ private ContainerStatus status;
+
+ public ComponentEvent(String name, ComponentEventType type) {
+ super(type);
+ this.name = name;
+ this.type = type;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public ComponentEventType getType() {
+ return type;
+ }
+
+ public long getDesired() {
+ return desired;
+ }
+
+ public ComponentEvent setDesired(long desired) {
+ this.desired = desired;
+ return this;
+ }
+
+ public Container getContainer() {
+ return container;
+ }
+
+ public ComponentEvent setContainer(Container container) {
+ this.container = container;
+ return this;
+ }
+
+ public ComponentInstance getInstance() {
+ return instance;
+ }
+
+ public ComponentEvent setInstance(ComponentInstance instance) {
+ this.instance = instance;
+ return this;
+ }
+
+ public ContainerStatus getStatus() {
+ return status;
+ }
+
+ public ComponentEvent setStatus(ContainerStatus status) {
+ this.status = status;
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
new file mode 100644
index 0000000..6729699
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+public enum ComponentEventType {
+ FLEX,
+ CONTAINER_ALLOCATED,
+ CONTAINER_STARTED,
+ CONTAINER_COMPLETED
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
new file mode 100644
index 0000000..a5f9ff4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+public enum ComponentState {
+ INIT,
+ FLEXING,
+ STABLE
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
new file mode 100644
index 0000000..7d6525b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -0,0 +1,493 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component.instance;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.client.api.NMClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.service.ServiceScheduler;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
+import org.apache.hadoop.yarn.state.SingleArcTransition;
+import org.apache.hadoop.yarn.state.StateMachine;
+import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.BoundedAppender;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
+import org.apache.hadoop.yarn.service.monitor.probe.ProbeStatus;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.util.Date;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.KILLED_BY_APPMASTER;
+import static org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE;
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.*;
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceState.*;
+
+public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
+ Comparable<ComponentInstance> {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ComponentInstance.class);
+
+ private StateMachine<ComponentInstanceState, ComponentInstanceEventType,
+ ComponentInstanceEvent> stateMachine;
+ private Component component;
+ private final ReadLock readLock;
+ private final WriteLock writeLock;
+
+ private ComponentInstanceId compInstanceId = null;
+ private Path compInstanceDir;
+ private Container container;
+ private YarnRegistryViewForProviders yarnRegistryOperations;
+ private FileSystem fs;
+ private boolean timelineServiceEnabled = false;
+ private ServiceTimelinePublisher serviceTimelinePublisher;
+ private ServiceScheduler scheduler;
+ private BoundedAppender diagnostics = new BoundedAppender(64 * 1024);
+ private volatile ScheduledFuture containerStatusFuture;
+ private volatile ContainerStatus status;
+ private long containerStartedTime = 0;
+ // This container object is used for rest API query
+ private org.apache.hadoop.yarn.service.api.records.Container containerSpec;
+
+ private static final StateMachineFactory<ComponentInstance,
+ ComponentInstanceState, ComponentInstanceEventType, ComponentInstanceEvent>
+ stateMachineFactory =
+ new StateMachineFactory<ComponentInstance, ComponentInstanceState,
+ ComponentInstanceEventType, ComponentInstanceEvent>(INIT)
+ .addTransition(INIT, STARTED, START,
+ new ContainerStartedTransition())
+
+ //From Running
+ .addTransition(STARTED, INIT, STOP,
+ new ContainerStoppedTransition())
+ .addTransition(STARTED, READY, BECOME_READY,
+ new ContainerBecomeReadyTransition())
+
+ // FROM READY
+ .addTransition(READY, STARTED, BECOME_NOT_READY,
+ new ContainerBecomeNotReadyTransition())
+ .addTransition(READY, INIT, STOP, new ContainerStoppedTransition())
+ .installTopology();
+
+
+
+ public ComponentInstance(Component component,
+ ComponentInstanceId compInstanceId) {
+ this.stateMachine = stateMachineFactory.make(this);
+ this.component = component;
+ this.compInstanceId = compInstanceId;
+ this.scheduler = component.getScheduler();
+ this.yarnRegistryOperations =
+ component.getScheduler().getYarnRegistryOperations();
+ this.serviceTimelinePublisher =
+ component.getScheduler().getServiceTimelinePublisher();
+ if (YarnConfiguration
+ .timelineServiceV2Enabled(component.getScheduler().getConfig())) {
+ this.timelineServiceEnabled = true;
+ }
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ this.readLock = lock.readLock();
+ this.writeLock = lock.writeLock();
+ this.fs = scheduler.getContext().fs.getFileSystem();
+ }
+
+ private static class ContainerStartedTransition extends BaseTransition {
+ @Override public void transition(ComponentInstance compInstance,
+ ComponentInstanceEvent event) {
+ // Query container status for ip and host
+ compInstance.containerStatusFuture =
+ compInstance.scheduler.executorService.scheduleAtFixedRate(
+ new ContainerStatusRetriever(compInstance.scheduler,
+ compInstance.getContainerId(), compInstance), 0, 1,
+ TimeUnit.SECONDS);
+
+ org.apache.hadoop.yarn.service.api.records.Container container =
+ new org.apache.hadoop.yarn.service.api.records.Container();
+ container.setId(compInstance.getContainerId().toString());
+ container.setLaunchTime(new Date());
+ container.setState(ContainerState.RUNNING_BUT_UNREADY);
+ container.setBareHost(compInstance.container.getNodeId().getHost());
+ container.setComponentName(compInstance.getCompInstanceName());
+ if (compInstance.containerSpec != null) {
+ // remove the previous container.
+ compInstance.getCompSpec().removeContainer(compInstance.containerSpec);
+ }
+ compInstance.containerSpec = container;
+ compInstance.getCompSpec().addContainer(container);
+ compInstance.containerStartedTime = System.currentTimeMillis();
+
+ if (compInstance.timelineServiceEnabled) {
+ compInstance.serviceTimelinePublisher
+ .componentInstanceStarted(container, compInstance);
+ }
+ }
+ }
+
+ private static class ContainerBecomeReadyTransition extends BaseTransition {
+ @Override
+ public void transition(ComponentInstance compInstance,
+ ComponentInstanceEvent event) {
+ compInstance.component.incContainersReady();
+ compInstance.containerSpec.setState(ContainerState.READY);
+ }
+ }
+
+ private static class ContainerBecomeNotReadyTransition extends BaseTransition {
+ @Override
+ public void transition(ComponentInstance compInstance,
+ ComponentInstanceEvent event) {
+ compInstance.component.decContainersReady();
+ compInstance.containerSpec.setState(ContainerState.RUNNING_BUT_UNREADY);
+ }
+ }
+
+ private static class ContainerStoppedTransition extends BaseTransition {
+ @Override
+ public void transition(ComponentInstance compInstance,
+ ComponentInstanceEvent event) {
+ // re-ask the failed container.
+ Component comp = compInstance.component;
+ comp.requestContainers(1);
+ LOG.info(compInstance.getCompInstanceId()
+ + ": Container completed. Requested a new container." + System
+ .lineSeparator() + " exitStatus={}, diagnostics={}.",
+ event.getStatus().getExitStatus(),
+ event.getStatus().getDiagnostics());
+ String containerDiag =
+ compInstance.getCompInstanceId() + ": " + event.getStatus()
+ .getDiagnostics();
+ compInstance.diagnostics.append(containerDiag + System.lineSeparator());
+
+ boolean shouldExit = false;
+ // check if it exceeds the failure threshold
+ if (comp.currentContainerFailure.get() > comp.maxContainerFailurePerComp) {
+ String exitDiag = MessageFormat.format(
+ "[COMPONENT {0}]: Failed {1} times, exceeded the limit - {2}. Shutting down now... "
+ + System.lineSeparator(),
+ comp.getName(), comp.currentContainerFailure.get(), comp.maxContainerFailurePerComp);
+ compInstance.diagnostics.append(exitDiag);
+ // append to global diagnostics that will be reported to RM.
+ comp.getScheduler().getDiagnostics().append(containerDiag);
+ comp.getScheduler().getDiagnostics().append(exitDiag);
+ LOG.warn(exitDiag);
+ shouldExit = true;
+ }
+
+ // clean up registry
+ // hdfs dir content will be overwritten when a new container gets started,
+ // so no need remove.
+ compInstance.scheduler.executorService
+ .submit(compInstance::cleanupRegistry);
+
+ // remove the failed ContainerId -> CompInstance mapping
+ comp.getScheduler().removeLiveCompInstance(event.getContainerId());
+
+ if (compInstance.timelineServiceEnabled) {
+ // record in ATS
+ compInstance.serviceTimelinePublisher
+ .componentInstanceFinished(compInstance,
+ event.getStatus().getExitStatus(), event.getStatus().getState(),
+ containerDiag);
+ }
+
+ compInstance.containerSpec.setState(ContainerState.STOPPED);
+ if (shouldExit) {
+ // Sleep for 5 seconds in hope that the state can be recorded in ATS.
+ // in case there's a client polling the comp state, it can be notified.
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted on sleep while exiting.", e);
+ }
+ ExitUtil.terminate(-1);
+ }
+ }
+ }
+
+ public ComponentInstanceState getState() {
+ this.readLock.lock();
+
+ try {
+ return this.stateMachine.getCurrentState();
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ @Override
+ public void handle(ComponentInstanceEvent event) {
+ try {
+ writeLock.lock();
+ ComponentInstanceState oldState = getState();
+ try {
+ stateMachine.doTransition(event.getType(), event);
+ } catch (InvalidStateTransitionException e) {
+ LOG.error(getCompInstanceId() + ": Invalid event " + event.getType() +
+ " at " + oldState, e);
+ }
+ if (oldState != getState()) {
+ LOG.info(getCompInstanceId() + " Transitioned from " + oldState + " to "
+ + getState() + " on " + event.getType() + " event");
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ public void setContainer(Container container) {
+ this.container = container;
+ this.compInstanceId.setContainerId(container.getId());
+ }
+
+ public String getCompInstanceName() {
+ return compInstanceId.getCompInstanceName();
+ }
+
+ public ContainerStatus getContainerStatus() {
+ return status;
+ }
+
+ public void updateContainerStatus(ContainerStatus status) {
+ this.status = status;
+ org.apache.hadoop.yarn.service.api.records.Container container =
+ getCompSpec().getContainer(getContainerId().toString());
+ if (container != null) {
+ container.setIp(StringUtils.join(",", status.getIPs()));
+ container.setHostname(status.getHost());
+ if (timelineServiceEnabled) {
+ serviceTimelinePublisher.componentInstanceUpdated(container);
+ }
+ }
+ updateServiceRecord(yarnRegistryOperations, status);
+ }
+
+ public ContainerId getContainerId() {
+ return container.getId();
+ }
+
+ public String getCompName() {
+ return compInstanceId.getCompName();
+ }
+
+ public void setCompInstanceDir(Path dir) {
+ this.compInstanceDir = dir;
+ }
+
+ public Component getComponent() {
+ return component;
+ }
+
+ public Container getContainer() {
+ return container;
+ }
+
+ public ComponentInstanceId getCompInstanceId() {
+ return compInstanceId;
+ }
+
+ public NodeId getNodeId() {
+ return this.container.getNodeId();
+ }
+
+ public org.apache.hadoop.yarn.service.api.records.Component getCompSpec() {
+ return component.getComponentSpec();
+ }
+
+ private static class BaseTransition implements
+ SingleArcTransition<ComponentInstance, ComponentInstanceEvent> {
+
+ @Override public void transition(ComponentInstance compInstance,
+ ComponentInstanceEvent event) {
+ }
+ }
+
+ public ProbeStatus ping() {
+ if (component.getProbe() == null) {
+ ProbeStatus status = new ProbeStatus();
+ status.setSuccess(true);
+ return status;
+ }
+ return component.getProbe().ping(this);
+ }
+
+ // Write service record into registry
+ private void updateServiceRecord(
+ YarnRegistryViewForProviders yarnRegistry, ContainerStatus status) {
+ ServiceRecord record = new ServiceRecord();
+ String containerId = status.getContainerId().toString();
+ record.set(YarnRegistryAttributes.YARN_ID, containerId);
+ record.description = getCompInstanceName();
+ record.set(YarnRegistryAttributes.YARN_PERSISTENCE,
+ PersistencePolicies.CONTAINER);
+ record.set("yarn:ip", status.getIPs());
+ record.set("yarn:hostname", status.getHost());
+ try {
+ yarnRegistry
+ .putComponent(RegistryPathUtils.encodeYarnID(containerId), record);
+ } catch (IOException e) {
+ LOG.error(
+ "Failed to update service record in registry: " + containerId + "");
+ }
+ }
+
+ // Release the container , cleanup registry, hdfs dir, and record in ATS
+ public void destroy() {
+ LOG.info(getCompInstanceId() + ": Flexed down by user, destroying.");
+ diagnostics.append(getCompInstanceId() + ": Flexed down by user");
+ if (container != null) {
+ scheduler.removeLiveCompInstance(container.getId());
+ component.getScheduler().getAmRMClient()
+ .releaseAssignedContainer(container.getId());
+ getCompSpec().removeContainer(containerSpec);
+ }
+ if (timelineServiceEnabled) {
+ serviceTimelinePublisher
+ .componentInstanceFinished(this, KILLED_BY_APPMASTER, COMPLETE,
+ diagnostics.toString());
+ }
+ scheduler.executorService.submit(this::cleanupRegistryAndCompHdfsDir);
+ }
+
+ private void cleanupRegistry() {
+ ContainerId containerId = getContainerId();
+ String cid = RegistryPathUtils.encodeYarnID(containerId.toString());
+ try {
+ yarnRegistryOperations.deleteComponent(getCompInstanceId(), cid);
+ } catch (IOException e) {
+ LOG.error(getCompInstanceId() + ": Failed to delete registry", e);
+ }
+ }
+
+ //TODO Maybe have a dedicated cleanup service.
+ public void cleanupRegistryAndCompHdfsDir() {
+ cleanupRegistry();
+ try {
+ if (compInstanceDir != null && fs.exists(compInstanceDir)) {
+ boolean deleted = fs.delete(compInstanceDir, true);
+ if (!deleted) {
+ LOG.error(getCompInstanceId()
+ + ": Failed to delete component instance dir: "
+ + compInstanceDir);
+ } else {
+ LOG.info(getCompInstanceId() + ": Deleted component instance dir: "
+ + compInstanceDir);
+ }
+ }
+ } catch (IOException e) {
+ LOG.warn(getCompInstanceId() + ": Failed to delete directory", e);
+ }
+ }
+
+ // Query container status until ip and hostname are available and update
+ // the service record into registry service
+ private static class ContainerStatusRetriever implements Runnable {
+ private ContainerId containerId;
+ private NodeId nodeId;
+ private NMClient nmClient;
+ private ComponentInstance instance;
+ ContainerStatusRetriever(ServiceScheduler scheduler,
+ ContainerId containerId, ComponentInstance instance) {
+ this.containerId = containerId;
+ this.nodeId = instance.getNodeId();
+ this.nmClient = scheduler.getNmClient().getClient();
+ this.instance = instance;
+ }
+ @Override public void run() {
+ ContainerStatus status = null;
+ try {
+ status = nmClient.getContainerStatus(containerId, nodeId);
+ } catch (Exception e) {
+ if (e instanceof YarnException) {
+ throw new YarnRuntimeException(
+ instance.compInstanceId + " Failed to get container status on "
+ + nodeId + " , cancelling.", e);
+ }
+ LOG.error(instance.compInstanceId + " Failed to get container status on "
+ + nodeId + ", will try again", e);
+ return;
+ }
+ if (SliderUtils.isEmpty(status.getIPs()) || SliderUtils
+ .isUnset(status.getHost())) {
+ return;
+ }
+ instance.updateContainerStatus(status);
+ LOG.info(
+ instance.compInstanceId + " IP = " + status.getIPs() + ", host = "
+ + status.getHost() + ", cancel container status retriever");
+ instance.containerStatusFuture.cancel(false);
+ }
+ }
+
+ @Override
+ public int compareTo(ComponentInstance to) {
+ long delta = containerStartedTime - to.containerStartedTime;
+ if (delta == 0) {
+ return getCompInstanceId().compareTo(to.getCompInstanceId());
+ } else if (delta < 0) {
+ return -1;
+ } else {
+ return 1;
+ }
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ ComponentInstance instance = (ComponentInstance) o;
+
+ if (containerStartedTime != instance.containerStartedTime)
+ return false;
+ return compInstanceId.equals(instance.compInstanceId);
+ }
+
+ @Override public int hashCode() {
+ int result = compInstanceId.hashCode();
+ result = 31 * result + (int) (containerStartedTime ^ (containerStartedTime
+ >>> 32));
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java
new file mode 100644
index 0000000..707b034
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component.instance;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+
+public class ComponentInstanceEvent
+ extends AbstractEvent<ComponentInstanceEventType> {
+
+ private ContainerId id;
+ private ContainerStatus status;
+ private boolean shouldDestroy = false;
+
+ public ComponentInstanceEvent(ContainerId containerId,
+ ComponentInstanceEventType componentInstanceEventType) {
+ super(componentInstanceEventType);
+ this.id = containerId;
+ }
+
+ public ContainerId getContainerId() {
+ return id;
+ }
+
+ public ContainerStatus getStatus() {
+ return this.status;
+ }
+
+ public ComponentInstanceEvent setStatus(ContainerStatus status) {
+ this.status = status;
+ return this;
+ }
+
+ public void setShouldDestroy() {
+ shouldDestroy = true;
+ }
+
+ public boolean shouldDestroy() {
+ return shouldDestroy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEventType.java
new file mode 100644
index 0000000..1a880ba
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEventType.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component.instance;
+
+public enum ComponentInstanceEventType {
+ START,
+ STOP,
+ BECOME_READY,
+ BECOME_NOT_READY
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceId.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceId.java
new file mode 100644
index 0000000..14387ba
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceId.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component.instance;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+public class ComponentInstanceId implements Comparable<ComponentInstanceId> {
+
+ private long Id;
+ private String name;
+ private ContainerId containerId;
+
+ public ComponentInstanceId(long id, String name) {
+ Id = id;
+ this.name = name;
+ }
+
+ public long getId() {
+ return Id;
+ }
+
+ public String getCompName() {
+ return name;
+ }
+
+ public String getCompInstanceName() {
+ return getCompName() + "-" + getId();
+ }
+
+ public void setContainerId(ContainerId containerId) {
+ this.containerId = containerId;
+ }
+
+ @Override
+ public String toString() {
+ if (containerId == null) {
+ return "[COMPINSTANCE " + getCompInstanceName() + "]";
+ } else {
+ return "[COMPINSTANCE " + getCompInstanceName() + " : " + containerId + "]";
+ }
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ ComponentInstanceId that = (ComponentInstanceId) o;
+
+ if (getId() != that.getId())
+ return false;
+ return getCompName() != null ? getCompName().equals(that.getCompName()) :
+ that.getCompName() == null;
+
+ }
+
+ @Override public int hashCode() {
+ int result = (int) (getId() ^ (getId() >>> 32));
+ result = 31 * result + (getCompName() != null ? getCompName().hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public int compareTo(ComponentInstanceId to) {
+ int delta = this.getCompName().compareTo(to.getCompName());
+ if (delta == 0) {
+ return Long.compare(this.getId(), to.getId());
+ } else if (delta < 0) {
+ return -1;
+ } else {
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceState.java
new file mode 100644
index 0000000..f5de5cb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceState.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component.instance;
+
+public enum ComponentInstanceState {
+ INIT,
+ STARTED,
+ READY,
+ UPGRADING
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
new file mode 100644
index 0000000..6de2dc0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+public interface RestApiConstants {
+
+ // Rest endpoints
+ String CONTEXT_ROOT = "/ws/v1";
+ String VERSION = "/services/version";
+ String SERVICE_ROOT_PATH = "/services";
+ String SERVICE_PATH = "/services/{service_name}";
+ String COMPONENT_PATH = "/services/{service_name}/components/{component_name}";
+
+ // Query param
+ String SERVICE_NAME = "service_name";
+ String COMPONENT_NAME = "component_name";
+
+ String DEFAULT_COMPONENT_NAME = "default";
+
+ String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST";
+ String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT";
+ Long DEFAULT_UNLIMITED_LIFETIME = -1l;
+
+ Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001;
+ Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002;
+ Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003;
+ Integer ERROR_CODE_APP_NAME_INVALID = 404004;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
new file mode 100644
index 0000000..ee270cb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
+
+public interface SliderExitCodes extends LauncherExitCodes {
+
+ /**
+ * starting point for exit codes; not an exception itself
+ */
+ int _EXIT_CODE_BASE = 64;
+
+ /**
+ * service entered the failed state: {@value}
+ */
+ int EXIT_YARN_SERVICE_FAILED = 65;
+
+ /**
+ * service was killed: {@value}
+ */
+ int EXIT_YARN_SERVICE_KILLED = 66;
+
+ /**
+ * timeout on monitoring client: {@value}
+ */
+ int EXIT_TIMED_OUT = 67;
+
+ /**
+ * service finished with an error: {@value}
+ */
+ int EXIT_YARN_SERVICE_FINISHED_WITH_ERROR = 68;
+
+ /**
+ * the service instance is unknown: {@value}
+ */
+ int EXIT_UNKNOWN_INSTANCE = 69;
+
+ /**
+ * the service instance is in the wrong state for that operation: {@value}
+ */
+ int EXIT_BAD_STATE = 70;
+
+ /**
+ * A spawned master process failed
+ */
+ int EXIT_PROCESS_FAILED = 71;
+
+ /**
+ * The instance failed -too many containers were
+ * failing or some other threshold was reached
+ */
+ int EXIT_DEPLOYMENT_FAILED = 72;
+
+ /**
+ * The service is live -and the requested operation
+ * does not work if the cluster is running
+ */
+ int EXIT_APPLICATION_IN_USE = 73;
+
+ /**
+ * There already is an service instance of that name
+ * when an attempt is made to create a new instance
+ */
+ int EXIT_INSTANCE_EXISTS = 75;
+
+ /**
+ * Exit code when the configurations in valid/incomplete: {@value}
+ */
+ int EXIT_BAD_CONFIGURATION = 77;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
new file mode 100644
index 0000000..1968e95
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+
+public class YarnServiceConf {
+
+ // Retry settings for the ServiceClient to talk to Service AppMaster
+ public static final String CLIENT_AM_RETRY_MAX_WAIT_MS = "yarn.service.client-am.retry.max-wait-ms";
+ public static final String CLIENT_AM_RETRY_MAX_INTERVAL_MS = "yarn.service.client-am.retry-interval-ms";
+
+ // Retry settings for container failures
+ public static final String CONTAINER_RETRY_MAX = "yarn.service.container-failure.retry.max";
+ public static final String CONTAINER_RETRY_INTERVAL = "yarn.service.container-failure.retry-interval";
+
+ public static final String AM_RESTART_MAX = "yarn.service.am-restart.max-attempts";
+ public static final String AM_RESOURCE_MEM = "yarn.service.am-resource.memory";
+ public static final long DEFAULT_KEY_AM_RESOURCE_MEM = 1024;
+
+ public static final String YARN_QUEUE = "yarn.service.queue";
+
+ public static final String API_SERVER_ADDRESS = "yarn.service.api-server.address";
+ public static final String DEFAULT_API_SERVER_ADDRESS = "0.0.0.0:";
+ public static final int DEFAULT_API_SERVER_PORT = 9191;
+
+ /**
+ * The yarn service base path:
+ * Defaults to HomeDir/.yarn/
+ */
+ public static final String YARN_SERVICE_BASE_PATH = "yarn.service.base.path";
+
+ //TODO rename
+ /** Declare that a keytab must be provided */
+ public static final String KEY_AM_LOGIN_KEYTAB_REQUIRED = "slider.am.login.keytab.required";
+ public static final String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
+ public static final String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir";
+ public static final String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
+
+ /**
+ * maximum number of failed containers (in a single component)
+ * before the app exits
+ */
+ public static final String CONTAINER_FAILURE_THRESHOLD =
+ "yarn.service.container-failure-per-component.threshold";
+ /**
+ * Maximum number of container failures on a node before the node is blacklisted
+ */
+ public static final String NODE_BLACKLIST_THRESHOLD =
+ "yarn.service.node-blacklist.threshold";
+
+ /**
+ * The failure count for CONTAINER_FAILURE_THRESHOLD and NODE_BLACKLIST_THRESHOLD
+ * gets reset periodically, the unit is seconds.
+ */
+ public static final String CONTAINER_FAILURE_WINDOW =
+ "yarn.service.failure-count-reset.window";
+
+ /**
+ * interval between readiness checks.
+ */
+ public static final String READINESS_CHECK_INTERVAL = "yarn.service.readiness-check-interval.seconds";
+ public static final int DEFAULT_READINESS_CHECK_INTERVAL = 30; // seconds
+
+ /**
+ * Get long value for the property. First get from the userConf, if not
+ * present, get from systemConf.
+ *
+ * @param name name of the property
+ * @param defaultValue default value of the property, if it is not defined in
+ * userConf and systemConf.
+ * @param userConf Configuration provided by client in the JSON definition
+ * @param systemConf The YarnConfiguration in the system.
+ * @return long value for the property
+ */
+ public static long getLong(String name, long defaultValue,
+ Configuration userConf, org.apache.hadoop.conf.Configuration systemConf) {
+ return userConf.getPropertyLong(name, systemConf.getLong(name, defaultValue));
+ }
+
+ public static int getInt(String name, int defaultValue,
+ Configuration userConf, org.apache.hadoop.conf.Configuration systemConf) {
+ return userConf.getPropertyInt(name, systemConf.getInt(name, defaultValue));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
new file mode 100644
index 0000000..e5ed703
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+public interface YarnServiceConstants {
+
+ /**
+ * The path under which cluster and temp data are stored
+ */
+ String SERVICE_BASE_DIRECTORY = ".yarn";
+
+ /**
+ * The paths under which Service AM dependency libraries are stored
+ */
+ String DEPENDENCY_LOCALIZED_DIR_LINK = "service_dep";
+ String DEPENDENCY_DIR = "/yarn-services/%s/";
+ String DEPENDENCY_TAR_GZ_FILE_NAME = "service-dep";
+ String DEPENDENCY_TAR_GZ_FILE_EXT = ".tar.gz";
+ String DEPENDENCY_DIR_PERMISSIONS = "755";
+
+ /**
+ * Service type for YARN service
+ */
+ String APP_TYPE = "yarn-service";
+
+ String KEYTAB_DIR = "keytabs";
+ String RESOURCE_DIR = "resources";
+
+
+ String SERVICES_DIRECTORY = "services";
+
+ /**
+ * JVM property to define the service lib directory;
+ * this is set by the yarn.sh script
+ */
+ String PROPERTY_LIB_DIR = "service.libdir";
+
+ /**
+ * name of generated dir for this conf
+ */
+ String SUBMITTED_CONF_DIR = "conf";
+
+ /**
+ * Service AM log4j file name
+ */
+ String YARN_SERVICE_LOG4J_FILENAME = "yarnservice-log4j.properties";
+
+ /**
+ * Log4j sysprop to name the resource
+ */
+ String SYSPROP_LOG4J_CONFIGURATION = "log4j.configuration";
+
+ /**
+ * sysprop for Service AM log4j directory
+ */
+ String SYSPROP_LOG_DIR = "LOG_DIR";
+
+ String TMP_DIR_PREFIX = "tmp";
+
+
+ String SERVICE_CORE_JAR = "yarn-service-core.jar";
+
+ String STDOUT_AM = "serviceam-out.txt";
+ String STDERR_AM = "serviceam-err.txt";
+
+ String HADOOP_USER_NAME = "HADOOP_USER_NAME";
+
+ String APP_CONF_DIR = "conf";
+
+ String APP_LIB_DIR = "lib";
+
+ String OUT_FILE = "stdout.txt";
+ String ERR_FILE = "stderr.txt";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
new file mode 100644
index 0000000..e4eae20
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
+import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.utils.CoreFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static org.apache.hadoop.yarn.service.provider.docker.DockerKeys.DEFAULT_DOCKER_NETWORK;
+
+/**
+ * Launcher of applications: base class
+ */
+public class AbstractLauncher {
+ private static final Logger log =
+ LoggerFactory.getLogger(AbstractLauncher.class);
+ public static final String CLASSPATH = "CLASSPATH";
+ /**
+ * Filesystem to use for the launch
+ */
+ protected final CoreFileSystem coreFileSystem;
+ /**
+ * Env vars; set up at final launch stage
+ */
+ protected final Map<String, String> envVars = new HashMap<>();
+ protected final ContainerLaunchContext containerLaunchContext =
+ Records.newRecord(ContainerLaunchContext.class);
+ protected final List<String> commands = new ArrayList<>(20);
+ protected final Map<String, LocalResource> localResources = new HashMap<>();
+ protected final Map<String, String> mountPaths = new HashMap<>();
+ private final Map<String, ByteBuffer> serviceData = new HashMap<>();
+ // security
+ protected final Credentials credentials;
+ protected boolean yarnDockerMode = false;
+ protected String dockerImage;
+ protected String dockerNetwork = DEFAULT_DOCKER_NETWORK;
+ protected String dockerHostname;
+ protected String runPrivilegedContainer;
+
+
+ /**
+ * Create instance.
+ * @param coreFileSystem filesystem
+ * @param credentials initial set of credentials -null is permitted
+ */
+ public AbstractLauncher(
+ CoreFileSystem coreFileSystem,
+ Credentials credentials) {
+ this.coreFileSystem = coreFileSystem;
+ this.credentials = credentials != null ? credentials: new Credentials();
+ }
+
+ public void setYarnDockerMode(boolean yarnDockerMode){
+ this.yarnDockerMode = yarnDockerMode;
+ }
+
+ /**
+ * Get the env vars to work on
+ * @return env vars
+ */
+ public Map<String, String> getEnv() {
+ return envVars;
+ }
+
+ /**
+ * Get the launch commands.
+ * @return the live list of commands
+ */
+ public List<String> getCommands() {
+ return commands;
+ }
+
+ public void addLocalResource(String subPath, LocalResource resource) {
+ localResources.put(subPath, resource);
+ }
+
+ public void addLocalResource(String subPath, LocalResource resource, String mountPath) {
+ localResources.put(subPath, resource);
+ mountPaths.put(subPath, mountPath);
+ }
+
+ /**
+ * Accessor to the credentials
+ * @return the credentials associated with this launcher
+ */
+ public Credentials getCredentials() {
+ return credentials;
+ }
+
+
+ public void addCommand(String cmd) {
+ commands.add(cmd);
+ }
+
+ /**
+ * Complete the launch context (copy in env vars, etc).
+ * @return the container to launch
+ */
+ public ContainerLaunchContext completeContainerLaunch() throws IOException {
+
+ String cmdStr = SliderUtils.join(commands, " ", false);
+ log.debug("Completed setting up container command {}", cmdStr);
+ containerLaunchContext.setCommands(commands);
+
+ //env variables
+ if (log.isDebugEnabled()) {
+ log.debug("Environment variables");
+ for (Map.Entry<String, String> envPair : envVars.entrySet()) {
+ log.debug(" \"{}\"=\"{}\"", envPair.getKey(), envPair.getValue());
+ }
+ }
+ containerLaunchContext.setEnvironment(envVars);
+
+ //service data
+ if (log.isDebugEnabled()) {
+ log.debug("Service Data size");
+ for (Map.Entry<String, ByteBuffer> entry : serviceData.entrySet()) {
+ log.debug("\"{}\"=> {} bytes of data", entry.getKey(),
+ entry.getValue().array().length);
+ }
+ }
+ containerLaunchContext.setServiceData(serviceData);
+
+ // resources
+ dumpLocalResources();
+ containerLaunchContext.setLocalResources(localResources);
+
+ //tokens
+ log.debug("{} tokens", credentials.numberOfTokens());
+ containerLaunchContext.setTokens(CredentialUtils.marshallCredentials(
+ credentials));
+
+ if(yarnDockerMode){
+ Map<String, String> env = containerLaunchContext.getEnvironment();
+ env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker");
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage);
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", dockerNetwork);
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME",
+ dockerHostname);
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer);
+ StringBuilder sb = new StringBuilder();
+ for (Entry<String,String> mount : mountPaths.entrySet()) {
+ if (sb.length() > 0) {
+ sb.append(",");
+ }
+ sb.append(mount.getKey());
+ sb.append(":");
+ sb.append(mount.getValue());
+ }
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS", sb.toString());
+ log.info("yarn docker env var has been set {}", containerLaunchContext.getEnvironment().toString());
+ }
+
+ return containerLaunchContext;
+ }
+
+ public void setRetryContext(int maxRetries, int retryInterval) {
+ ContainerRetryContext retryContext = ContainerRetryContext
+ .newInstance(ContainerRetryPolicy.RETRY_ON_ALL_ERRORS, null, maxRetries,
+ retryInterval);
+ containerLaunchContext.setContainerRetryContext(retryContext);
+ }
+
+ /**
+ * Dump local resources at debug level
+ */
+ private void dumpLocalResources() {
+ if (log.isDebugEnabled()) {
+ log.debug("{} resources: ", localResources.size());
+ for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) {
+
+ String key = entry.getKey();
+ LocalResource val = entry.getValue();
+ log.debug(key + "=" + SliderUtils.stringify(val.getResource()));
+ }
+ }
+ }
+
+ /**
+ * This is critical for an insecure cluster -it passes
+ * down the username to YARN, and so gives the code running
+ * in containers the rights it needs to work with
+ * data.
+ * @throws IOException problems working with current user
+ */
+ protected void propagateUsernameInInsecureCluster() throws IOException {
+ //insecure cluster: propagate user name via env variable
+ String userName = UserGroupInformation.getCurrentUser().getUserName();
+ envVars.put(YarnServiceConstants.HADOOP_USER_NAME, userName);
+ }
+
+ /**
+ * Utility method to set up the classpath
+ * @param classpath classpath to use
+ */
+ public void setClasspath(ClasspathConstructor classpath) {
+ setEnv(CLASSPATH, classpath.buildClasspath());
+ }
+
+ /**
+ * Set an environment variable in the launch context
+ * @param var variable name
+ * @param value value (must be non null)
+ */
+ public void setEnv(String var, String value) {
+ Preconditions.checkArgument(var != null, "null variable name");
+ Preconditions.checkArgument(value != null, "null value");
+ envVars.put(var, value);
+ }
+
+
+ public void putEnv(Map<String, String> map) {
+ envVars.putAll(map);
+ }
+
+
+ public void setDockerImage(String dockerImage) {
+ this.dockerImage = dockerImage;
+ }
+
+ public void setDockerNetwork(String dockerNetwork) {
+ this.dockerNetwork = dockerNetwork;
+ }
+
+ public void setDockerHostname(String dockerHostname) {
+ this.dockerHostname = dockerHostname;
+ }
+
+ public void setRunPrivilegedContainer(boolean runPrivilegedContainer) {
+ if (runPrivilegedContainer) {
+ this.runPrivilegedContainer = Boolean.toString(true);
+ } else {
+ this.runPrivilegedContainer = Boolean.toString(false);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
new file mode 100644
index 0000000..22b3877
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * build a classpath -allows for entries to be injected in front of
+ * YARN classpath as well as behind, adds appropriate separators,
+ * extraction of local classpath, etc.
+ */
+public class ClasspathConstructor {
+
+ public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR;
+ private final List<String> pathElements = new ArrayList<>();
+
+ public ClasspathConstructor() {
+ }
+
+
+ /**
+ * Get the list of JARs from the YARN settings
+ * @param config configuration
+ */
+ public List<String> yarnApplicationClasspath(Configuration config) {
+ String[] cp = config.getTrimmedStrings(
+ YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+ YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH);
+ return cp != null ? Arrays.asList(cp) : new ArrayList<String>(0);
+
+ }
+
+
+ @Override
+ public String toString() {
+ return buildClasspath();
+ }
+
+ public String buildClasspath() {
+ return SliderUtils.join(pathElements,
+ CLASS_PATH_SEPARATOR,
+ false);
+ }
+
+ /**
+ * Get a copy of the path list
+ * @return the JARs
+ */
+ public List<String> getPathElements() {
+ return Collections.unmodifiableList(pathElements);
+ }
+
+ /**
+ * Append an entry
+ * @param path path
+ */
+ public void append(String path) {
+ pathElements.add(path);
+ }
+
+ /**
+ * Insert a path at the front of the list. This places it ahead of
+ * the standard YARN artifacts
+ * @param path path to the JAR. Absolute or relative -on the target
+ * system
+ */
+ public void insert(String path) {
+ pathElements.add(0, path);
+ }
+
+ public void appendAll(Collection<String> paths) {
+ pathElements.addAll(paths);
+ }
+
+ public void insertAll(Collection<String> paths) {
+ pathElements.addAll(0, paths);
+ }
+
+
+ public void addLibDir(String pathToLibDir) {
+ append(buildLibDir(pathToLibDir));
+ }
+
+ public void insertLibDir(String pathToLibDir) {
+ insert(buildLibDir(pathToLibDir));
+ }
+
+ public void addClassDirectory(String pathToDir) {
+ append(appendDirectoryTerminator(pathToDir));
+ }
+
+ public void insertClassDirectory(String pathToDir) {
+ insert(buildLibDir(appendDirectoryTerminator(pathToDir)));
+ }
+
+
+ public void addRemoteClasspathEnvVar() {
+ append(ApplicationConstants.Environment.CLASSPATH.$$());
+ }
+
+
+ public void insertRemoteClasspathEnvVar() {
+ append(ApplicationConstants.Environment.CLASSPATH.$$());
+ }
+
+
+ /**
+ * Build a lib dir path
+ * @param pathToLibDir path to the directory; may or may not end with a
+ * trailing space
+ * @return a path to a lib dir that is compatible with the java classpath
+ */
+ public String buildLibDir(String pathToLibDir) {
+ String dir = appendDirectoryTerminator(pathToLibDir);
+ dir += "*";
+ return dir;
+ }
+
+ private String appendDirectoryTerminator(String pathToLibDir) {
+ String dir = pathToLibDir.trim();
+ if (!dir.endsWith("/")) {
+ dir += "/";
+ }
+ return dir;
+ }
+
+ /**
+ * Split a classpath. This uses the local path separator so MUST NOT
+ * be used to work with remote classpaths
+ * @param localpath local path
+ * @return a splite
+ */
+ public Collection<String> splitClasspath(String localpath) {
+ String separator = System.getProperty("path.separator");
+ return StringUtils.getStringCollection(localpath, separator);
+ }
+
+ /**
+ * Get the local JVM classpath split up
+ * @return the list of entries on the JVM classpath env var
+ */
+ public Collection<String> localJVMClasspath() {
+ return splitClasspath(System.getProperty("java.class.path"));
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[74/86] [abbrv] hadoop git commit: YARN-7113. Clean up packaging and
dependencies for yarn-native-services. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-7113. Clean up packaging and dependencies for yarn-native-services. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7bfd8815
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7bfd8815
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7bfd8815
Branch: refs/heads/yarn-native-services
Commit: 7bfd881503987a48a36db522ed6ce13ef2a1ccae
Parents: 4f8fe17
Author: Jian He <ji...@apache.org>
Authored: Tue Aug 29 11:09:00 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 25 16:37:23 2017 -0700
----------------------------------------------------------------------
NOTICE.txt | 14 +
.../resources/assemblies/hadoop-yarn-dist.xml | 8 -
.../assemblies/hadoop-yarn-services-api.xml | 36 -
.../assemblies/hadoop-yarn-services-dist.xml | 30 -
hadoop-project/pom.xml | 19 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 37 +-
.../hadoop-yarn-services-api/pom.xml | 104 +-
.../yarn/service/webapp/ApiServerWebApp.java | 4 +-
.../src/main/resources/webapps/api-server/app | 16 +
.../resources/webapps/services-rest-api/app | 16 -
.../hadoop-yarn-services-core/pom.xml | 213 +---
.../service/client/params/ActionKDiagArgs.java | 76 --
.../yarn/service/client/params/ClientArgs.java | 5 -
.../registry/YarnRegistryViewForProviders.java | 8 +-
.../yarn/service/utils/KerberosDiags.java | 680 -----------
.../hadoop/yarn/service/utils/SliderUtils.java | 1088 ------------------
.../hadoop/yarn/service/ServiceTestUtils.java | 28 +
.../hadoop/yarn/service/TestServiceApiUtil.java | 38 +-
.../yarn/service/TestYarnNativeServices.java | 10 +-
.../yarn/service/client/TestServiceCLI.java | 1 -
.../yarn/service/conf/TestAppJsonResolve.java | 30 +-
.../service/conf/TestLoadExampleAppJson.java | 11 +-
.../providers/TestAbstractClientProvider.java | 10 +-
hadoop-yarn-project/pom.xml | 4 +
24 files changed, 175 insertions(+), 2311 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/NOTICE.txt
----------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
index 0718909..f3af2f7 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -581,3 +581,17 @@ The binary distribution of this product bundles binaries of
Ehcache 3.3.1,
which has the following notices:
* Ehcache V3 Copyright 2014-2016 Terracotta, Inc.
+
+JCommander (https://github.com/cbeust/jcommander),
+which has the following notices:
+ * Copyright 2010 Cedric Beust cedric@beust.com
+
+The binary distribution of this product bundles binaries of
+snakeyaml (https://bitbucket.org/asomov/snakeyaml),
+which has the following notices:
+ * Copyright (c) 2008, http://www.snakeyaml.org
+
+The binary distribution of this product bundles binaries of
+swagger-annotations (https://github.com/swagger-api/swagger-core),
+which has the following notices:
+ * Copyright 2016 SmartBear Software
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 8aeeabd..8b3d292 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -98,10 +98,6 @@
<outputDirectory>etc/hadoop</outputDirectory>
</fileSet>
<fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target/hadoop-yarn-services-core-${project.version}</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/lib/services</outputDirectory>
- </fileSet>
- <fileSet>
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
<includes>
@@ -109,10 +105,6 @@
</includes>
</fileSet>
<fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target/hadoop-yarn-services-api-${project.version}</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/lib/services-api</outputDirectory>
- </fileSet>
- <fileSet>
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
<includes>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
deleted file mode 100644
index 589f724..0000000
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
+++ /dev/null
@@ -1,36 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.01
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
- <id>hadoop-yarn-services-api-dist</id>
- <formats>
- <format>dir</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <dependencySets>
- <dependencySet>
- <useProjectArtifact>false</useProjectArtifact>
- <includes>
- <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base</include>
- <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider</include>
- <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations</include>
- <include>io.swagger:swagger-annotations</include>
- </includes>
- </dependencySet>
- </dependencySets>
-</assembly>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml
deleted file mode 100644
index 1b81f98..0000000
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.01
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
- <id>hadoop-yarn-services-dist</id>
- <formats>
- <format>dir</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <dependencySets>
- <dependencySet>
- <useProjectArtifact>false</useProjectArtifact>
- </dependencySet>
- </dependencySets>
-</assembly>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 426dd62..1697744 100755
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -143,7 +143,8 @@
<declared.hadoop.version>${project.version}</declared.hadoop.version>
<swagger-annotations-version>1.5.4</swagger-annotations-version>
- <maven-doxia-module-markdown.version>1.4</maven-doxia-module-markdown.version>
+ <snakeyaml.version>1.16</snakeyaml.version>
+ <jcommander.version>1.30</jcommander.version>
</properties>
<dependencyManagement>
@@ -606,6 +607,11 @@
<version>3.1.0</version>
</dependency>
<dependency>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>jsr311-api</artifactId>
+ <version>1.1.1</version>
+ </dependency>
+ <dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>${jetty.version}</version>
@@ -1336,9 +1342,14 @@
<version>${jackson2.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-module-markdown</artifactId>
- <version>${maven-doxia-module-markdown.version}</version>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>${snakeyaml.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.beust</groupId>
+ <artifactId>jcommander</artifactId>
+ <version>${jcommander.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index c93ed41..43bcb6b 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -31,6 +31,7 @@ function hadoop_usage
hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
hadoop_add_option "--workers" "turn on worker mode"
+ hadoop_add_subcommand "apiserver" "run yarn-native-service rest server"
hadoop_add_subcommand "application" client "prints application(s) report/kill application"
hadoop_add_subcommand "applicationattempt" client "prints applicationattempt(s) report"
hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries"
@@ -48,9 +49,8 @@ function hadoop_usage
hadoop_add_subcommand "rmadmin" admin "admin tools"
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
- hadoop_add_subcommand "apiserver" "run yarn-native-service rest server"
- hadoop_add_subcommand "sharedcachemanager" admin "run the SharedCacheManager daemon"
hadoop_add_subcommand "service" "run a service"
+ hadoop_add_subcommand "sharedcachemanager" admin "run the SharedCacheManager daemon"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
hadoop_add_subcommand "timelineserver" daemon "run the timeline server"
hadoop_add_subcommand "top" client "view cluster information"
@@ -69,6 +69,18 @@ function yarncmd_case
shift
case ${subcmd} in
+ apiserver)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApiServerWebApp'
+ local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
+${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
+${HADOOP_HDFS_HOME}/${HDFS_DIR},\
+${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
+${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
+${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
+ hadoop_translate_cygwin_path sld
+ hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
+ ;;
application|applicationattempt|container)
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ApplicationCLI
set -- "${subcmd}" "$@"
@@ -147,14 +159,10 @@ function yarncmd_case
scmadmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
;;
- apiserver)
- HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services"'/*'
- hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*'
- HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApiServerWebApp'
+ service)
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.client.ServiceCLI'
local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
-${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services,\
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
@@ -166,19 +174,6 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
;;
- service)
- hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services"'/*'
- HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.client.ServiceCLI'
- local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
-${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
-${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services,\
-${HADOOP_HDFS_HOME}/${HDFS_DIR},\
-${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
-${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
-${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
- hadoop_translate_cygwin_path sld
- hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
- ;;
timelinereader)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index 7d9f15c..1077ccd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -23,7 +23,6 @@
</parent>
<artifactId>hadoop-yarn-services-api</artifactId>
<name>Apache Hadoop YARN Services API</name>
- <version>3.0.0-beta1-SNAPSHOT</version>
<packaging>jar</packaging>
<description>Hadoop YARN REST APIs for services</description>
@@ -46,7 +45,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
- <version>${maven-jar-plugin.version}</version>
<!-- The configuration of the plugin -->
<configuration>
<!-- Configuration of the archiver -->
@@ -59,9 +57,6 @@
<manifest>
</manifest>
</archive>
- <excludes>
- <exclude>**/run_rest_service.sh</exclude>
- </excludes>
</configuration>
<executions>
<execution>
@@ -92,97 +87,34 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-services-core</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
- <groupId>io.swagger</groupId>
- <artifactId>swagger-annotations</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
</dependency>
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
</dependency>
<dependency>
- <groupId>com.fasterxml.jackson.jaxrs</groupId>
- <artifactId>jackson-jaxrs-json-provider</artifactId>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>jsr311-api</artifactId>
</dependency>
</dependencies>
-
- <profiles>
- <profile>
- <id>dist</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-assemblies</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <id>dist</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <appendAssemblyId>false</appendAssemblyId>
- <attach>false</attach>
- <finalName>${project.artifactId}-${project.version}</finalName>
- <descriptorRefs>
- <descriptorRef>hadoop-yarn-services-api</descriptorRef>
- </descriptorRefs>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
-
- <profile>
- <id>rat</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>${apache-rat-plugin.version}</version>
- <executions>
- <execution>
- <id>check-licenses</id>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <excludes>
- <exclude>**/*.json</exclude>
- <exclude>**/THIRD-PARTY.properties</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
-
- </profiles>
-
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
index b226df7..fc65a63 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
@@ -59,8 +59,10 @@ public class ApiServerWebApp extends AbstractService {
public static void main(String[] args) throws IOException {
ApiServerWebApp apiWebApp = new ApiServerWebApp();
try {
- apiWebApp.startWebApp();
+ apiWebApp.init(new YarnConfiguration());
+ apiWebApp.serviceStart();
} catch (Exception e) {
+ logger.error("Got exception starting", e);
apiWebApp.close();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app
new file mode 100644
index 0000000..6a077b1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DON'T DELETE. REST WEBAPP RUN SCRIPT WILL STOP WORKING.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app
deleted file mode 100644
index 6a077b1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-DON'T DELETE. REST WEBAPP RUN SCRIPT WILL STOP WORKING.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index 1f8a408..d9b7adb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -104,7 +104,6 @@
<dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
- <version>1.30</version>
</dependency>
<dependency>
@@ -126,66 +125,51 @@
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-jaxrs</artifactId>
- <scope>compile</scope>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
- <scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-xc</artifactId>
- <scope>compile</scope>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
+ <artifactId>hadoop-hdfs-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
+ <artifactId>hadoop-yarn-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs-client</artifactId>
+ <artifactId>hadoop-yarn-registry</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- <scope>compile</scope>
+ <artifactId>hadoop-yarn-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-web-proxy</artifactId>
- <scope>compile</scope>
+ <artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-registry</artifactId>
- <scope>compile</scope>
+ <artifactId>hadoop-annotations</artifactId>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
</dependency>
<dependency>
@@ -195,13 +179,12 @@
<dependency>
<groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
+ <artifactId>commons-configuration2</artifactId>
</dependency>
<dependency>
- <groupId>commons-digester</groupId>
- <artifactId>commons-digester</artifactId>
- <version>1.8</version>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
</dependency>
<dependency>
@@ -215,37 +198,13 @@
</dependency>
<dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
</dependency>
<dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-servlets</artifactId>
- <version>3.0.1</version>
- </dependency>
-
- <!-- ======================================================== -->
- <!-- service registry -->
- <!-- ======================================================== -->
-
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </dependency>
-
- <!-- ======================================================== -->
- <!-- Jersey and webapp support -->
- <!-- ======================================================== -->
-
- <dependency>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
</dependency>
<dependency>
@@ -254,38 +213,23 @@
</dependency>
<dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
</dependency>
<dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
+ <groupId>io.swagger</groupId>
+ <artifactId>swagger-annotations</artifactId>
</dependency>
- <dependency>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-servlet</artifactId>
- </dependency>
+ <!-- ======================================================== -->
+ <!-- Test dependencies -->
+ <!-- ======================================================== -->
<dependency>
- <groupId>com.sun.jersey.contribs</groupId>
- <artifactId>jersey-guice</artifactId>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
@@ -295,114 +239,23 @@
</dependency>
<dependency>
- <groupId>org.easymock</groupId>
- <artifactId>easymock</artifactId>
- <version>3.1</version>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.objenesis</groupId>
- <artifactId>objenesis</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-easymock</artifactId>
- <version>1.6.5</version>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- <version>1.6.5</version>
- <exclusions>
- <exclusion>
- <groupId>org.javassist</groupId>
- <artifactId>javassist</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.objenesis</groupId>
- <artifactId>objenesis</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>javax.servlet.jsp</groupId>
- <artifactId>jsp-api</artifactId>
- <scope>runtime</scope>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jettison</groupId>
- <artifactId>jettison</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.yaml</groupId>
- <artifactId>snakeyaml</artifactId>
- <version>1.16</version>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>io.swagger</groupId>
- <artifactId>swagger-annotations</artifactId>
- <version>1.5.4</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
-
- <profiles>
- <profile>
- <id>dist</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-assemblies</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <id>dist</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <appendAssemblyId>false</appendAssemblyId>
- <attach>false</attach>
- <finalName>${project.artifactId}-${project.version}</finalName>
- <descriptorRefs>
- <descriptorRef>hadoop-yarn-services-dist</descriptorRef>
- </descriptorRefs>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
-
- </profiles>
-
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
deleted file mode 100644
index 061121e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-
-@Parameters(commandNames = { SliderActions.ACTION_KDIAG},
- commandDescription = SliderActions.DESCRIBE_ACTION_KDIAG)
-
-public class ActionKDiagArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_KDIAG;
- }
-
- @Parameter(names = {ARG_SERVICES}, variableArity = true,
- description =" list of services to check")
- public List<String> services = new ArrayList<>();
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for report")
- public File out;
-
- @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
- public File keytab;
-
- @Parameter(names = {ARG_KEYLEN}, description = "minimum key length")
- public int keylen = 256;
-
- @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
- public String principal;
-
- @Parameter(names = {ARG_SECURE}, description = "Is security required")
- public boolean secure = false;
-
- @Override
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
- if (keytab != null && SliderUtils.isUnset(principal)) {
- throw new UsageException("Missing argument " + ARG_PRINCIPAL);
- }
- if (keytab == null && SliderUtils.isSet(principal)) {
- throw new UsageException("Missing argument " + ARG_KEYTAB);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
index 7b957fa..09cae24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
@@ -47,7 +47,6 @@ public class ClientArgs extends CommonArgs {
private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs();
private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
- private final ActionKDiagArgs actionKDiagArgs = new ActionKDiagArgs();
private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs();
private final ActionListArgs actionListArgs = new ActionListArgs();
private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
@@ -207,10 +206,6 @@ public class ClientArgs extends CommonArgs {
bindCoreAction(actionHelpArgs);
break;
- case ACTION_KDIAG:
- bindCoreAction(actionKDiagArgs);
- break;
-
case ACTION_KEYTAB:
bindCoreAction(actionKeytabArgs);
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
index add2475..62d7a6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.service.registry;
import com.google.common.base.Preconditions;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -32,6 +30,8 @@ import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
@@ -43,8 +43,8 @@ import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join;
* is registered, offers access to the record and other things.
*/
public class YarnRegistryViewForProviders {
- private static final Log LOG =
- LogFactory.getLog(YarnRegistryViewForProviders.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(YarnRegistryViewForProviders.class);
private final RegistryOperations registryOperations;
private final String user;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
deleted file mode 100644
index c0712c3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
+++ /dev/null
@@ -1,680 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SaslPropertiesResolver;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.Shell;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.crypto.Cipher;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.lang.reflect.InvocationTargetException;
-import java.net.InetAddress;
-import java.security.NoSuchAlgorithmException;
-import java.util.Collection;
-import java.util.Date;
-import java.util.List;
-import java.util.regex.Pattern;
-
-import static org.apache.hadoop.security.UserGroupInformation.*;
-import static org.apache.hadoop.security.authentication.util.KerberosUtil.*;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
-
-/**
- * Kerberos diagnostics
- * At some point this may move to hadoop core, so please keep use of slider
- * methods and classes to ~0.
- *
- * This operation expands some of the diagnostic output of the security code,
- * but not all. For completeness
- *
- * Set the environment variable {@code HADOOP_JAAS_DEBUG=true}
- * Set the log level for {@code org.apache.hadoop.security=DEBUG}
- */
-public class KerberosDiags implements Closeable {
-
- private static final Logger LOG = LoggerFactory.getLogger(KerberosDiags.class);
- public static final String KRB5_CCNAME = "KRB5CCNAME";
- public static final String JAVA_SECURITY_KRB5_CONF
- = "java.security.krb5.conf";
- public static final String JAVA_SECURITY_KRB5_REALM
- = "java.security.krb5.realm";
- public static final String SUN_SECURITY_KRB5_DEBUG
- = "sun.security.krb5.debug";
- public static final String SUN_SECURITY_SPNEGO_DEBUG
- = "sun.security.spnego.debug";
- public static final String SUN_SECURITY_JAAS_FILE
- = "java.security.auth.login.config";
- public static final String KERBEROS_KINIT_COMMAND
- = "hadoop.kerberos.kinit.command";
- public static final String HADOOP_AUTHENTICATION_IS_DISABLED
- = "Hadoop authentication is disabled";
- public static final String UNSET = "(unset)";
- public static final String NO_DEFAULT_REALM = "Cannot locate default realm";
-
- private final Configuration conf;
- private final List<String> services;
- private final PrintStream out;
- private final File keytab;
- private final String principal;
- private final long minKeyLength;
- private final boolean securityRequired;
-
- public static final String CAT_JVM = "JVM";
- public static final String CAT_JAAS = "JAAS";
- public static final String CAT_CONFIG = "CONFIG";
- public static final String CAT_LOGIN = "LOGIN";
- public static final String CAT_KERBEROS = "KERBEROS";
- public static final String CAT_SASL = "SASL";
-
- @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
- public KerberosDiags(Configuration conf,
- PrintStream out,
- List<String> services,
- File keytab,
- String principal,
- long minKeyLength,
- boolean securityRequired) {
- this.conf = conf;
- this.services = services;
- this.keytab = keytab;
- this.principal = principal;
- this.out = out;
- this.minKeyLength = minKeyLength;
- this.securityRequired = securityRequired;
- }
-
- @Override
- public void close() throws IOException {
- flush();
- }
-
- /**
- * Execute diagnostics.
- * <p>
- * Things it would be nice if UGI made accessible
- * <ol>
- * <li>A way to enable JAAS debug programatically</li>
- * <li>Access to the TGT</li>
- * </ol>
- * @return true if security was enabled and all probes were successful
- * @throws KerberosDiagsFailure explicitly raised failure
- * @throws Exception other security problems
- */
- @SuppressWarnings("deprecation")
- public boolean execute() throws Exception {
-
- title("Kerberos Diagnostics scan at %s",
- new Date(System.currentTimeMillis()));
-
- // check that the machine has a name
- println("Hostname: %s",
- InetAddress.getLocalHost().getCanonicalHostName());
-
- // Fail fast on a JVM without JCE installed.
- validateKeyLength();
-
- // look at realm
- println("JVM Kerberos Login Module = %s", getKrb5LoginModuleName());
- printDefaultRealm();
-
- title("System Properties");
- for (String prop : new String[]{
- JAVA_SECURITY_KRB5_CONF,
- JAVA_SECURITY_KRB5_REALM,
- SUN_SECURITY_KRB5_DEBUG,
- SUN_SECURITY_SPNEGO_DEBUG,
- SUN_SECURITY_JAAS_FILE
- }) {
- printSysprop(prop);
- }
-
- title("Environment Variables");
- for (String env : new String[]{
- "HADOOP_JAAS_DEBUG",
- KRB5_CCNAME,
- "HADOOP_USER_NAME",
- "HADOOP_PROXY_USER",
- HADOOP_TOKEN_FILE_LOCATION,
- }) {
- printEnv(env);
- }
-
- for (String prop : new String[]{
- KERBEROS_KINIT_COMMAND,
- HADOOP_SECURITY_AUTHENTICATION,
- HADOOP_SECURITY_AUTHORIZATION,
- "hadoop.kerberos.min.seconds.before.relogin", // not in 2.6
- "hadoop.security.dns.interface", // not in 2.6
- "hadoop.security.dns.nameserver", // not in 2.6
- HADOOP_RPC_PROTECTION,
- HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
- HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX,
- HADOOP_SECURITY_GROUP_MAPPING,
- "hadoop.security.impersonation.provider.class", // not in 2.6
- "dfs.data.transfer.protection" // HDFS
- }) {
- printConfOpt(prop);
- }
-
- // check that authentication is enabled
- if (SecurityUtil.getAuthenticationMethod(conf)
- .equals(AuthenticationMethod.SIMPLE)) {
- println(HADOOP_AUTHENTICATION_IS_DISABLED);
- failif(securityRequired, CAT_CONFIG, HADOOP_AUTHENTICATION_IS_DISABLED);
- // no security, skip rest of test
- return false;
- }
-
- validateKrb5File();
- validateSasl(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS);
- validateSasl("dfs.data.transfer.saslproperties.resolver.class");
- validateKinitExecutable();
- validateJAAS();
- // now the big test: login, then try again
- boolean krb5Debug = getAndSet(SUN_SECURITY_KRB5_DEBUG);
- boolean spnegoDebug = getAndSet(SUN_SECURITY_SPNEGO_DEBUG);
- try {
- title("Logging in");
-
- if (keytab != null) {
- dumpKeytab(keytab);
- loginFromKeytab();
- } else {
- UserGroupInformation loginUser = getLoginUser();
- dumpUGI("Log in user", loginUser);
- validateUGI("Login user", loginUser);
- println("Ticket based login: %b", isLoginTicketBased());
- println("Keytab based login: %b", isLoginKeytabBased());
- }
-
- return true;
- } finally {
- // restore original system properties
- System.setProperty(SUN_SECURITY_KRB5_DEBUG,
- Boolean.toString(krb5Debug));
- System.setProperty(SUN_SECURITY_SPNEGO_DEBUG,
- Boolean.toString(spnegoDebug));
- }
- }
-
- /**
- * Fail fast on a JVM without JCE installed.
- *
- * This is a recurrent problem
- * (that is: it keeps creeping back with JVM updates);
- * a fast failure is the best tactic
- * @throws NoSuchAlgorithmException
- */
-
- protected void validateKeyLength() throws NoSuchAlgorithmException {
- int aesLen = Cipher.getMaxAllowedKeyLength("AES");
- println("Maximum AES encryption key length %d bits", aesLen);
- failif (aesLen < minKeyLength,
- CAT_JVM,
- "Java Cryptography Extensions are not installed on this JVM."
- +" Maximum supported key length %s - minimum required %d",
- aesLen, minKeyLength);
- }
-
- /**
- * Get the default realm.
- * <p>
- * Not having a default realm may be harmless, so is noted at info.
- * All other invocation failures are downgraded to warn, as
- * follow-on actions may still work.
- * failure to invoke the method via introspection is rejected,
- * as it's a sign of JVM compatibility issues that may have other
- * consequences
- */
- protected void printDefaultRealm() {
- try {
- println("Default Realm = %s",
- getDefaultRealm());
- } catch (ClassNotFoundException
- | IllegalAccessException
- | NoSuchMethodException e) {
-
- throw new KerberosDiagsFailure(CAT_JVM, e,
- "Failed to invoke krb5.Config.getDefaultRealm: %s", e);
- } catch (InvocationTargetException e) {
- Throwable cause = e.getCause() != null ? e.getCause() : e;
- if (cause.toString().contains(NO_DEFAULT_REALM)) {
- // exception raised if there is no default realm. This is not
- // always a problem, so downgrade to a message.
- println("Host has no default realm");
- LOG.debug(cause.toString(), cause);
- } else {
- println("Kerberos.getDefaultRealm() failed: %s\n%s",
- cause,
- org.apache.hadoop.util.StringUtils.stringifyException(cause));
- }
- }
- }
-
- /**
- * Locate the krb5.conf file and dump it.
- * No-op on windows.
- * @throws IOException
- */
- private void validateKrb5File() throws IOException {
- if (!Shell.WINDOWS) {
- title("Locating Kerberos configuration file");
- String krbPath = "/etc/krb5.conf";
- String jvmKrbPath = System.getProperty(JAVA_SECURITY_KRB5_CONF);
- if (jvmKrbPath != null) {
- println("Setting kerberos path from sysprop %s: %s",
- JAVA_SECURITY_KRB5_CONF, jvmKrbPath);
- krbPath = jvmKrbPath;
- }
-
- String krb5name = System.getenv(KRB5_CCNAME);
- if (krb5name != null) {
- println("Setting kerberos path from environment variable %s: %s",
- KRB5_CCNAME, krb5name);
- krbPath = krb5name;
- if (jvmKrbPath != null) {
- println("Warning - both %s and %s were set - %s takes priority",
- JAVA_SECURITY_KRB5_CONF, KRB5_CCNAME, KRB5_CCNAME);
- }
- }
-
- File krbFile = new File(krbPath);
- println("Kerberos configuration file = %s", krbFile);
- failif(!krbFile.exists(),
- CAT_KERBEROS,
- "Kerberos configuration file %s not found", krbFile);
- dump(krbFile);
- }
- }
-
- /**
- * Dump a keytab: list all principals.
- * @param keytabFile the keytab file
- * @throws IOException IO problems
- */
- public void dumpKeytab(File keytabFile) throws IOException {
- title("Examining keytab %s", keytabFile);
- File kt = keytabFile.getCanonicalFile();
- failif(!kt.exists(), CAT_CONFIG, "Keytab not found: %s", kt);
- failif(!kt.isFile(), CAT_CONFIG, "Keytab is not a valid file: %s", kt);
-
- String[] names = getPrincipalNames(keytabFile.getCanonicalPath(),
- Pattern.compile(".*"));
- println("keytab entry count: %d", names.length);
- for (String name : names) {
- println(" %s", name);
- }
- println("-----");
- }
-
- /**
- * Log in from a keytab, dump the UGI, validate it, then try and log in again.
- * That second-time login catches JVM/Hadoop compatibility problems.
- * @throws IOException
- */
- private void loginFromKeytab() throws IOException {
- UserGroupInformation ugi;
- String identity;
- if (keytab != null) {
- File kt = keytab.getCanonicalFile();
- println("Using keytab %s principal %s", kt, principal);
- identity = principal;
-
- failif(StringUtils.isEmpty(principal), CAT_KERBEROS,
- "No principal defined");
- ugi = loginUserFromKeytabAndReturnUGI(principal, kt.getPath());
- dumpUGI(identity, ugi);
- validateUGI(principal, ugi);
-
- title("Attempting to log in from keytab again");
- // package scoped -hence the reason why this class must be in the
- // hadoop.security package
- setShouldRenewImmediatelyForTests(true);
- // attempt a new login
- ugi.reloginFromKeytab();
- } else {
- println("No keytab: logging is as current user");
- }
- }
-
- /**
- * Dump a UGI.
- * @param title title of this section
- * @param ugi UGI to dump
- * @throws IOException
- */
- private void dumpUGI(String title, UserGroupInformation ugi)
- throws IOException {
- title(title);
- println("UGI instance = %s", ugi);
- println("Has kerberos credentials: %b", ugi.hasKerberosCredentials());
- println("Authentication method: %s", ugi.getAuthenticationMethod());
- println("Real Authentication method: %s",
- ugi.getRealAuthenticationMethod());
- title("Group names");
- for (String name : ugi.getGroupNames()) {
- println(name);
- }
- title("Credentials");
- Credentials credentials = ugi.getCredentials();
- List<Text> secretKeys = credentials.getAllSecretKeys();
- title("Secret keys");
- if (!secretKeys.isEmpty()) {
- for (Text secret: secretKeys) {
- println("%s", secret);
- }
- } else {
- println("(none)");
- }
-
- dumpTokens(ugi);
- }
-
- /**
- * Validate the UGI: verify it is kerberized.
- * @param messagePrefix message in exceptions
- * @param user user to validate
- */
- private void validateUGI(String messagePrefix, UserGroupInformation user) {
- failif(!user.hasKerberosCredentials(),
- CAT_LOGIN, "%s: No kerberos credentials for %s", messagePrefix, user);
- failif(user.getAuthenticationMethod() == null,
- CAT_LOGIN, "%s: Null AuthenticationMethod for %s", messagePrefix, user);
- }
-
- /**
- * A cursory look at the {@code kinit} executable.
- * If it is an absolute path: it must exist with a size > 0.
- * If it is just a command, it has to be on the path. There's no check
- * for that -but the PATH is printed out.
- */
- private void validateKinitExecutable() {
- String kinit = conf.getTrimmed(KERBEROS_KINIT_COMMAND, "");
- if (!kinit.isEmpty()) {
- File kinitPath = new File(kinit);
- println("%s = %s", KERBEROS_KINIT_COMMAND, kinitPath);
- if (kinitPath.isAbsolute()) {
- failif(!kinitPath.exists(), CAT_KERBEROS,
- "%s executable does not exist: %s",
- KERBEROS_KINIT_COMMAND, kinitPath);
- failif(!kinitPath.isFile(), CAT_KERBEROS,
- "%s path does not refer to a file: %s",
- KERBEROS_KINIT_COMMAND, kinitPath);
- failif(kinitPath.length() == 0, CAT_KERBEROS,
- "%s file is empty: %s",
- KERBEROS_KINIT_COMMAND, kinitPath);
- } else {
- println("Executable %s is relative -must be on the PATH", kinit);
- printEnv("PATH");
- }
- }
- }
-
- /**
- * Try to load the SASL resolver.
- * @param saslPropsResolverKey key for the SASL resolver
- */
- private void validateSasl(String saslPropsResolverKey) {
- title("Resolving SASL property %s", saslPropsResolverKey);
- String saslPropsResolver = conf.getTrimmed(saslPropsResolverKey);
- try {
- Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass(
- saslPropsResolverKey,
- SaslPropertiesResolver.class, SaslPropertiesResolver.class);
- println("Resolver is %s", resolverClass);
- } catch (RuntimeException e) {
- throw new KerberosDiagsFailure(CAT_SASL, e,
- "Failed to load %s class %s",
- saslPropsResolverKey, saslPropsResolver);
- }
- }
-
- /**
- * Validate any JAAS entry referenced in the {@link #SUN_SECURITY_JAAS_FILE}
- * property.
- */
- private void validateJAAS() {
- String jaasFilename = System.getProperty(SUN_SECURITY_JAAS_FILE);
- if (jaasFilename != null) {
- title("JAAS");
- File jaasFile = new File(jaasFilename);
- println("JAAS file is defined in %s: %s",
- SUN_SECURITY_JAAS_FILE, jaasFile);
- failif(!jaasFile.exists(), CAT_JAAS,
- "JAAS file does not exist: %s", jaasFile);
- failif(!jaasFile.isFile(), CAT_JAAS,
- "Specified JAAS file is not a file: %s", jaasFile);
- }
- }
-
- /**
- * Dump all tokens of a user
- * @param user user
- */
- public void dumpTokens(UserGroupInformation user) {
- Collection<Token<? extends TokenIdentifier>> tokens
- = user.getCredentials().getAllTokens();
- title("Token Count: %d", tokens.size());
- for (Token<? extends TokenIdentifier> token : tokens) {
- println("Token %s", token.getKind());
- }
- }
-
- /**
- * Set the System property to true; return the old value for caching
- * @param sysprop property
- * @return the previous value
- */
- private boolean getAndSet(String sysprop) {
- boolean old = Boolean.getBoolean(sysprop);
- System.setProperty(sysprop, "true");
- return old;
- }
-
- /**
- * Flush all active output channels, including {@Code System.err},
- * so as to stay in sync with any JRE log messages.
- */
- private void flush() {
- if (out != null) {
- out.flush();
- } else {
- System.out.flush();
- }
- System.err.flush();
- }
-
- /**
- * Format and print a line of output.
- * This goes to any output file, or
- * is logged at info. The output is flushed before and after, to
- * try and stay in sync with JRE logging.
- * @param format format string
- * @param args any arguments
- */
- @VisibleForTesting
- public void println(String format, Object... args) {
- println(format(format, args));
- }
-
- /**
- * Print a line of output. This goes to any output file, or
- * is logged at info. The output is flushed before and after, to
- * try and stay in sync with JRE logging.
- * @param msg message string
- */
- @VisibleForTesting
- private void println(String msg) {
- flush();
- if (out != null) {
- out.println(msg);
- } else {
- LOG.info(msg);
- }
- flush();
- }
-
- /**
- * Print a title entry
- * @param format format string
- * @param args any arguments
- */
- private void title(String format, Object... args) {
- println("");
- println("");
- String msg = "== " + format(format, args) + " ==";
- println(msg);
- println("");
- }
-
- /**
- * Print a system property, or {@link #UNSET} if unset.
- * @param property property to print
- */
- private void printSysprop(String property) {
- println("%s = \"%s\"", property,
- System.getProperty(property, UNSET));
- }
-
- /**
- * Print a configuration option, or {@link #UNSET} if unset.
- * @param option option to print
- */
- private void printConfOpt(String option) {
- println("%s = \"%s\"", option, conf.get(option, UNSET));
- }
-
- /**
- * Print an environment variable's name and value; printing
- * {@link #UNSET} if it is not set
- * @param variable environment variable
- */
- private void printEnv(String variable) {
- String env = System.getenv(variable);
- println("%s = \"%s\"", variable, env != null ? env : UNSET);
- }
-
- /**
- * Dump any file to standard out; add a trailing newline
- * @param file file to dump
- * @throws IOException IO problems
- */
- public void dump(File file) throws IOException {
- try (FileInputStream in = new FileInputStream(file)) {
- for (String line : IOUtils.readLines(in)) {
- println("%s", line);
- }
- }
- println("");
- }
-
- /**
- * Format and raise a failure
- *
- * @param category category for exception
- * @param message string formatting message
- * @param args any arguments for the formatting
- * @throws KerberosDiagsFailure containing the formatted text
- */
- private void fail(String category, String message, Object... args)
- throws KerberosDiagsFailure {
- throw new KerberosDiagsFailure(category, message, args);
- }
-
- /**
- * Conditional failure with string formatted arguments
- * @param condition failure condition
- * @param category category for exception
- * @param message string formatting message
- * @param args any arguments for the formatting
- * @throws KerberosDiagsFailure containing the formatted text
- * if the condition was met
- */
- private void failif(boolean condition,
- String category,
- String message,
- Object... args)
- throws KerberosDiagsFailure {
- if (condition) {
- fail(category, message, args);
- }
- }
-
- /**
- * Format a string, treating a call where there are no varags values
- * as a string to pass through unformatted.
- * @param message message, which is either a format string + args, or
- * a general string
- * @param args argument array
- * @return a string for printing.
- */
- public static String format(String message, Object... args) {
- if (args.length == 0) {
- return message;
- } else {
- return String.format(message, args);
- }
- }
-
- /**
- * Diagnostics failures return the exit code 41, "unauthorized".
- *
- * They have a category, initially for testing: the category can be
- * validated without having to match on the entire string.
- */
- public static class KerberosDiagsFailure extends ExitUtil.ExitException {
- private final String category;
-
- public KerberosDiagsFailure(String category, String message) {
- super(41, category + ": " + message);
- this.category = category;
- }
-
- public KerberosDiagsFailure(String category, String message, Object... args) {
- this(category, format(message, args));
- }
-
- public KerberosDiagsFailure(String category, Throwable throwable,
- String message, Object... args) {
- this(category, message, args);
- initCause(throwable);
- }
-
- public String getCategory() {
- return category;
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[58/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
new file mode 100644
index 0000000..b742553
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
+import org.apache.hadoop.yarn.client.api.TimelineV2Client;
+import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
+import org.apache.hadoop.yarn.service.api.records.PlacementPolicy;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test class for ServiceTimelinePublisher.
+ */
+public class TestServiceTimelinePublisher {
+ private TimelineV2Client timelineClient;
+ private Configuration config;
+ private ServiceTimelinePublisher serviceTimelinePublisher;
+ private static String SERVICE_NAME = "HBASE";
+ private static String SERVICEID = "application_1490093646524_0005";
+ private static String ARTIFACTID = "ARTIFACTID";
+ private static String COMPONENT_NAME = "DEFAULT";
+ private static String CONTAINER_ID =
+ "container_e02_1490093646524_0005_01_000001";
+ private static String CONTAINER_IP =
+ "localhost";
+ private static String CONTAINER_HOSTNAME =
+ "cnl124-localhost.site";
+ private static String CONTAINER_BAREHOST =
+ "localhost.com";
+
+ @Before
+ public void setUp() throws Exception {
+ config = new Configuration();
+ config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+ timelineClient =
+ new DummyTimelineClient(ApplicationId.fromString(SERVICEID));
+ serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
+ timelineClient.init(config);
+ serviceTimelinePublisher.init(config);
+ timelineClient.start();
+ serviceTimelinePublisher.start();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (serviceTimelinePublisher != null) {
+ serviceTimelinePublisher.stop();
+ }
+ if (timelineClient != null) {
+ timelineClient.stop();
+ }
+ }
+
+ @Test
+ public void testServiceAttemptEntity() {
+ Service service = createMockApplication();
+ serviceTimelinePublisher
+ .serviceAttemptRegistered(service, new YarnConfiguration());
+
+ Collection<TimelineEntity> lastPublishedEntities =
+ ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
+ // 2 entities because during registration component also registered.
+ assertEquals(2, lastPublishedEntities.size());
+ for (TimelineEntity timelineEntity : lastPublishedEntities) {
+ if (timelineEntity.getType() == ServiceTimelineEntityType.COMPONENT
+ .toString()) {
+ verifyComponentTimelineEntity(timelineEntity);
+ } else {
+ verifyServiceAttemptTimelineEntity(timelineEntity, null, true);
+ }
+ }
+
+ ServiceContext context = new ServiceContext();
+ context.attemptId = ApplicationAttemptId
+ .newInstance(ApplicationId.fromString(service.getId()), 1);
+ String exitDiags = "service killed";
+ serviceTimelinePublisher.serviceAttemptUnregistered(context, exitDiags);
+ lastPublishedEntities =
+ ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
+ for (TimelineEntity timelineEntity : lastPublishedEntities) {
+ if (timelineEntity.getType() == ServiceTimelineEntityType.SERVICE_ATTEMPT
+ .toString()) {
+ verifyServiceAttemptTimelineEntity(timelineEntity, exitDiags,
+ false);
+ }
+ }
+ }
+
+ @Test
+ public void testComponentInstanceEntity() {
+ Container container = new Container();
+ container.id(CONTAINER_ID).ip(CONTAINER_IP).bareHost(CONTAINER_BAREHOST)
+ .hostname(CONTAINER_HOSTNAME).state(ContainerState.RUNNING_BUT_UNREADY)
+ .launchTime(new Date());
+ ComponentInstanceId id = new ComponentInstanceId(0, COMPONENT_NAME);
+ ComponentInstance instance = mock(ComponentInstance.class);
+ when(instance.getCompName()).thenReturn(COMPONENT_NAME);
+ when(instance.getCompInstanceName()).thenReturn("comp_instance_name");
+ serviceTimelinePublisher.componentInstanceStarted(container,
+ instance);
+
+ Collection<TimelineEntity> lastPublishedEntities =
+ ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
+ assertEquals(1, lastPublishedEntities.size());
+ TimelineEntity entity = lastPublishedEntities.iterator().next();
+
+ assertEquals(1, entity.getEvents().size());
+ assertEquals(CONTAINER_ID, entity.getId());
+ assertEquals(CONTAINER_BAREHOST,
+ entity.getInfo().get(ServiceTimelineMetricsConstants.BARE_HOST));
+ assertEquals(COMPONENT_NAME,
+ entity.getInfo().get(ServiceTimelineMetricsConstants.COMPONENT_NAME));
+ assertEquals(ContainerState.RUNNING_BUT_UNREADY.toString(),
+ entity.getInfo().get(ServiceTimelineMetricsConstants.STATE));
+
+ // updated container state
+ container.setState(ContainerState.READY);
+ serviceTimelinePublisher.componentInstanceUpdated(container);
+ lastPublishedEntities =
+ ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
+ assertEquals(1, lastPublishedEntities.size());
+ entity = lastPublishedEntities.iterator().next();
+ assertEquals(2, entity.getEvents().size());
+ assertEquals(ContainerState.READY.toString(),
+ entity.getInfo().get(ServiceTimelineMetricsConstants.STATE));
+
+ }
+
+ private void verifyServiceAttemptTimelineEntity(TimelineEntity timelineEntity,
+ String message, boolean isRegistedEntity) {
+ assertEquals(SERVICEID, timelineEntity.getId());
+ assertEquals(SERVICE_NAME,
+ timelineEntity.getInfo().get(ServiceTimelineMetricsConstants.NAME));
+ if (isRegistedEntity) {
+ assertEquals(ServiceState.STARTED.toString(),
+ timelineEntity.getInfo().get(ServiceTimelineMetricsConstants.STATE));
+ assertEquals(ServiceTimelineEvent.SERVICE_ATTEMPT_REGISTERED.toString(),
+ timelineEntity.getEvents().iterator().next().getId());
+ } else {
+ assertEquals("ENDED",
+ timelineEntity.getInfo().get(ServiceTimelineMetricsConstants.STATE).toString());
+ assertEquals(message, timelineEntity.getInfo()
+ .get(ServiceTimelineMetricsConstants.DIAGNOSTICS_INFO));
+ assertEquals(2, timelineEntity.getEvents().size());
+ assertEquals(ServiceTimelineEvent.SERVICE_ATTEMPT_UNREGISTERED.toString(),
+ timelineEntity.getEvents().iterator().next().getId());
+ }
+ }
+
+ private void verifyComponentTimelineEntity(TimelineEntity entity) {
+ Map<String, Object> info = entity.getInfo();
+ assertEquals("DEFAULT", entity.getId());
+ assertEquals(ARTIFACTID,
+ info.get(ServiceTimelineMetricsConstants.ARTIFACT_ID));
+ assertEquals("DOCKER",
+ info.get(ServiceTimelineMetricsConstants.ARTIFACT_TYPE));
+ assertEquals("medium",
+ info.get(ServiceTimelineMetricsConstants.RESOURCE_PROFILE));
+ assertEquals(1, info.get(ServiceTimelineMetricsConstants.RESOURCE_CPU));
+ assertEquals("1024",
+ info.get(ServiceTimelineMetricsConstants.RESOURCE_MEMORY));
+ assertEquals("sleep 1",
+ info.get(ServiceTimelineMetricsConstants.LAUNCH_COMMAND));
+ assertEquals("false",
+ info.get(ServiceTimelineMetricsConstants.RUN_PRIVILEGED_CONTAINER));
+ assertEquals("label",
+ info.get(ServiceTimelineMetricsConstants.PLACEMENT_POLICY));
+ }
+
+ private static Service createMockApplication() {
+ Service service = mock(Service.class);
+
+ when(service.getId()).thenReturn(SERVICEID);
+ when(service.getLaunchTime()).thenReturn(new Date());
+ when(service.getState()).thenReturn(ServiceState.STARTED);
+ when(service.getName()).thenReturn(SERVICE_NAME);
+ when(service.getConfiguration()).thenReturn(
+ new org.apache.hadoop.yarn.service.api.records.Configuration());
+
+ Component component = mock(Component.class);
+ Artifact artifact = new Artifact();
+ artifact.setId(ARTIFACTID);
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory(1024 + "");
+ resource.setProfile("medium");
+ when(component.getArtifact()).thenReturn(artifact);
+ when(component.getName()).thenReturn(COMPONENT_NAME);
+ when(component.getResource()).thenReturn(resource);
+ when(component.getLaunchCommand()).thenReturn("sleep 1");
+ PlacementPolicy placementPolicy = new PlacementPolicy();
+ placementPolicy.setLabel("label");
+ when(component.getPlacementPolicy()).thenReturn(placementPolicy);
+ when(component.getConfiguration()).thenReturn(
+ new org.apache.hadoop.yarn.service.api.records.Configuration());
+ List<Component> components = new ArrayList<Component>();
+ components.add(component);
+
+ when(service.getComponents()).thenReturn(components);
+ return service;
+ }
+
+ protected static class DummyTimelineClient extends TimelineV2ClientImpl {
+ private Map<Identifier, TimelineEntity> lastPublishedEntities =
+ new HashMap<>();
+
+ public DummyTimelineClient(ApplicationId appId) {
+ super(appId);
+ }
+
+ @Override
+ public void putEntitiesAsync(TimelineEntity... entities)
+ throws IOException, YarnException {
+ putEntities(entities);
+ }
+
+ @Override
+ public void putEntities(TimelineEntity... entities)
+ throws IOException, YarnException {
+ for (TimelineEntity timelineEntity : entities) {
+ TimelineEntity entity =
+ lastPublishedEntities.get(timelineEntity.getIdentifier());
+ if (entity == null) {
+ lastPublishedEntities.put(timelineEntity.getIdentifier(),
+ timelineEntity);
+ } else {
+ entity.addMetrics(timelineEntity.getMetrics());
+ entity.addEvents(timelineEntity.getEvents());
+ entity.addInfo(timelineEntity.getInfo());
+ entity.addConfigs(timelineEntity.getConfigs());
+ entity.addRelatesToEntities(timelineEntity.getRelatesToEntities());
+ entity
+ .addIsRelatedToEntities(timelineEntity.getIsRelatedToEntities());
+ }
+ }
+ }
+
+ public Collection<TimelineEntity> getLastPublishedEntities() {
+ return lastPublishedEntities.values();
+ }
+
+ public void reset() {
+ lastPublishedEntities = null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/example-app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/example-app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/example-app.json
new file mode 100644
index 0000000..5dfbd64
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/example-app.json
@@ -0,0 +1,15 @@
+{
+ "name": "example-app",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
new file mode 100644
index 0000000..d7e2fd0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
@@ -0,0 +1,72 @@
+{
+ "name": "app-1",
+ "lifetime": "3600",
+ "launch_command": "sleep 3600",
+ "configuration": {
+ "properties": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "files": [
+ {
+ "type": "PROPERTIES",
+ "dest_file": "file1",
+ "props": {
+ "k1": "v1",
+ "k2": "v2"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "file2",
+ "props": {
+ "k3": "v3"
+ }
+ }
+ ]
+ },
+ "resource": {
+ "cpus": 1,
+ "memory": "512"
+ },
+ "number_of_containers": 2,
+ "components": [
+ {
+ "name": "simple",
+ "configuration": {
+ "files": [
+ {
+ "type": "PROPERTIES",
+ "dest_file": "file1",
+ "props": {
+ "k1": "overridden"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "master",
+ "configuration": {
+ "properties": {
+ "name": "m",
+ "g1": "overridden"
+ }
+ }
+ },
+ {
+ "name": "worker",
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "configuration": {
+ "properties": {
+ "name": "worker",
+ "g1": "overridden-by-worker",
+ "timeout": "1000"
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
new file mode 100644
index 0000000..12b51e4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
@@ -0,0 +1,48 @@
+{
+ "name": "service-1",
+ "id" : "application_1503358878042_0011",
+ "lifetime": "3600",
+ "launch_command": "sleep 3600",
+ "configuration": {
+ "properties": {
+ "g1": "a",
+ "g2": "b",
+ "yarn.service.failure-count-reset.window": "60"
+ }
+ },
+ "resource": {
+ "cpus": 1,
+ "memory": "512"
+ },
+ "number_of_containers": 2,
+ "components": [
+ {
+ "name": "simple"
+ },
+ {
+ "name": "master",
+ "number_of_containers": 1,
+ "configuration": {
+ "properties": {
+ "g1": "overridden",
+ "g3": "will-be-overridden",
+ "jvm.heapsize": "512M"
+ }
+ }
+ },
+ {
+ "name": "worker",
+ "number_of_containers": 5,
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "configuration": {
+ "properties": {
+ "g1": "overridden-by-worker",
+ "jvm.heapsize": "512M"
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json
new file mode 100644
index 0000000..73d4e7b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json
@@ -0,0 +1,16 @@
+{
+ "name": "default-app-1",
+ "lifetime": "3600",
+ "components" :
+ [
+ {
+ "name": "sleep",
+ "number_of_containers": 1,
+ "launch_command": "sleep 3600",
+ "resource": {
+ "cpus": 2,
+ "memory": "256"
+ }
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
new file mode 100644
index 0000000..0857f62
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
@@ -0,0 +1,8 @@
+{
+ "name": "external-0",
+ "lifetime": "3600",
+ "artifact": {
+ "type": "SERVICE",
+ "id": "app-1"
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json
new file mode 100644
index 0000000..4afdb8b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json
@@ -0,0 +1,30 @@
+{
+ "name": "external-1",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "simple",
+ "artifact": {
+ "type": "SERVICE",
+ "id": "app-1"
+ }
+ },
+ {
+ "name": "master",
+ "configuration": {
+ "properties": {
+ "g3": "is-overridden"
+ }
+ }
+ },
+ {
+ "name": "other",
+ "launch_command": "sleep 3600",
+ "number_of_containers": 2,
+ "resource": {
+ "cpus": 1,
+ "memory": "512"
+ }
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json
new file mode 100644
index 0000000..0df8e0a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json
@@ -0,0 +1,22 @@
+{
+ "name": "external-2",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "ext",
+ "artifact": {
+ "type": "SERVICE",
+ "id": "external-1"
+ }
+ },
+ {
+ "name": "another",
+ "launch_command": "sleep 3600",
+ "number_of_containers": 1,
+ "resource": {
+ "cpus": 1,
+ "memory": "512"
+ }
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/yarn-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/yarn-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/yarn-site.xml
new file mode 100644
index 0000000..266caa9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/yarn-site.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+ <!-- Dummy (invalid) config file to be overwriten by TestYarnNativeServices with MiniCluster configuration. -->
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
new file mode 100644
index 0000000..1233804
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>hadoop-yarn-applications</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <version>3.0.0-beta1-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hadoop-yarn-services</artifactId>
+ <name>Apache Hadoop YARN Services</name>
+ <packaging>pom</packaging>
+
+ <properties>
+ <hadoop.common.build.dir>${basedir}/../../../../hadoop-common-project/hadoop-common/target</hadoop.common.build.dir>
+ </properties>
+
+ <!-- Do not add dependencies here, add them to the POM of the leaf module -->
+
+ <modules>
+ <module>hadoop-yarn-services-core</module>
+ </modules>
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
deleted file mode 100644
index 2814cca..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
+++ /dev/null
@@ -1,48 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<FindBugsFilter>
- <Match>
- <Package name="org.apache.hadoop.yarn.proto" />
- </Match>
- <Match>
- <class name="org.apache.hadoop.yarn.service.utils.ServiceApiUtil" />
- <Bug pattern="MS_CANNOT_BE_FINAL" />
- </Match>
- <Match>
- <Class name="org.apache.hadoop.yarn.service.utils.JsonSerDeser" />
- <Bug pattern="OBL_UNSATISFIED_OBLIGATION" />
- </Match>
- <Match>
- <Class name="org.apache.hadoop.yarn.service.utils.JsonSerDeser" />
- <Bug pattern="UI_INHERITANCE_UNSAFE_GETRESOURCE" />
- </Match>
- <Match>
- <Package name="org.apache.hadoop.yarn.service.client.params"/>
- <Bug pattern="UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD"/>
- </Match>
- <Match>
- <Package name="org.apache.hadoop.yarn.service.client.params"/>
- <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"/>
- </Match>
- <Match>
- <Class name="org.apache.hadoop.yarn.service.client.ServiceClient"/>
- <Field name="registryClient" />
- <Bug pattern="IS2_INCONSISTENT_SYNC"/>
- </Match>
-
-</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
deleted file mode 100644
index 58c8e27..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
+++ /dev/null
@@ -1,62 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This is the log4j configuration for Slider Application Master
-
-# Log rotation based on size (256MB) with a max of 20 backup files
-log4j.rootLogger=INFO, amlog
-log4j.threshhold=ALL
-log4j.appender.amlog=org.apache.log4j.RollingFileAppender
-log4j.appender.amlog.layout=org.apache.log4j.PatternLayout
-log4j.appender.amlog.File=${LOG_DIR}/serviceam.log
-log4j.appender.amlog.MaxFileSize=256MB
-log4j.appender.amlog.MaxBackupIndex=20
-
-# log layout skips stack-trace creation operations by avoiding line numbers and method
-log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
-
-# debug edition is much more expensive
-#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
-
-# configure stderr
-# set the conversion pattern of stderr
-# Print the date in ISO 8601 format
-log4j.appender.stderr=org.apache.log4j.ConsoleAppender
-log4j.appender.stderr.Target=System.err
-log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
-log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
-
-log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
-log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
-log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
-
-# for debugging yarn-service framework
-#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
-
-# uncomment for YARN operations
-#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
-
-# uncomment this to debug security problems
-#log4j.logger.org.apache.hadoop.security=DEBUG
-
-#crank back on some noise
-log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-log4j.logger.org.apache.hadoop.hdfs=WARN
-
-log4j.logger.org.apache.zookeeper=WARN
-log4j.logger.org.apache.curator.framework.state=ERROR
-log4j.logger.org.apache.curator.framework.imps=WARN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
deleted file mode 100644
index c8de037..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ /dev/null
@@ -1,409 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-slider</artifactId>
- <version>3.0.0-beta1-SNAPSHOT</version>
- </parent>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-slider-core</artifactId>
- <packaging>jar</packaging>
- <name>Apache Hadoop YARN Slider Core</name>
-
- <properties>
- <!-- Needed for generating FindBugs warnings using parent pom -->
- <yarn.basedir>${project.parent.basedir}</yarn.basedir>
- </properties>
-
- <build>
- <!-- resources are filtered for dynamic updates. This gets build info in-->
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- <filtering>true</filtering>
- </resource>
- </resources>
-
- <plugins>
- <plugin>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-maven-plugins</artifactId>
- <executions>
- <execution>
- <id>compile-protoc</id>
- <goals>
- <goal>protoc</goal>
- </goals>
- <configuration>
- <protocVersion>${protobuf.version}</protocVersion>
- <protocCommand>${protoc.path}</protocCommand>
- <imports>
- <param>${basedir}/src/main/proto</param>
- </imports>
- <source>
- <directory>${basedir}/src/main/proto</directory>
- <includes>
- <include>ClientAMProtocol.proto</include>
- </includes>
- </source>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <environmentVariables>
- <JAVA_HOME>${java.home}</JAVA_HOME>
- </environmentVariables>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <excludes>
- <exclude>**/*.json</exclude>
- </excludes>
- </configuration>
- </plugin>
-
- </plugins>
- </build>
- <dependencies>
- <dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- <version>1.30</version>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <scope>runtime</scope>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-jaxrs</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-xc</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs-client</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-web-proxy</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-registry</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-digester</groupId>
- <artifactId>commons-digester</artifactId>
- <version>1.8</version>
- </dependency>
-
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-servlets</artifactId>
- <version>3.0.1</version>
- </dependency>
-
- <!-- ======================================================== -->
- <!-- service registry -->
- <!-- ======================================================== -->
-
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </dependency>
-
- <!-- ======================================================== -->
- <!-- Jersey and webapp support -->
- <!-- ======================================================== -->
-
- <dependency>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>javax.xml.bind</groupId>
- <artifactId>jaxb-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-servlet</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey.contribs</groupId>
- <artifactId>jersey-guice</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.easymock</groupId>
- <artifactId>easymock</artifactId>
- <version>3.1</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.objenesis</groupId>
- <artifactId>objenesis</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-easymock</artifactId>
- <version>1.6.5</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- <version>1.6.5</version>
- <exclusions>
- <exclusion>
- <groupId>org.javassist</groupId>
- <artifactId>javassist</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.objenesis</groupId>
- <artifactId>objenesis</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>javax.servlet.jsp</groupId>
- <artifactId>jsp-api</artifactId>
- <scope>runtime</scope>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jettison</groupId>
- <artifactId>jettison</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.yaml</groupId>
- <artifactId>snakeyaml</artifactId>
- <version>1.16</version>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>io.swagger</groupId>
- <artifactId>swagger-annotations</artifactId>
- <version>1.5.4</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
-
-
- <profiles>
- <profile>
- <id>dist</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-assemblies</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <id>dist</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <appendAssemblyId>false</appendAssemblyId>
- <attach>false</attach>
- <finalName>${project.artifactId}-${project.version}</finalName>
- <descriptorRefs>
- <descriptorRef>hadoop-yarn-services-dist</descriptorRef>
- </descriptorRefs>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
-
- </profiles>
-
-</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/assembly/executable-jar.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/assembly/executable-jar.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/assembly/executable-jar.xml
deleted file mode 100644
index 23383c8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/assembly/executable-jar.xml
+++ /dev/null
@@ -1,47 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<assembly
- xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
- <id>distribution</id>
- <formats>
- <format>zip</format>
- </formats>
- <includeBaseDirectory>true</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${project.build.directory}</directory>
- <outputDirectory>/</outputDirectory>
- <includes>
- <include>*.jar</include>
- </includes>
- </fileSet>
- </fileSets>
- <dependencySets>
- <dependencySet>
- <scope>runtime</scope>
- <outputDirectory>/lib</outputDirectory>
- <!-- dont copy JAR into /lib-->
- <useProjectArtifact>false</useProjectArtifact>
- <!-- <includeBaseDirectory>false</includeBaseDirectory>-->
- <unpack>false</unpack>
- </dependencySet>
- </dependencySets>
-</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
deleted file mode 100644
index 516d23d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
-
-import java.io.IOException;
-
-public interface ClientAMProtocol {
- FlexComponentsResponseProto flexComponents(FlexComponentsRequestProto request)
- throws IOException, YarnException;
-
- GetStatusResponseProto getStatus(GetStatusRequestProto requestProto)
- throws IOException, YarnException;
-
- StopResponseProto stop(StopRequestProto requestProto)
- throws IOException, YarnException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
deleted file mode 100644
index 6884757..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
-import org.apache.hadoop.yarn.service.component.ComponentEvent;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX;
-
-public class ClientAMService extends AbstractService
- implements ClientAMProtocol {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ClientAMService.class);
-
- private ServiceContext context;
- private Server server;
-
- private InetSocketAddress bindAddress;
-
- public ClientAMService(ServiceContext context) {
- super("Client AM Service");
- this.context = context;
- }
-
- @Override protected void serviceStart() throws Exception {
- Configuration conf = getConfig();
- YarnRPC rpc = YarnRPC.create(conf);
- InetSocketAddress address = new InetSocketAddress(0);
- server = rpc.getServer(ClientAMProtocol.class, this, address, conf,
- context.secretManager, 1);
- server.start();
-
- String nodeHostString =
- System.getenv(ApplicationConstants.Environment.NM_HOST.name());
-
- bindAddress = NetUtils.createSocketAddrForHost(nodeHostString,
- server.getListenerAddress().getPort());
-
- LOG.info("Instantiated ClientAMService at " + bindAddress);
- super.serviceStart();
- }
-
- @Override protected void serviceStop() throws Exception {
- if (server != null) {
- server.stop();
- }
- super.serviceStop();
- }
-
- @Override public FlexComponentsResponseProto flexComponents(
- FlexComponentsRequestProto request) throws IOException {
- if (!request.getComponentsList().isEmpty()) {
- for (ComponentCountProto component : request.getComponentsList()) {
- ComponentEvent event = new ComponentEvent(component.getName(), FLEX)
- .setDesired(component.getNumberOfContainers());
- context.scheduler.getDispatcher().getEventHandler().handle(event);
- LOG.info("Flexing component {} to {}", component.getName(),
- component.getNumberOfContainers());
- }
- }
- return FlexComponentsResponseProto.newBuilder().build();
- }
-
- @Override
- public GetStatusResponseProto getStatus(GetStatusRequestProto request)
- throws IOException, YarnException {
- String stat = ServiceApiUtil.jsonSerDeser.toJson(context.application);
- return GetStatusResponseProto.newBuilder().setStatus(stat).build();
- }
-
- @Override
- public StopResponseProto stop(StopRequestProto requestProto)
- throws IOException, YarnException {
- LOG.info("Stop the service.");
- // Stop the service in 2 seconds delay to make sure this rpc call is completed.
- // shutdown hook will be executed which will stop AM gracefully.
- Thread thread = new Thread() {
- @Override
- public void run() {
- try {
- Thread.sleep(2000);
- ExitUtil.terminate(0);
- } catch (InterruptedException e) {
- LOG.error("Interrupted while stopping", e);
- }
- }
- };
- thread.start();
- return StopResponseProto.newBuilder().build();
- }
-
- public InetSocketAddress getBindAddress() {
- return bindAddress;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
deleted file mode 100644
index 4743f28..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.yarn.service.component.Component;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.NODE_BLACKLIST_THRESHOLD;
-
-/**
- * This tracks the container failures per node. If the failure counter exceeds
- * the maxFailurePerNode limit, it'll blacklist that node.
- *
- */
-public class ContainerFailureTracker {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerFailureTracker.class);
-
- // Host -> num container failures
- private Map<String, Integer> failureCountPerNode = new HashMap<>();
- private Set<String> blackListedNodes = new HashSet<>();
- private ServiceContext context;
- private int maxFailurePerNode;
- private Component component;
-
- public ContainerFailureTracker(ServiceContext context, Component component) {
- this.context = context;
- this.component = component;
- maxFailurePerNode = component.getComponentSpec().getConfiguration()
- .getPropertyInt(NODE_BLACKLIST_THRESHOLD, 3);
- }
-
-
- public synchronized void incNodeFailure(String host) {
- int num = 0;
- if (failureCountPerNode.containsKey(host)) {
- num = failureCountPerNode.get(host);
- }
- num++;
- failureCountPerNode.put(host, num);
-
- // black list the node if exceed max failure
- if (num > maxFailurePerNode && !blackListedNodes.contains(host)) {
- List<String> blacklists = new ArrayList<>();
- blacklists.add(host);
- blackListedNodes.add(host);
- context.scheduler.getAmRMClient().updateBlacklist(blacklists, null);
- LOG.info("[COMPONENT {}]: Failed {} times on this host, blacklisted {}."
- + " Current list of blacklisted nodes: {}",
- component.getName(), num, host, blackListedNodes);
- }
- }
-
- public synchronized void resetContainerFailures() {
- // reset container failure counter per node
- failureCountPerNode.clear();
- context.scheduler.getAmRMClient()
- .updateBlacklist(null, new ArrayList<>(blackListedNodes));
- LOG.info("[COMPONENT {}]: Clearing blacklisted nodes {} ",
- component.getName(), blackListedNodes);
- blackListedNodes.clear();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
deleted file mode 100644
index c7616af..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import com.google.common.cache.LoadingCache;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-
-public class ServiceContext {
- public Application application = null;
- public SliderFileSystem fs;
- public String serviceHdfsDir = "";
- public ApplicationAttemptId attemptId;
- public LoadingCache<ConfigFile, Object> configCache;
- public ServiceScheduler scheduler;
- public ClientToAMTokenSecretManager secretManager;
- public ClientAMService clientAMService;
-
- public ServiceContext() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
deleted file mode 100644
index d099f8c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
-import org.apache.hadoop.yarn.service.client.params.SliderAMArgs;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.servicemonitor.ServiceMonitor;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Map;
-
-public class ServiceMaster extends CompositeService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceMaster.class);
-
- private static SliderAMArgs amArgs;
- protected ServiceContext context;
-
- public ServiceMaster(String name) {
- super(name);
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- //TODO Deprecate slider conf, make sure works with yarn conf
- printSystemEnv();
- if (UserGroupInformation.isSecurityEnabled()) {
- UserGroupInformation.setConfiguration(conf);
- }
- LOG.info("Login user is {}", UserGroupInformation.getLoginUser());
-
- context = new ServiceContext();
- Path appDir = getAppDir();
- context.serviceHdfsDir = appDir.toString();
- SliderFileSystem fs = new SliderFileSystem(conf);
- context.fs = fs;
- fs.setAppDir(appDir);
- loadApplicationJson(context, fs);
-
- ContainerId amContainerId = getAMContainerId();
-
- ApplicationAttemptId attemptId = amContainerId.getApplicationAttemptId();
- LOG.info("Application attemptId: " + attemptId);
- context.attemptId = attemptId;
-
- // configure AM to wait forever for RM
- conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, -1);
- conf.unset(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS);
-
- DefaultMetricsSystem.initialize("ServiceAppMaster");
-
- context.secretManager = new ClientToAMTokenSecretManager(attemptId, null);
- ClientAMService clientAMService = new ClientAMService(context);
- context.clientAMService = clientAMService;
- addService(clientAMService);
-
- ServiceScheduler scheduler = createServiceScheduler(context);
- addService(scheduler);
- context.scheduler = scheduler;
-
- ServiceMonitor monitor = new ServiceMonitor("Service Monitor", context);
- addService(monitor);
-
- super.serviceInit(conf);
- }
-
- protected ContainerId getAMContainerId() throws BadClusterStateException {
- return ContainerId.fromString(SliderUtils.mandatoryEnvVariable(
- ApplicationConstants.Environment.CONTAINER_ID.name()));
- }
-
- protected Path getAppDir() {
- return new Path(amArgs.getAppDefPath()).getParent();
- }
-
- protected ServiceScheduler createServiceScheduler(ServiceContext context)
- throws IOException, YarnException {
- return new ServiceScheduler(context);
- }
-
- protected void loadApplicationJson(ServiceContext context,
- SliderFileSystem fs) throws IOException {
- context.application = ServiceApiUtil
- .loadApplicationFrom(fs, new Path(amArgs.getAppDefPath()));
- LOG.info(context.application.toString());
- }
-
- @Override
- protected void serviceStop() throws Exception {
- LOG.info("Stopping app master");
- super.serviceStop();
- }
-
- private void printSystemEnv() {
- for (Map.Entry<String, String> envs : System.getenv().entrySet()) {
- LOG.info("{} = {}", envs.getKey(), envs.getValue());
- }
- }
-
- public static void main(String[] args) throws Exception {
- Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
- StringUtils.startupShutdownMessage(ServiceMaster.class, args, LOG);
- amArgs = new SliderAMArgs(args);
- amArgs.parse();
- try {
- ServiceMaster serviceMaster = new ServiceMaster("Service Master");
- ShutdownHookManager.get()
- .addShutdownHook(new CompositeServiceShutdownHook(serviceMaster), 30);
- YarnConfiguration conf = new YarnConfiguration();
- new GenericOptionsParser(conf, args);
- serviceMaster.init(conf);
- serviceMaster.start();
- } catch (Throwable t) {
- LOG.error("Error starting service master", t);
- ExitUtil.terminate(1, "Error starting service master");
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[51/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
deleted file mode 100644
index 83c9961..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.provider;
-
-import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
-import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Base class for factories.
- */
-public abstract class ProviderFactory {
- protected static final Logger LOG =
- LoggerFactory.getLogger(ProviderFactory.class);
-
- protected ProviderFactory() {}
-
- public abstract AbstractClientProvider createClientProvider();
-
- public abstract ProviderService createServerProvider();
-
- public static synchronized ProviderService getProviderService(Artifact
- artifact) {
- return createSliderProviderFactory(artifact).createServerProvider();
- }
-
- public static synchronized AbstractClientProvider getClientProvider(Artifact
- artifact) {
- return createSliderProviderFactory(artifact).createClientProvider();
- }
-
- /**
- * Create a provider for a specific application
- * @param artifact artifact
- * @return provider factory
- */
- public static synchronized ProviderFactory createSliderProviderFactory(
- Artifact artifact) {
- if (artifact == null || artifact.getType() == null) {
- LOG.debug("Loading service provider type default");
- return DefaultProviderFactory.getInstance();
- }
- LOG.debug("Loading service provider type {}", artifact.getType());
- switch (artifact.getType()) {
- // TODO add handling for custom types?
- // TODO handle application
- case DOCKER:
- return DockerProviderFactory.getInstance();
- case TARBALL:
- return TarballProviderFactory.getInstance();
- default:
- throw new IllegalArgumentException(String.format("Resolution error, " +
- "%s should not be passed to createSliderProviderFactory",
- artifact.getType()));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
deleted file mode 100644
index 9ef0176..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.provider;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-
-import java.io.IOException;
-
-public interface ProviderService {
-
- /**
- * Set up the entire container launch context
- */
- void buildContainerLaunchContext(AbstractLauncher containerLauncher,
- Application application, ComponentInstance instance,
- SliderFileSystem sliderFileSystem, Configuration yarnConf)
- throws IOException, SliderException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
deleted file mode 100644
index a044838..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.provider;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-import org.apache.hadoop.yarn.service.api.records.Configuration;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.utils.PublishedConfiguration;
-import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.regex.Pattern;
-
-import static org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants.*;
-
-/**
- * This is a factoring out of methods handy for providers. It's bonded to a log
- * at construction time.
- */
-public class ProviderUtils implements YarnServiceConstants {
-
- protected static final Logger log =
- LoggerFactory.getLogger(ProviderUtils.class);
-
-
- /**
- * Add oneself to the classpath. This does not work
- * on minicluster test runs where the JAR is not built up.
- * @param providerResources map of provider resources to add these entries to
- * @param providerClass provider to add
- * @param jarName name of the jar to use
- * @param sliderFileSystem target filesystem
- * @param tempPath path in the cluster FS for temp files
- * @param libdir relative directory to place resources
- * @param miniClusterTestRun true if minicluster is being used
- * @return true if the class was found in a JAR
- *
- * @throws FileNotFoundException if the JAR was not found and this is NOT
- * a mini cluster test run
- * @throws IOException IO problems
- * @throws SliderException any Slider problem
- */
- public static boolean addProviderJar(
- Map<String, LocalResource> providerResources,
- Class providerClass,
- String jarName,
- SliderFileSystem sliderFileSystem,
- Path tempPath,
- String libdir,
- boolean miniClusterTestRun) throws
- IOException,
- SliderException {
- try {
- SliderUtils.putJar(providerResources,
- sliderFileSystem,
- providerClass,
- tempPath,
- libdir,
- jarName);
- return true;
- } catch (FileNotFoundException e) {
- if (miniClusterTestRun) {
- return false;
- } else {
- throw e;
- }
- }
- }
-
- /**
- * Loads all dependency jars from the default path.
- * @param providerResources map of provider resources to add these entries to
- * @param sliderFileSystem target filesystem
- * @param tempPath path in the cluster FS for temp files
- * @param libDir relative directory to place resources
- * @param libLocalSrcDir explicitly supplied local libs dir
- * @throws IOException trouble copying to HDFS
- * @throws SliderException trouble copying to HDFS
- */
- public static void addAllDependencyJars(
- Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Path tempPath,
- String libDir,
- String libLocalSrcDir)
- throws IOException, SliderException {
- if (SliderUtils.isSet(libLocalSrcDir)) {
- File file = new File(libLocalSrcDir);
- if (!file.exists() || !file.isDirectory()) {
- throw new BadCommandArgumentsException(
- "Supplied lib src dir %s is not valid", libLocalSrcDir);
- }
- }
- SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
- libDir, libLocalSrcDir);
- }
-
- public static String substituteStrWithTokens(String content,
- Map<String, String> tokensForSubstitution) {
- for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
- content =
- content.replaceAll(Pattern.quote(token.getKey()), token.getValue());
- }
- return content;
- }
-
- // configs will be substituted by corresponding env in tokenMap
- public static void substituteMapWithTokens(Map<String, String> configs,
- Map<String, String> tokenMap) {
- for (Map.Entry<String, String> entry : configs.entrySet()) {
- String value = entry.getValue();
- if (tokenMap != null) {
- for (Map.Entry<String, String> token : tokenMap.entrySet()) {
- value =
- value.replaceAll(Pattern.quote(token.getKey()), token.getValue());
- }
- }
- entry.setValue(value);
- }
- }
-
- /**
- * Localize the service keytabs for the application.
- * @param launcher container launcher
- * @param fileSystem file system
- * @throws IOException trouble uploading to HDFS
- */
- public void localizeServiceKeytabs(AbstractLauncher launcher,
- SliderFileSystem fileSystem, Application application) throws IOException {
-
- Configuration conf = application.getConfiguration();
- String keytabPathOnHost =
- conf.getProperty(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
- if (SliderUtils.isUnset(keytabPathOnHost)) {
- String amKeytabName =
- conf.getProperty(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
- String keytabDir =
- conf.getProperty(YarnServiceConf.KEY_HDFS_KEYTAB_DIR);
- // we need to localize the keytab files in the directory
- Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
- application.getName());
- boolean serviceKeytabsDeployed = false;
- if (fileSystem.getFileSystem().exists(keytabDirPath)) {
- FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(
- keytabDirPath);
- LocalResource keytabRes;
- for (FileStatus keytab : keytabs) {
- if (!amKeytabName.equals(keytab.getPath().getName())
- && keytab.getPath().getName().endsWith(".keytab")) {
- serviceKeytabsDeployed = true;
- log.info("Localizing keytab {}", keytab.getPath().getName());
- keytabRes = fileSystem.createAmResource(keytab.getPath(),
- LocalResourceType.FILE);
- launcher.addLocalResource(KEYTAB_DIR + "/" +
- keytab.getPath().getName(),
- keytabRes);
- }
- }
- }
- if (!serviceKeytabsDeployed) {
- log.warn("No service keytabs for the application have been localized. "
- + "If the application requires keytabs for secure operation, "
- + "please ensure that the required keytabs have been uploaded "
- + "to the folder {}", keytabDirPath);
- }
- }
- }
-
- // 1. Create all config files for a component on hdfs for localization
- // 2. Add the config file to localResource
- public static synchronized void createConfigFileAndAddLocalResource(
- AbstractLauncher launcher, SliderFileSystem fs, Component component,
- Map<String, String> tokensForSubstitution, ComponentInstance instance,
- ServiceContext context) throws IOException {
- Path compDir =
- new Path(new Path(fs.getAppDir(), "components"), component.getName());
- Path compInstanceDir =
- new Path(compDir, instance.getCompInstanceName());
- if (!fs.getFileSystem().exists(compInstanceDir)) {
- log.info(instance.getCompInstanceId() + ": Creating dir on hdfs: " + compInstanceDir);
- fs.getFileSystem().mkdirs(compInstanceDir,
- new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
- instance.setCompInstanceDir(compInstanceDir);
- } else {
- log.info("Component instance conf dir already exists: " + compInstanceDir);
- }
-
- if (log.isDebugEnabled()) {
- log.debug("Tokens substitution for component instance: " + instance
- .getCompInstanceName() + System.lineSeparator()
- + tokensForSubstitution);
- }
-
- for (ConfigFile originalFile : component.getConfiguration().getFiles()) {
- ConfigFile configFile = originalFile.copy();
- String fileName = new Path(configFile.getDestFile()).getName();
-
- // substitute file name
- for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
- configFile.setDestFile(configFile.getDestFile()
- .replaceAll(Pattern.quote(token.getKey()), token.getValue()));
- }
-
- Path remoteFile = new Path(compInstanceDir, fileName);
- if (!fs.getFileSystem().exists(remoteFile)) {
- log.info("Saving config file on hdfs for component " + instance
- .getCompInstanceName() + ": " + configFile);
-
- if (configFile.getSrcFile() != null) {
- // Load config file template
- switch (configFile.getType()) {
- case HADOOP_XML:
- // Hadoop_xml_template
- resolveHadoopXmlTemplateAndSaveOnHdfs(fs.getFileSystem(),
- tokensForSubstitution, configFile, remoteFile, context);
- break;
- case TEMPLATE:
- // plain-template
- resolvePlainTemplateAndSaveOnHdfs(fs.getFileSystem(),
- tokensForSubstitution, configFile, remoteFile, context);
- break;
- default:
- log.info("Not supporting loading src_file for " + configFile);
- break;
- }
- } else {
- // non-template
- resolveNonTemplateConfigsAndSaveOnHdfs(fs, tokensForSubstitution,
- instance, configFile, fileName, remoteFile);
- }
- }
-
- // Add resource for localization
- LocalResource configResource =
- fs.createAmResource(remoteFile, LocalResourceType.FILE);
- File destFile = new File(configFile.getDestFile());
- String symlink = APP_CONF_DIR + "/" + fileName;
- if (destFile.isAbsolute()) {
- launcher.addLocalResource(symlink, configResource,
- configFile.getDestFile());
- log.info("Add config file for localization: " + symlink + " -> "
- + configResource.getResource().getFile() + ", dest mount path: "
- + configFile.getDestFile());
- } else {
- launcher.addLocalResource(symlink, configResource);
- log.info("Add config file for localization: " + symlink + " -> "
- + configResource.getResource().getFile());
- }
- }
- }
-
- private static void resolveNonTemplateConfigsAndSaveOnHdfs(SliderFileSystem fs,
- Map<String, String> tokensForSubstitution, ComponentInstance instance,
- ConfigFile configFile, String fileName, Path remoteFile)
- throws IOException {
- // substitute non-template configs
- substituteMapWithTokens(configFile.getProps(), tokensForSubstitution);
-
- // write configs onto hdfs
- PublishedConfiguration publishedConfiguration =
- new PublishedConfiguration(fileName,
- configFile.getProps().entrySet());
- if (!fs.getFileSystem().exists(remoteFile)) {
- PublishedConfigurationOutputter configurationOutputter =
- PublishedConfigurationOutputter.createOutputter(
- ConfigFormat.resolve(configFile.getType().toString()),
- publishedConfiguration);
- try (FSDataOutputStream os = fs.getFileSystem().create(remoteFile)) {
- configurationOutputter.save(os);
- os.flush();
- }
- } else {
- log.info("Component instance = " + instance.getCompInstanceName()
- + ", config file already exists: " + remoteFile);
- }
- }
-
- // 1. substitute config template - only handle hadoop_xml format
- // 2. save on hdfs
- @SuppressWarnings("unchecked")
- private static void resolveHadoopXmlTemplateAndSaveOnHdfs(FileSystem fs,
- Map<String, String> tokensForSubstitution, ConfigFile configFile,
- Path remoteFile, ServiceContext context) throws IOException {
- Map<String, String> conf;
- try {
- conf = (Map<String, String>) context.configCache.get(configFile);
- } catch (ExecutionException e) {
- log.info("Failed to load config file: " + configFile, e);
- return;
- }
- // make a copy for substitution
- org.apache.hadoop.conf.Configuration confCopy =
- new org.apache.hadoop.conf.Configuration(false);
- for (Map.Entry<String, String> entry : conf.entrySet()) {
- confCopy.set(entry.getKey(), entry.getValue());
- }
- // substitute properties
- for (Map.Entry<String, String> entry : configFile.getProps().entrySet()) {
- confCopy.set(entry.getKey(), entry.getValue());
- }
- // substitute env variables
- for (Map.Entry<String, String> entry : confCopy) {
- String val = entry.getValue();
- if (val != null) {
- for (Map.Entry<String, String> token : tokensForSubstitution
- .entrySet()) {
- val = val.replaceAll(Pattern.quote(token.getKey()), token.getValue());
- confCopy.set(entry.getKey(), val);
- }
- }
- }
- // save on hdfs
- try (OutputStream output = fs.create(remoteFile)) {
- confCopy.writeXml(output);
- log.info("Reading config from: " + configFile.getSrcFile()
- + ", writing to: " + remoteFile);
- }
- }
-
- // 1) read the template as a string
- // 2) do token substitution
- // 3) save on hdfs
- private static void resolvePlainTemplateAndSaveOnHdfs(FileSystem fs,
- Map<String, String> tokensForSubstitution, ConfigFile configFile,
- Path remoteFile, ServiceContext context) {
- String content;
- try {
- content = (String) context.configCache.get(configFile);
- } catch (ExecutionException e) {
- log.info("Failed to load config file: " + configFile, e);
- return;
- }
- // substitute tokens
- content = substituteStrWithTokens(content, tokensForSubstitution);
-
- try (OutputStream output = fs.create(remoteFile)) {
- org.apache.commons.io.IOUtils.write(content, output);
- } catch (IOException e) {
- log.info("Failed to create " + remoteFile);
- }
- }
-
- /**
- * Get initial component token map to be substituted into config values.
- * @return tokens to replace
- */
- public static Map<String, String> initCompTokensForSubstitute(
- ComponentInstance instance) {
- Map<String, String> tokens = new HashMap<>();
- tokens.put(COMPONENT_NAME, instance.getCompSpec().getName());
- tokens
- .put(COMPONENT_NAME_LC, instance.getCompSpec().getName().toLowerCase());
- tokens.put(COMPONENT_INSTANCE_NAME, instance.getCompInstanceName());
- tokens.put(CONTAINER_ID, instance.getContainer().getId().toString());
- tokens.put(COMPONENT_ID,
- String.valueOf(instance.getCompInstanceId().getId()));
- return tokens;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
deleted file mode 100644
index 0920a9c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.defaultImpl;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-
-import java.io.IOException;
-import java.nio.file.Paths;
-
-public class DefaultClientProvider extends AbstractClientProvider {
-
- public DefaultClientProvider() {
- }
-
- @Override
- public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
- }
-
- @Override
- protected void validateConfigFile(ConfigFile configFile, FileSystem
- fileSystem) throws IOException {
- // validate dest_file is not absolute
- if (Paths.get(configFile.getDestFile()).isAbsolute()) {
- throw new IllegalArgumentException(
- "Dest_file must not be absolute path: " + configFile.getDestFile());
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
deleted file mode 100644
index 868bba8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.defaultImpl;
-
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.provider.ProviderService;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-
-public final class DefaultProviderFactory extends ProviderFactory {
- private static final ProviderFactory FACTORY = new
- DefaultProviderFactory();
-
- private DefaultProviderFactory() {}
-
- private static class Client {
- static final AbstractClientProvider PROVIDER = new DefaultClientProvider();
- }
-
- private static class Server {
- static final ProviderService PROVIDER = new DefaultProviderService();
- }
-
- @Override
- public AbstractClientProvider createClientProvider() {
- return Client.PROVIDER;
- }
-
- @Override
- public ProviderService createServerProvider() {
- return Server.PROVIDER;
- }
-
- public static ProviderFactory getInstance() {
- return FACTORY;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
deleted file mode 100644
index 33f8278..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.defaultImpl;
-
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
-
-import java.io.IOException;
-
-public class DefaultProviderService extends AbstractProviderService {
-
- @Override
- public void processArtifact(AbstractLauncher launcher,
- ComponentInstance compInstance, SliderFileSystem fileSystem,
- Application application)
- throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
deleted file mode 100644
index d4a2254..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.docker;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
-
-import java.io.IOException;
-
-public class DockerClientProvider extends AbstractClientProvider
- implements YarnServiceConstants {
-
- public DockerClientProvider() {
- super();
- }
-
- @Override
- public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
- if (artifact == null) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
- }
- if (StringUtils.isEmpty(artifact.getId())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
- }
- }
-
- @Override
- protected void validateConfigFile(ConfigFile configFile, FileSystem
- fileSystem) throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
deleted file mode 100644
index f30c002..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.docker;
-
-public interface DockerKeys {
- String PROVIDER_DOCKER = "docker";
- String DOCKER_PREFIX = "docker.";
- String DOCKER_IMAGE = DOCKER_PREFIX + "image";
- String DOCKER_NETWORK = DOCKER_PREFIX + "network";
- String DOCKER_USE_PRIVILEGED = DOCKER_PREFIX + "usePrivileged";
- String DOCKER_START_COMMAND = DOCKER_PREFIX + "startCommand";
-
- String DEFAULT_DOCKER_NETWORK = "bridge";
- Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
deleted file mode 100644
index 57330ab..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.docker;
-
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.provider.ProviderService;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-
-public class DockerProviderFactory extends ProviderFactory {
- private static final ProviderFactory FACTORY = new
- DockerProviderFactory();
-
- private DockerProviderFactory() {
- }
-
- private static class Client {
- static final AbstractClientProvider PROVIDER = new DockerClientProvider();
- }
-
- private static class Server {
- static final ProviderService PROVIDER = new DockerProviderService();
- }
-
- @Override
- public AbstractClientProvider createClientProvider() {
- return Client.PROVIDER;
- }
-
- @Override
- public ProviderService createServerProvider() {
- return Server.PROVIDER;
- }
-
- public static ProviderFactory getInstance() {
- return FACTORY;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
deleted file mode 100644
index 236ddd9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.docker;
-
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
-
-import java.io.IOException;
-import java.text.MessageFormat;
-
-public class DockerProviderService extends AbstractProviderService
- implements DockerKeys {
-
- public void processArtifact(AbstractLauncher launcher,
- ComponentInstance compInstance, SliderFileSystem fileSystem,
- Application application) throws IOException{
- launcher.setYarnDockerMode(true);
- launcher.setDockerImage(compInstance.getCompSpec().getArtifact().getId());
- launcher.setDockerNetwork(compInstance.getCompSpec().getConfiguration()
- .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK));
- String domain = compInstance.getComponent().getScheduler().getConfig()
- .get(RegistryConstants.KEY_DNS_DOMAIN);
- String hostname;
- if (domain == null || domain.isEmpty()) {
- hostname = MessageFormat
- .format("{0}.{1}.{2}", compInstance.getCompInstanceName(),
- application.getName(), RegistryUtils.currentUser());
- } else {
- hostname = MessageFormat
- .format("{0}.{1}.{2}.{3}", compInstance.getCompInstanceName(),
- application.getName(), RegistryUtils.currentUser(), domain);
- }
- launcher.setDockerHostname(hostname);
- launcher.setRunPrivilegedContainer(
- compInstance.getCompSpec().getRunPrivilegedContainer());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
deleted file mode 100644
index 01f7b20..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.tarball;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
-
-import java.io.IOException;
-import java.nio.file.Paths;
-
-public class TarballClientProvider extends AbstractClientProvider
- implements YarnServiceConstants {
-
- public TarballClientProvider() {
- }
-
- @Override
- public void validateArtifact(Artifact artifact, FileSystem fs)
- throws IOException {
- if (artifact == null) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
- }
- if (StringUtils.isEmpty(artifact.getId())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
- }
- Path p = new Path(artifact.getId());
- if (!fs.exists(p)) {
- throw new IllegalArgumentException( "Artifact tarball does not exist "
- + artifact.getId());
- }
- }
-
- @Override
- protected void validateConfigFile(ConfigFile configFile, FileSystem
- fileSystem) throws IOException {
- // validate dest_file is not absolute
- if (Paths.get(configFile.getDestFile()).isAbsolute()) {
- throw new IllegalArgumentException(
- "Dest_file must not be absolute path: " + configFile.getDestFile());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
deleted file mode 100644
index 9d81f66..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.tarball;
-
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.provider.ProviderService;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-
-public class TarballProviderFactory extends ProviderFactory {
- private static final ProviderFactory FACTORY = new
- TarballProviderFactory();
-
- private TarballProviderFactory() {
- }
-
- private static class Client {
- static final AbstractClientProvider PROVIDER = new TarballClientProvider();
- }
-
- private static class Server {
- static final ProviderService PROVIDER = new TarballProviderService();
- }
-
- @Override
- public AbstractClientProvider createClientProvider() {
- return Client.PROVIDER;
- }
-
- @Override
- public ProviderService createServerProvider() {
- return Server.PROVIDER;
- }
-
- public static ProviderFactory getInstance() {
- return FACTORY;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
deleted file mode 100644
index 2403255..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.tarball;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
-
-import java.io.IOException;
-
-public class TarballProviderService extends AbstractProviderService {
-
- @Override
- public void processArtifact(AbstractLauncher launcher,
- ComponentInstance instance, SliderFileSystem fileSystem,
- Application application)
- throws IOException {
- Path artifact = new Path(instance.getCompSpec().getArtifact().getId());
- if (!fileSystem.isFile(artifact)) {
- throw new IOException(
- "Package doesn't exist as a resource: " + artifact.toString());
- }
- log.info("Adding resource {}", artifact.toString());
- LocalResourceType type = LocalResourceType.ARCHIVE;
- LocalResource packageResource = fileSystem.createAmResource(artifact, type);
- launcher.addLocalResource(APP_LIB_DIR, packageResource);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
deleted file mode 100644
index 56634f6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.registry;
-
-/**
- * These are constants unique to the Slider AM
- */
-public class CustomRegistryConstants {
-
- public static final String MANAGEMENT_REST_API =
- "classpath:org.apache.slider.management";
-
- public static final String REGISTRY_REST_API =
- "classpath:org.apache.slider.registry";
-
- public static final String PUBLISHER_REST_API =
- "classpath:org.apache.slider.publisher";
-
- public static final String PUBLISHER_CONFIGURATIONS_API =
- "classpath:org.apache.slider.publisher.configurations";
-
- public static final String PUBLISHER_EXPORTS_API =
- "classpath:org.apache.slider.publisher.exports";
-
- public static final String PUBLISHER_DOCUMENTS_API =
- "classpath:org.apache.slider.publisher.documents";
-
- public static final String AGENT_SECURE_REST_API =
- "classpath:org.apache.slider.agents.secure";
-
- public static final String AGENT_ONEWAY_REST_API =
- "classpath:org.apache.slider.agents.oneway";
-
- public static final String AM_IPC_PROTOCOL =
- "classpath:org.apache.slider.appmaster.ipc";
-
- public static final String AM_REST_BASE =
- "classpath:org.apache.slider.client.rest";
-
- public static final String WEB_UI = "http://";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
deleted file mode 100644
index ef5ed91..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.registry;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.registry.client.api.BindFlags;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceId;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join;
-
-/**
- * Registry view for providers. This tracks where the service
- * is registered, offers access to the record and other things.
- */
-public class YarnRegistryViewForProviders {
- private static final Log LOG =
- LogFactory.getLog(YarnRegistryViewForProviders.class);
-
- private final RegistryOperations registryOperations;
- private final String user;
- private final String sliderServiceClass;
- private final String instanceName;
- /**
- * Record used where the service registered itself.
- * Null until the service is registered
- */
- private ServiceRecord selfRegistration;
-
- /**
- * Path where record was registered
- * Null until the service is registered
- */
- private String selfRegistrationPath;
-
- public YarnRegistryViewForProviders(RegistryOperations registryOperations,
- String user,
- String sliderServiceClass,
- String instanceName,
- ApplicationAttemptId applicationAttemptId) {
- Preconditions.checkArgument(registryOperations != null,
- "null registry operations");
- Preconditions.checkArgument(user != null, "null user");
- Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass),
- "unset service class");
- Preconditions.checkArgument(SliderUtils.isSet(instanceName),
- "instanceName");
- Preconditions.checkArgument(applicationAttemptId != null,
- "null applicationAttemptId");
- this.registryOperations = registryOperations;
- this.user = user;
- this.sliderServiceClass = sliderServiceClass;
- this.instanceName = instanceName;
- }
-
- public String getUser() {
- return user;
- }
-
-
- private void setSelfRegistration(ServiceRecord selfRegistration) {
- this.selfRegistration = selfRegistration;
- }
-
- /**
- * Get the path to where the service has registered itself.
- * Null until the service is registered
- * @return the service registration path.
- */
- public String getSelfRegistrationPath() {
- return selfRegistrationPath;
- }
-
- /**
- * Get the absolute path to where the service has registered itself.
- * This includes the base registry path
- * Null until the service is registered
- * @return the service registration path.
- */
- public String getAbsoluteSelfRegistrationPath() {
- if (selfRegistrationPath == null) {
- return null;
- }
- String root = registryOperations.getConfig().getTrimmed(
- RegistryConstants.KEY_REGISTRY_ZK_ROOT,
- RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
- return RegistryPathUtils.join(root, selfRegistrationPath);
- }
-
- /**
- * Add a component under the slider name/entry
- * @param componentName component name
- * @param record record to put
- * @throws IOException
- */
- public void putComponent(String componentName,
- ServiceRecord record) throws
- IOException {
- putComponent(sliderServiceClass, instanceName,
- componentName,
- record);
- }
-
- /**
- * Add a component
- * @param serviceClass service class to use under ~user
- * @param componentName component name
- * @param record record to put
- * @throws IOException
- */
- public void putComponent(String serviceClass,
- String serviceName,
- String componentName,
- ServiceRecord record) throws IOException {
- String path = RegistryUtils.componentPath(
- user, serviceClass, serviceName, componentName);
- registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
- registryOperations.bind(path, record, BindFlags.OVERWRITE);
- }
-
- /**
- * Add a service under a path, optionally purging any history
- * @param username user
- * @param serviceClass service class to use under ~user
- * @param serviceName name of the service
- * @param record service record
- * @param deleteTreeFirst perform recursive delete of the path first.
- * @return the path the service was created at
- * @throws IOException
- */
- public String putService(String username,
- String serviceClass,
- String serviceName,
- ServiceRecord record,
- boolean deleteTreeFirst) throws IOException {
- String path = RegistryUtils.servicePath(
- username, serviceClass, serviceName);
- if (deleteTreeFirst) {
- registryOperations.delete(path, true);
- }
- registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
- registryOperations.bind(path, record, BindFlags.OVERWRITE);
- return path;
- }
-
- /**
- * Add a service under a path for the current user
- * @param record service record
- * @param deleteTreeFirst perform recursive delete of the path first
- * @return the path the service was created at
- * @throws IOException
- */
- public String registerSelf(
- ServiceRecord record,
- boolean deleteTreeFirst) throws IOException {
- selfRegistrationPath =
- putService(user, sliderServiceClass, instanceName, record, deleteTreeFirst);
- setSelfRegistration(record);
- return selfRegistrationPath;
- }
-
- /**
- * Delete a component
- * @param containerId component name
- * @throws IOException
- */
- public void deleteComponent(ComponentInstanceId instanceId,
- String containerId) throws IOException {
- String path = RegistryUtils.componentPath(
- user, sliderServiceClass, instanceName,
- containerId);
- LOG.info(instanceId + ": Deleting registry path " + path);
- registryOperations.delete(path, false);
- }
-
- /**
- * Delete the children of a path -but not the path itself.
- * It is not an error if the path does not exist
- * @param path path to delete
- * @param recursive flag to request recursive deletes
- * @throws IOException IO problems
- */
- public void deleteChildren(String path, boolean recursive) throws IOException {
- List<String> childNames = null;
- try {
- childNames = registryOperations.list(path);
- } catch (PathNotFoundException e) {
- return;
- }
- for (String childName : childNames) {
- String child = join(path, childName);
- registryOperations.delete(child, recursive);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/BaseRestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/BaseRestClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/BaseRestClient.java
deleted file mode 100644
index 2d01bef..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/BaseRestClient.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.rest;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.GenericType;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.core.MediaType;
-import java.io.IOException;
-import java.net.URI;
-
-
-/**
- * This is a base class for Jersey REST clients in Slider.
- * It supports the execution of operations —with
- * exceptions uprated to IOExceptions when needed.
- * <p>
- * Subclasses can use these operations to provide an API-like view
- * of the REST model
- */
-public class BaseRestClient {
- private static final Logger log =
- LoggerFactory.getLogger(BaseRestClient.class);
- private final Client client;
-
- public BaseRestClient(
- Client client) {
- Preconditions.checkNotNull(client, "null jersey client");
- this.client = client;
- }
-
- /**
- * Get the jersey client
- * @return jersey client
- */
- public Client getClient() {
- return client;
- }
-
- /**
- * Execute the operation. Failures are raised as IOException subclasses
- * @param method method to execute
- * @param resource resource to work against
- * @param c class to build
- * @param <T> type expected
- * @return an instance of the type T
- * @throws IOException on any failure
- */
- public <T> T exec(HttpVerb method, WebResource resource, Class<T> c)
- throws IOException {
- try {
- Preconditions.checkArgument(c != null);
- log.debug("{}} {}", method, resource.getURI());
- return resource.accept(MediaType.APPLICATION_JSON_TYPE)
- .method(method.getVerb(), c);
- } catch (ClientHandlerException ex) {
- throw ExceptionConverter.convertJerseyException(method.getVerb(),
- resource.getURI().toString(),
- ex);
- } catch (UniformInterfaceException ex) {
- throw UgiJerseyBinding.uprateFaults(method,
- resource.getURI().toString(),
- ex);
- }
- }
-
- /**
- * Execute the operation. Failures are raised as IOException subclasses
- * @param method method to execute
- * @param resource resource to work against
- * @param t type to work with
- * @param <T> type expected
- * @return an instance of the type T
- * @throws IOException on any failure
- */
- public <T> T exec(HttpVerb method, WebResource resource, GenericType<T> t)
- throws IOException {
- try {
- Preconditions.checkArgument(t != null);
- log.debug("{}} {}", method, resource.getURI());
- resource.accept(MediaType.APPLICATION_JSON_TYPE);
- return resource.method(method.getVerb(), t);
- } catch (ClientHandlerException ex) {
- throw ExceptionConverter.convertJerseyException(method.getVerb(),
- resource.getURI().toString(),
- ex);
- } catch (UniformInterfaceException ex) {
- throw UgiJerseyBinding.uprateFaults(method, resource.getURI().toString(),
- ex);
- }
- }
-
-
- /**
- * Execute the GET operation. Failures are raised as IOException subclasses
- * @param resource resource to work against
- * @param c class to build
- * @param <T> type expected
- * @return an instance of the type T
- * @throws IOException on any failure
- */
- public <T> T get(WebResource resource, Class<T> c) throws IOException {
- return exec(HttpVerb.GET, resource, c);
- }
-
- /**
- * Create a Web resource from the client.
- *
- * @param u the URI of the resource.
- * @return the Web resource.
- */
- public WebResource resource(URI u) {
- return client.resource(u);
- }
-
- /**
- * Create a Web resource from the client.
- *
- * @param url the URI of the resource.
- * @return the Web resource.
- */
-
- public WebResource resource(String url) {
- return client.resource(url);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/ExceptionConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/ExceptionConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/ExceptionConverter.java
deleted file mode 100644
index 12fdc79..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/ExceptionConverter.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.rest;
-
-import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import org.apache.hadoop.fs.InvalidRequestException;
-import org.apache.hadoop.fs.PathAccessDeniedException;
-import org.apache.hadoop.fs.PathIOException;
-import org.apache.hadoop.yarn.webapp.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletResponse;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
-/**
- * static methods to convert exceptions into different types, including
- * extraction of details and finer-grained conversions.
- */
-public class ExceptionConverter {
- private static final Logger
- log = LoggerFactory.getLogger(ExceptionConverter.class);
-
- /**
- * Uprate error codes 400 and up into faults;
- * 404 is converted to a {@link FileNotFoundException},
- * 401 to {@link ForbiddenException}
- * FileNotFoundException for an unknown resource
- * PathAccessDeniedException for access denied
- * PathIOException for anything else
- * @param verb HTTP Verb used
- * @param targetURL URL being targeted
- * @param exception original exception
- * @return a new exception, the original one nested as a cause
- */
- public static IOException convertJerseyException(String verb,
- String targetURL,
- UniformInterfaceException exception) {
-
- IOException ioe = null;
- ClientResponse response = exception.getResponse();
- if (response != null) {
- int status = response.getStatus();
- String body = "";
- try {
- if (response.hasEntity()) {
- body = response.getEntity(String.class);
- log.error("{} {} returned status {} and body\n{}",
- verb, targetURL, status, body);
- } else {
- log.error("{} {} returned status {} and empty body",
- verb, targetURL, status);
- }
- } catch (Exception e) {
- log.warn("Failed to extract body from client response", e);
- }
-
- if (status == HttpServletResponse.SC_UNAUTHORIZED
- || status == HttpServletResponse.SC_FORBIDDEN) {
- ioe = new PathAccessDeniedException(targetURL);
- } else if (status == HttpServletResponse.SC_BAD_REQUEST
- || status == HttpServletResponse.SC_NOT_ACCEPTABLE
- || status == HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE) {
- // bad request
- ioe = new InvalidRequestException(
- String.format("Bad %s request: status code %d against %s",
- verb, status, targetURL));
- } else if (status > 400 && status < 500) {
- ioe = new FileNotFoundException(targetURL);
- }
- if (ioe == null) {
- ioe = new PathIOException(targetURL,
- verb + " " + targetURL
- + " failed with status code : " + status
- + ":" + exception);
- }
- } else {
- ioe = new PathIOException(targetURL,
- verb + " " + targetURL + " failed: " + exception);
- }
- ioe.initCause(exception);
- return ioe;
- }
-
- /**
- * Handle a client-side Jersey exception.
- * <p>
- * If there's an inner IOException, return that.
- * <p>
- * Otherwise: create a new wrapper IOE including verb and target details
- * @param verb HTTP Verb used
- * @param targetURL URL being targeted
- * @param exception original exception
- * @return an exception to throw
- */
- public static IOException convertJerseyException(String verb,
- String targetURL,
- ClientHandlerException exception) {
- if (exception.getCause() instanceof IOException) {
- return (IOException)exception.getCause();
- } else {
- IOException ioe = new IOException(
- verb + " " + targetURL + " failed: " + exception);
- ioe.initCause(exception);
- return ioe;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/HttpVerb.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/HttpVerb.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/HttpVerb.java
deleted file mode 100644
index 93f9082..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/HttpVerb.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.rest;
-
-/**
- * Http verbs with details on what they support in terms of submit and
- * response bodies.
- * <p>
- * Those verbs which do support bodies in the response MAY NOT return it;
- * if the response code is 204 then the answer is "no body", but the operation
- * is considered a success.
- */
-public enum HttpVerb {
- GET("GET", false, true),
- POST("POST", true, true),
- PUT("PUT", true, true),
- DELETE("DELETE", false, true),
- HEAD("HEAD", false, false);
-
- private final String verb;
- private final boolean hasUploadBody;
- private final boolean hasResponseBody;
-
- HttpVerb(String verb, boolean hasUploadBody, boolean hasResponseBody) {
- this.verb = verb;
- this.hasUploadBody = hasUploadBody;
- this.hasResponseBody = hasResponseBody;
- }
-
- public String getVerb() {
- return verb;
- }
-
- public boolean hasUploadBody() {
- return hasUploadBody;
- }
-
- public boolean hasResponseBody() {
- return hasResponseBody;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/SliderURLConnectionFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/SliderURLConnectionFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/SliderURLConnectionFactory.java
deleted file mode 100644
index fcd7f55..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/SliderURLConnectionFactory.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.rest;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.web.KerberosUgiAuthenticator;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
-import org.apache.hadoop.security.ssl.SSLFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLSocketFactory;
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.net.URLConnection;
-import java.security.GeneralSecurityException;
-
-/**
- * Factory for URL connections; used behind the scenes in the Jersey integration.
- * <p>
- * Derived from the WebHDFS implementation.
- */
-public class SliderURLConnectionFactory {
- private static final Logger log =
- LoggerFactory.getLogger(SliderURLConnectionFactory.class);
-
- /**
- * Timeout for socket connects and reads
- */
- public final static int DEFAULT_SOCKET_TIMEOUT = 60 * 1000; // 1 minute
- private final ConnectionConfigurator connConfigurator;
-
- private static final ConnectionConfigurator DEFAULT_CONFIGURATOR = new BasicConfigurator();
-
- /**
- * Construct a new URLConnectionFactory based on the configuration. It will
- * try to load SSL certificates when it is specified.
- */
- public static SliderURLConnectionFactory newInstance(Configuration conf) {
- ConnectionConfigurator conn;
- try {
- conn = newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
- } catch (Exception e) {
- log.debug("Cannot load customized SSL configuration.", e);
- conn = DEFAULT_CONFIGURATOR;
- }
- return new SliderURLConnectionFactory(conn);
- }
-
- private SliderURLConnectionFactory(ConnectionConfigurator connConfigurator) {
- this.connConfigurator = connConfigurator;
- }
-
- /**
- * Create a new ConnectionConfigurator for SSL connections
- */
- private static ConnectionConfigurator newSslConnConfigurator(final int timeout,
- Configuration conf) throws IOException, GeneralSecurityException {
- final SSLFactory factory;
- final SSLSocketFactory sf;
- final HostnameVerifier hv;
-
- factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
- factory.init();
- sf = factory.createSSLSocketFactory();
- hv = factory.getHostnameVerifier();
-
- return new ConnectionConfigurator() {
- @Override
- public HttpURLConnection configure(HttpURLConnection conn)
- throws IOException {
- if (conn instanceof HttpsURLConnection) {
- HttpsURLConnection c = (HttpsURLConnection) conn;
- c.setSSLSocketFactory(sf);
- c.setHostnameVerifier(hv);
- }
- SliderURLConnectionFactory.setupConnection(conn, timeout);
- return conn;
- }
- };
- }
-
- /**
- * Opens a url with read and connect timeouts
- *
- * @param url
- * to open
- * @return URLConnection
- * @throws IOException
- */
- public URLConnection openConnection(URL url) throws IOException {
- try {
- return openConnection(url, false);
- } catch (AuthenticationException e) {
- // Unreachable
- return null;
- }
- }
-
- /**
- * Opens a url with read and connect timeouts
- *
- * @param url
- * URL to open
- * @param isSpnego
- * whether the url should be authenticated via SPNEGO
- * @return URLConnection
- * @throws IOException
- * @throws AuthenticationException
- */
- public URLConnection openConnection(URL url, boolean isSpnego)
- throws IOException, AuthenticationException {
- if (isSpnego) {
- log.debug("open AuthenticatedURL connection {}", url);
- UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
- final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
- return new AuthenticatedURL(new KerberosUgiAuthenticator(),
- connConfigurator).openConnection(url, authToken);
- } else {
- log.debug("open URL connection {}", url);
- URLConnection connection = url.openConnection();
- if (connection instanceof HttpURLConnection) {
- connConfigurator.configure((HttpURLConnection) connection);
- }
- return connection;
- }
- }
-
- /**
- * Sets connection parameters on the given URLConnection
- *
- * @param connection
- * URLConnection to set
- * @param socketTimeout
- * the connection and read timeout of the connection.
- */
- private static void setupConnection(URLConnection connection, int socketTimeout) {
- connection.setConnectTimeout(socketTimeout);
- connection.setReadTimeout(socketTimeout);
- connection.setUseCaches(false);
- if (connection instanceof HttpURLConnection) {
- ((HttpURLConnection) connection).setInstanceFollowRedirects(true);
- }
- }
-
- private static class BasicConfigurator implements ConnectionConfigurator {
- @Override
- public HttpURLConnection configure(HttpURLConnection conn)
- throws IOException {
- SliderURLConnectionFactory.setupConnection(conn, DEFAULT_SOCKET_TIMEOUT);
- return conn;
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
deleted file mode 100644
index 2e40a9b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.rpc;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolSignature;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.slider.api.SliderClusterProtocol;
-import org.apache.slider.api.proto.Messages;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-public class SliderClusterProtocolProxy implements SliderClusterProtocol {
-
- private static final RpcController NULL_CONTROLLER = null;
- private final SliderClusterProtocolPB endpoint;
- private final InetSocketAddress address;
-
- public SliderClusterProtocolProxy(SliderClusterProtocolPB endpoint,
- InetSocketAddress address) {
- Preconditions.checkArgument(endpoint != null, "null endpoint");
- Preconditions.checkNotNull(address != null, "null address");
- this.endpoint = endpoint;
- this.address = address;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("SliderClusterProtocolProxy{");
- sb.append("address=").append(address);
- sb.append('}');
- return sb.toString();
- }
-
- @Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion,
- int clientMethodsHash)
- throws IOException {
- if (!protocol.equals(RPC.getProtocolName(SliderClusterProtocolPB.class))) {
- throw new IOException("Serverside implements " +
- RPC.getProtocolName(SliderClusterProtocolPB.class) +
- ". The following requested protocol is unknown: " +
- protocol);
- }
-
- return ProtocolSignature.getProtocolSignature(clientMethodsHash,
- RPC.getProtocolVersion(
- SliderClusterProtocol.class),
- SliderClusterProtocol.class);
- }
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return SliderClusterProtocol.versionID;
- }
-
- private IOException convert(ServiceException se) {
- IOException ioe = ProtobufHelper.getRemoteException(se);
- if (ioe instanceof RemoteException) {
- RemoteException remoteException = (RemoteException) ioe;
- return remoteException.unwrapRemoteException();
- }
- return ioe;
- }
-
- @Override public Messages.StopClusterResponseProto stopCluster(
- Messages.StopClusterRequestProto request)
- throws IOException, YarnException {
- try {
- return endpoint.stopCluster(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
- @Override
- public Messages.UpgradeContainersResponseProto upgradeContainers(
- Messages.UpgradeContainersRequestProto request) throws IOException,
- YarnException {
- try {
- return endpoint.upgradeContainers(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
- @Override
- public Messages.FlexComponentsResponseProto flexComponents(
- Messages.FlexComponentsRequestProto request) throws IOException {
- try {
- return endpoint.flexComponents(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
- @Override
- public Messages.GetJSONClusterStatusResponseProto getJSONClusterStatus(
- Messages.GetJSONClusterStatusRequestProto request) throws
- IOException,
- YarnException {
- try {
- return endpoint.getJSONClusterStatus(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
- @Override
- public Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(Messages.ListNodeUUIDsByRoleRequestProto request) throws
- IOException,
- YarnException {
- try {
- return endpoint.listNodeUUIDsByRole(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
- @Override
- public Messages.GetNodeResponseProto getNode(Messages.GetNodeRequestProto request) throws
- IOException,
- YarnException {
- try {
- return endpoint.getNode(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
- @Override
- public Messages.GetClusterNodesResponseProto getClusterNodes(Messages.GetClusterNodesRequestProto request) throws
- IOException,
- YarnException {
- try {
- return endpoint.getClusterNodes(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
-
- @Override
- public Messages.EchoResponseProto echo(Messages.EchoRequestProto request) throws
- IOException,
- YarnException {
- try {
- return endpoint.echo(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
-
- @Override
- public Messages.KillContainerResponseProto killContainer(Messages.KillContainerRequestProto request) throws
- IOException,
- YarnException {
- try {
- return endpoint.killContainer(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
- @Override
- public Messages.AMSuicideResponseProto amSuicide(Messages.AMSuicideRequestProto request) throws
- IOException {
- try {
- return endpoint.amSuicide(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
- @Override
- public Messages.ApplicationLivenessInformationProto getLivenessInformation(
- Messages.GetApplicationLivenessRequestProto request) throws IOException {
- try {
- return endpoint.getLivenessInformation(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
- @Override
- public Messages.GetLiveContainersResponseProto getLiveContainers(Messages.GetLiveContainersRequestProto request) throws
- IOException {
- try {
- return endpoint.getLiveContainers(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
- @Override
- public Messages.ContainerInformationProto getLiveContainer(Messages.GetLiveContainerRequestProto request) throws
- IOException {
- try {
- return endpoint.getLiveContainer(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
- @Override
- public Messages.GetLiveComponentsResponseProto getLiveComponents(Messages.GetLiveComponentsRequestProto request) throws
- IOException {
- try {
- return endpoint.getLiveComponents(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
- @Override
- public Messages.ComponentInformationProto getLiveComponent(Messages.GetLiveComponentRequestProto request) throws
- IOException {
- try {
- return endpoint.getLiveComponent(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
- @Override
- public Messages.GetLiveNodesResponseProto getLiveNodes(Messages.GetLiveNodesRequestProto request)
- throws IOException {
- try {
- return endpoint.getLiveNodes(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-
- @Override
- public Messages.NodeInformationProto getLiveNode(Messages.GetLiveNodeRequestProto request)
- throws IOException {
- try {
- return endpoint.getLiveNode(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
deleted file mode 100644
index 22f9bc3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
+++ /dev/null
@@ -1,406 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.rpc;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ipc.ProtocolSignature;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.slider.api.SliderClusterProtocol;
-import org.apache.slider.api.proto.Messages;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.types.ApplicationLivenessInformation;
-import org.apache.slider.api.types.ComponentInformation;
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.api.types.NodeInformationList;
-import org.apache.slider.core.exceptions.ServiceNotReadyException;
-import org.apache.slider.core.main.LauncherExitCodes;
-import org.apache.slider.core.persist.JsonSerDeser;
-import org.apache.slider.server.appmaster.AppMasterActionOperations;
-import org.apache.slider.server.appmaster.actions.ActionFlexCluster;
-import org.apache.slider.server.appmaster.actions.ActionHalt;
-import org.apache.slider.server.appmaster.actions.ActionKillContainer;
-import org.apache.slider.server.appmaster.actions.ActionStopSlider;
-import org.apache.slider.server.appmaster.actions.ActionUpgradeContainers;
-import org.apache.slider.server.appmaster.actions.AsyncAction;
-import org.apache.slider.server.appmaster.actions.QueueAccess;
-import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.slider.api.types.RestTypeMarshalling.marshall;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
-
-/**
- * Implement the {@link SliderClusterProtocol}.
- */
-@SuppressWarnings("unchecked")
-
-public class SliderIPCService extends AbstractService
- implements SliderClusterProtocol {
-
- protected static final Logger log =
- LoggerFactory.getLogger(SliderIPCService.class);
-
- private final QueueAccess actionQueues;
- private final StateAccessForProviders state;
- private final MetricsAndMonitoring metricsAndMonitoring;
- private final AppMasterActionOperations amOperations;
- private final ContentCache cache;
- private static final JsonSerDeser<Application> jsonSerDeser =
- new JsonSerDeser<Application>(Application.class);
-
-
- /**
- * This is the prefix used for metrics
- */
- public static final String METRICS_PREFIX =
- "org.apache.slider.api.SliderIPCService.";
-
- /**
- * Constructor
- * @param amOperations access to any AM operations
- * @param state state view
- * @param actionQueues queues for actions
- * @param metricsAndMonitoring metrics
- * @param cache
- */
- public SliderIPCService(AppMasterActionOperations amOperations,
- StateAccessForProviders state, QueueAccess actionQueues,
- MetricsAndMonitoring metricsAndMonitoring, ContentCache cache) {
- super("SliderIPCService");
- Preconditions.checkArgument(amOperations != null, "null amOperations");
- Preconditions.checkArgument(state != null, "null appState");
- Preconditions.checkArgument(actionQueues != null, "null actionQueues");
- Preconditions.checkArgument(metricsAndMonitoring != null,
- "null metricsAndMonitoring");
- Preconditions.checkArgument(cache != null, "null cache");
- this.state = state;
- this.actionQueues = actionQueues;
- this.metricsAndMonitoring = metricsAndMonitoring;
- this.amOperations = amOperations;
- this.cache = cache;
- }
-
- @Override //SliderClusterProtocol
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion,
- int clientMethodsHash) throws IOException {
- return ProtocolSignature.getProtocolSignature(
- this, protocol, clientVersion, clientMethodsHash);
- }
-
-
- @Override //SliderClusterProtocol
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return SliderClusterProtocol.versionID;
- }
-
- /**
- * General actions to perform on a slider RPC call coming in
- * @param operation operation to log
- * @throws IOException problems
- * @throws ServiceNotReadyException if the RPC service is constructed
- * but not fully initialized
- */
- protected void onRpcCall(String operation) throws IOException {
- log.debug("Received call to {}", operation);
- metricsAndMonitoring.markMeterAndCounter(METRICS_PREFIX + operation);
- }
-
- /**
- * Schedule an action
- * @param action for delayed execution
- */
- public void schedule(AsyncAction action) {
- actionQueues.schedule(action);
- }
-
- /**
- * Queue an action for immediate execution in the executor thread
- * @param action action to execute
- */
- public void queue(AsyncAction action) {
- actionQueues.put(action);
- }
-
- @Override //SliderClusterProtocol
- public Messages.StopClusterResponseProto stopCluster(Messages.StopClusterRequestProto request)
- throws IOException, YarnException {
- onRpcCall("stop");
- String message = request.getMessage();
- if (message == null) {
- message = "application stopped by client";
- }
- ActionStopSlider stopSlider =
- new ActionStopSlider(message,
- 1000, TimeUnit.MILLISECONDS,
- LauncherExitCodes.EXIT_SUCCESS,
- FinalApplicationStatus.SUCCEEDED,
- message);
- log.info("SliderAppMasterApi.stopCluster: {}", stopSlider);
- schedule(stopSlider);
- return Messages.StopClusterResponseProto.getDefaultInstance();
- }
-
- @Override //SliderClusterProtocol
- public Messages.UpgradeContainersResponseProto upgradeContainers(
- Messages.UpgradeContainersRequestProto request) throws IOException,
- YarnException {
- onRpcCall("upgrade");
- String message = request.getMessage();
- if (message == null) {
- message = "application containers upgraded by client";
- }
- ActionUpgradeContainers upgradeContainers =
- new ActionUpgradeContainers(
- "Upgrade containers",
- 1000, TimeUnit.MILLISECONDS,
- LauncherExitCodes.EXIT_SUCCESS,
- FinalApplicationStatus.SUCCEEDED,
- request.getContainerList(),
- request.getComponentList(),
- message);
- log.info("SliderAppMasterApi.upgradeContainers: {}", upgradeContainers);
- schedule(upgradeContainers);
- return Messages.UpgradeContainersResponseProto.getDefaultInstance();
- }
-
- @Override
- public Messages.FlexComponentsResponseProto flexComponents(
- Messages.FlexComponentsRequestProto request) throws IOException {
- onRpcCall("flex");
- schedule(new ActionFlexCluster("flex", 1, TimeUnit.MILLISECONDS, request));
- return Messages.FlexComponentsResponseProto.newBuilder().build();
- }
-
- @Override //SliderClusterProtocol
- public Messages.GetJSONClusterStatusResponseProto getJSONClusterStatus(
- Messages.GetJSONClusterStatusRequestProto request)
- throws IOException, YarnException {
- onRpcCall("getstatus");
- String result;
- //quick update
- //query and json-ify
- Application application = state.refreshClusterStatus();
- String stat = jsonSerDeser.toJson(application);
- return Messages.GetJSONClusterStatusResponseProto.newBuilder()
- .setClusterSpec(stat).build();
- }
-
- @Override //SliderClusterProtocol
- public Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(Messages.ListNodeUUIDsByRoleRequestProto request)
- throws IOException, YarnException {
- onRpcCall("listnodes)");
- String role = request.getRole();
- Messages.ListNodeUUIDsByRoleResponseProto.Builder builder =
- Messages.ListNodeUUIDsByRoleResponseProto.newBuilder();
- List<RoleInstance> nodes = state.enumLiveInstancesInRole(role);
- for (RoleInstance node : nodes) {
- builder.addUuid(node.id);
- }
- return builder.build();
- }
-
- @Override //SliderClusterProtocol
- public Messages.GetNodeResponseProto getNode(Messages.GetNodeRequestProto request)
- throws IOException, YarnException {
- onRpcCall("getnode");
- RoleInstance instance = state.getLiveInstanceByContainerID(
- request.getUuid());
- return Messages.GetNodeResponseProto.newBuilder()
- .setClusterNode(instance.toProtobuf())
- .build();
- }
-
- @Override //SliderClusterProtocol
- public Messages.GetClusterNodesResponseProto getClusterNodes(
- Messages.GetClusterNodesRequestProto request)
- throws IOException, YarnException {
- onRpcCall("getclusternodes");
- List<RoleInstance>
- clusterNodes = state.getLiveInstancesByContainerIDs(
- request.getUuidList());
-
- Messages.GetClusterNodesResponseProto.Builder builder =
- Messages.GetClusterNodesResponseProto.newBuilder();
- for (RoleInstance node : clusterNodes) {
- builder.addClusterNode(node.toProtobuf());
- }
- //at this point: a possibly empty list of nodes
- return builder.build();
- }
-
- @Override
- public Messages.EchoResponseProto echo(Messages.EchoRequestProto request)
- throws IOException, YarnException {
- onRpcCall("echo");
- Messages.EchoResponseProto.Builder builder =
- Messages.EchoResponseProto.newBuilder();
- String text = request.getText();
- log.info("Echo request size ={}", text.length());
- log.info(text);
- //now return it
- builder.setText(text);
- return builder.build();
- }
-
- @Override
- public Messages.KillContainerResponseProto killContainer(Messages.KillContainerRequestProto request)
- throws IOException, YarnException {
- onRpcCall("killcontainer");
- String containerID = request.getId();
- log.info("Kill Container {}", containerID);
- //throws NoSuchNodeException if it is missing
- RoleInstance instance =
- state.getLiveInstanceByContainerID(containerID);
- queue(new ActionKillContainer(instance.getContainerId(), 0, TimeUnit.MILLISECONDS,
- amOperations));
- Messages.KillContainerResponseProto.Builder builder =
- Messages.KillContainerResponseProto.newBuilder();
- builder.setSuccess(true);
- return builder.build();
- }
-
-
- @Override
- public Messages.AMSuicideResponseProto amSuicide(
- Messages.AMSuicideRequestProto request)
- throws IOException {
- onRpcCall("amsuicide");
- int signal = request.getSignal();
- String text = request.getText();
- if (text == null) {
- text = "";
- }
- int delay = request.getDelay();
- log.info("AM Suicide with signal {}, message {} delay = {}", signal, text,
- delay);
- ActionHalt action = new ActionHalt(signal, text, delay,
- TimeUnit.MILLISECONDS);
- schedule(action);
- return Messages.AMSuicideResponseProto.getDefaultInstance();
- }
-
- @Override
- public Messages.ApplicationLivenessInformationProto getLivenessInformation(
- Messages.GetApplicationLivenessRequestProto request) throws IOException {
- ApplicationLivenessInformation info =
- state.getApplicationLivenessInformation();
- return marshall(info);
- }
-
- @Override
- public Messages.GetLiveContainersResponseProto getLiveContainers(
- Messages.GetLiveContainersRequestProto request)
- throws IOException {
- Map<String, ContainerInformation> infoMap =
- (Map<String, ContainerInformation>) cache.lookupWithIOE(LIVE_CONTAINERS);
- Messages.GetLiveContainersResponseProto.Builder builder =
- Messages.GetLiveContainersResponseProto.newBuilder();
-
- for (Map.Entry<String, ContainerInformation> entry : infoMap.entrySet()) {
- builder.addNames(entry.getKey());
- builder.addContainers(marshall(entry.getValue()));
- }
- return builder.build();
- }
-
- @Override
- public Messages.ContainerInformationProto getLiveContainer(Messages.GetLiveContainerRequestProto request)
- throws IOException {
- String containerId = request.getContainerId();
- RoleInstance id = state.getLiveInstanceByContainerID(containerId);
- ContainerInformation containerInformation = id.serialize();
- return marshall(containerInformation);
- }
-
- @Override
- public Messages.GetLiveComponentsResponseProto getLiveComponents(Messages.GetLiveComponentsRequestProto request)
- throws IOException {
- Map<String, ComponentInformation> infoMap =
- (Map<String, ComponentInformation>) cache.lookupWithIOE(LIVE_COMPONENTS);
- Messages.GetLiveComponentsResponseProto.Builder builder =
- Messages.GetLiveComponentsResponseProto.newBuilder();
-
- for (Map.Entry<String, ComponentInformation> entry : infoMap.entrySet()) {
- builder.addNames(entry.getKey());
- builder.addComponents(marshall(entry.getValue()));
- }
- return builder.build();
- }
-
-
- @Override
- public Messages.ComponentInformationProto getLiveComponent(Messages.GetLiveComponentRequestProto request)
- throws IOException {
- String name = request.getName();
- try {
- return marshall(state.getComponentInformation(name));
- } catch (YarnRuntimeException e) {
- throw new FileNotFoundException("Unknown component: " + name);
- }
- }
-
- @Override
- public Messages.GetLiveNodesResponseProto getLiveNodes(Messages.GetLiveNodesRequestProto request)
- throws IOException {
- NodeInformationList info = (NodeInformationList) cache.lookupWithIOE(LIVE_NODES);
- Messages.GetLiveNodesResponseProto.Builder builder =
- Messages.GetLiveNodesResponseProto.newBuilder();
-
- for (NodeInformation nodeInformation : info) {
- builder.addNodes(marshall(nodeInformation));
- }
- return builder.build();
- }
-
-
- @Override
- public Messages.NodeInformationProto getLiveNode(Messages.GetLiveNodeRequestProto request)
- throws IOException {
- String name = request.getName();
- NodeInformation nodeInformation = state.getNodeInformation(name);
- if (nodeInformation != null) {
- return marshall(nodeInformation);
- } else {
- throw new FileNotFoundException("Unknown host: " + name);
- }
- }
-
- private Messages.WrappedJsonProto wrap(String json) {
- Messages.WrappedJsonProto.Builder builder =
- Messages.WrappedJsonProto.newBuilder();
- builder.setJson(json);
- return builder.build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java
deleted file mode 100644
index 5b127b8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.slider.server.appmaster.rpc;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.KerberosInfo;
-import org.apache.hadoop.security.SecurityInfo;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.TokenInfo;
-import org.apache.hadoop.security.token.TokenSelector;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenSelector;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-
-import java.lang.annotation.Annotation;
-
-/**
- * This is where security information goes.
- * It is referred to in the <code>META-INF/services/org.apache.hadoop.security.SecurityInfo</code>
- * resource of this JAR, which is used to find the binding info
- */
-public class SliderRPCSecurityInfo extends SecurityInfo {
-
- @Override
- public KerberosInfo getKerberosInfo(Class<?> protocol, Configuration conf) {
- if (!protocol.equals(SliderClusterProtocolPB.class)) {
- return null;
- }
- return new KerberosInfo() {
-
- @Override
- public Class<? extends Annotation> annotationType() {
- return null;
- }
-
- @Override
- public String serverPrincipal() {
- return SliderXmlConfKeys.KEY_KERBEROS_PRINCIPAL;
- }
-
- @Override
- public String clientPrincipal() {
- return null;
- }
- };
- }
-
- @Override
- public TokenInfo getTokenInfo(Class<?> protocol, Configuration conf) {
- if (!protocol.equals(SliderClusterProtocolPB.class)) {
- return null;
- }
- return new TokenInfo() {
-
- @Override
- public Class<? extends Annotation> annotationType() {
- return null;
- }
-
- @Override
- public Class<? extends TokenSelector<? extends TokenIdentifier>>
- value() {
- return ClientToAMTokenSelector.class;
- }
-
- @Override
- public String toString() {
- return "SliderClusterProtocolPB token info";
- }
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
deleted file mode 100644
index 75eccd0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.security;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.UserGroupInformation;
-import static org.apache.slider.core.main.LauncherExitCodes.EXIT_UNAUTHORIZED;
-
-import org.apache.slider.api.resource.Application;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-
-/**
- * Class keeping code security information
- */
-public class SecurityConfiguration {
-
- protected static final Logger log =
- LoggerFactory.getLogger(SecurityConfiguration.class);
- private final Configuration configuration;
- private final Application application;
- private String clusterName;
-
- public SecurityConfiguration(Configuration configuration,
- Application application,
- String clusterName) throws SliderException {
- Preconditions.checkNotNull(configuration);
- Preconditions.checkNotNull(application);
- Preconditions.checkNotNull(clusterName);
- this.configuration = configuration;
- this.application = application;
- this.clusterName = clusterName;
- validate();
- }
-
- private void validate() throws SliderException {
- if (isSecurityEnabled()) {
- // TODO use AM configuration rather than app configuration
- String principal = application.getConfiguration().getProperty(
- SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
- if(SliderUtils.isUnset(principal)) {
- // if no login identity is available, fail
- UserGroupInformation loginUser = null;
- try {
- loginUser = getLoginUser();
- } catch (IOException e) {
- throw new SliderException(EXIT_UNAUTHORIZED, e,
- "No principal configured for the application and "
- + "exception raised during retrieval of login user. "
- + "Unable to proceed with application "
- + "initialization. Please ensure a value "
- + "for %s exists in the application "
- + "configuration or the login issue is addressed",
- SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
- }
- if (loginUser == null) {
- throw new SliderException(EXIT_UNAUTHORIZED,
- "No principal configured for the application "
- + "and no login user found. "
- + "Unable to proceed with application "
- + "initialization. Please ensure a value "
- + "for %s exists in the application "
- + "configuration or the login issue is addressed",
- SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
- }
- }
- // ensure that either local or distributed keytab mechanism is enabled,
- // but not both
- String keytabFullPath = application.getConfiguration().getProperty(
- SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
- String keytabName = application.getConfiguration().getProperty(
- SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
- if (SliderUtils.isSet(keytabFullPath) && SliderUtils.isSet(keytabName)) {
- throw new SliderException(EXIT_UNAUTHORIZED,
- "Both a keytab on the cluster host (%s) and a"
- + " keytab to be retrieved from HDFS (%s) are"
- + " specified. Please configure only one keytab"
- + " retrieval mechanism.",
- SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH,
- SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
-
- }
- }
- }
-
- protected UserGroupInformation getLoginUser() throws IOException {
- return UserGroupInformation.getLoginUser();
- }
-
- public boolean isSecurityEnabled() {
- return SliderUtils.isHadoopClusterSecure(configuration);
- }
-
- public String getPrincipal() throws IOException {
- String principal = application.getConfiguration().getProperty(
- SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
- if (SliderUtils.isUnset(principal)) {
- principal = UserGroupInformation.getLoginUser().getShortUserName();
- log.info("No principal set in the slider configuration. Will use AM " +
- "login identity {} to attempt keytab-based login", principal);
- }
-
- return principal;
- }
-
- public boolean isKeytabProvided() {
- String keytabLocalPath = application.getConfiguration().getProperty(
- SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
- String keytabName = application.getConfiguration().getProperty(
- SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
- return StringUtils.isNotBlank(keytabLocalPath)
- || StringUtils.isNotBlank(keytabName);
-
- }
-
- public File getKeytabFile()
- throws SliderException, IOException {
- //TODO implement this for dash semantic
- String keytabFullPath = application.getConfiguration().getProperty(
- SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
- File localKeytabFile;
- if (SliderUtils.isUnset(keytabFullPath)) {
- // get the keytab
- String keytabName = application.getConfiguration().getProperty(
- SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
- log.info("No host keytab file path specified. Will attempt to retrieve"
- + " keytab file {} as a local resource for the container",
- keytabName);
- // download keytab to local, protected directory
- localKeytabFile = new File(SliderKeys.KEYTAB_DIR, keytabName);
- } else {
- log.info("Using host keytab file {} for login", keytabFullPath);
- localKeytabFile = new File(keytabFullPath);
- }
- return localKeytabFile;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java
deleted file mode 100644
index 54f384b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
-
-/**
- * Cluster services offered by the YARN infrastructure.
- */
-public abstract class AbstractClusterServices {
-
- private final DefaultResourceCalculator
- defaultResourceCalculator = new DefaultResourceCalculator();
-
- /**
- * Create a resource for requests
- * @return a resource which can be built up.
- */
- public abstract Resource newResource();
-
- public abstract Resource newResource(int memory, int cores);
-
- /**
- * Normalise memory, CPU and other resources according to the YARN AM-supplied
- * values and the resource calculator in use (currently hard-coded to the
- * {@link DefaultResourceCalculator}.
- * Those resources which aren't normalized (currently: CPU) are left
- * as is.
- * @param resource resource requirements of a role
- * @param minR minimum values of this queue
- * @param maxR max values of this queue
- * @return a normalized value.
- */
- public Resource normalize(Resource resource, Resource minR, Resource maxR) {
- Preconditions.checkArgument(resource != null, "null resource");
- Preconditions.checkArgument(minR != null, "null minR");
- Preconditions.checkArgument(maxR != null, "null maxR");
-
- Resource normalize = defaultResourceCalculator.normalize(resource, minR,
- maxR, minR);
- return newResource(normalize.getMemory(), resource.getVirtualCores());
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[52/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
deleted file mode 100644
index 7baa284..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.containerlaunch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Build a single command line to include in the container commands;
- * Special support for JVM command buildup.
- */
-public class CommandLineBuilder {
- protected final List<String> argumentList = new ArrayList<>(20);
-
- /**
- * Add an entry to the command list
- * @param args arguments -these will be converted strings
- */
- public void add(Object... args) {
- for (Object arg : args) {
- argumentList.add(arg.toString());
- }
- }
-
- // Get the number of arguments
- public int size() {
- return argumentList.size();
- }
-
- /**
- * Append the output and error files to the tail of the command
- * @param stdout out
- * @param stderr error. Set this to null to append into stdout
- */
- public void addOutAndErrFiles(String stdout, String stderr) {
- Preconditions.checkNotNull(stdout, "Null output file");
- Preconditions.checkState(!stdout.isEmpty(), "output filename invalid");
- // write out the path output
- argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
- stdout);
- if (stderr != null) {
- argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
- stderr);
- } else {
- argumentList.add("2>&1");
- }
- }
-
- /**
- * This just returns the command line
- * @see #build()
- * @return the command line
- */
- @Override
- public String toString() {
- return build();
- }
-
- /**
- * Build the command line
- * @return the command line
- */
- public String build() {
- return SliderUtils.join(argumentList, " ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
deleted file mode 100644
index fcbb69b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.containerlaunch;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.provider.ProviderService;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-public class ContainerLaunchService extends AbstractService{
-
- protected static final Logger LOG =
- LoggerFactory.getLogger(ContainerLaunchService.class);
-
- private ExecutorService executorService;
- private SliderFileSystem fs;
-
- public ContainerLaunchService(SliderFileSystem fs) {
- super(ContainerLaunchService.class.getName());
- this.fs = fs;
- }
-
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- executorService = Executors.newCachedThreadPool();
- super.serviceInit(conf);
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (executorService != null) {
- executorService.shutdownNow();
- }
- super.serviceStop();
- }
-
- public void launchCompInstance(Application application,
- ComponentInstance instance, Container container) {
- ContainerLauncher launcher =
- new ContainerLauncher(application, instance, container);
- executorService.execute(launcher);
- }
-
- private class ContainerLauncher implements Runnable {
- public final Container container;
- public final Application application;
- public ComponentInstance instance;
-
- public ContainerLauncher(
- Application application,
- ComponentInstance instance, Container container) {
- this.container = container;
- this.application = application;
- this.instance = instance;
- }
-
- @Override public void run() {
- Component compSpec = instance.getCompSpec();
- ProviderService provider = ProviderFactory.getProviderService(
- compSpec.getArtifact());
- AbstractLauncher launcher = new AbstractLauncher(fs, null);
- try {
- provider.buildContainerLaunchContext(launcher, application,
- instance, fs, getConfig());
- instance.getComponent().getScheduler().getNmClient()
- .startContainerAsync(container,
- launcher.completeContainerLaunch());
- } catch (Exception e) {
- LOG.error(instance.getCompInstanceId()
- + ": Failed to launch container. ", e);
-
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
deleted file mode 100644
index fce58e5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
+++ /dev/null
@@ -1,319 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.containerlaunch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.client.ClientRMProxy;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.conf.HAUtil;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.text.DateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
-
-/**
- * Utils to work with credentials and tokens.
- *
- * Designed to be movable to Hadoop core
- */
-public final class CredentialUtils {
-
- private CredentialUtils() {
- }
-
- private static final Logger LOG =
- LoggerFactory.getLogger(CredentialUtils.class);
-
- /**
- * Save credentials to a byte buffer. Returns null if there were no
- * credentials to save
- * @param credentials credential set
- * @return a byte buffer of serialized tokens
- * @throws IOException if the credentials could not be written to the stream
- */
- public static ByteBuffer marshallCredentials(Credentials credentials) throws IOException {
- ByteBuffer buffer = null;
- if (!credentials.getAllTokens().isEmpty()) {
- DataOutputBuffer dob = new DataOutputBuffer();
- try {
- credentials.writeTokenStorageToStream(dob);
- } finally {
- dob.close();
- }
- buffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
- }
- return buffer;
- }
-
- /**
- * Save credentials to a file
- * @param file file to save to (will be overwritten)
- * @param credentials credentials to write
- * @throws IOException
- */
- public static void saveTokens(File file,
- Credentials credentials) throws IOException {
- try(DataOutputStream daos = new DataOutputStream(
- new FileOutputStream(file))) {
- credentials.writeTokenStorageToStream(daos);
- }
- }
-
- /**
- * Look up and return the resource manager's principal. This method
- * automatically does the <code>_HOST</code> replacement in the principal and
- * correctly handles HA resource manager configurations.
- *
- * From: YARN-4629
- * @param conf the {@link Configuration} file from which to read the
- * principal
- * @return the resource manager's principal string
- * @throws IOException thrown if there's an error replacing the host name
- */
- public static String getRMPrincipal(Configuration conf) throws IOException {
- String principal = conf.get(RM_PRINCIPAL, "");
- String hostname;
- Preconditions.checkState(!principal.isEmpty(), "Not set: " + RM_PRINCIPAL);
-
- if (HAUtil.isHAEnabled(conf)) {
- YarnConfiguration yarnConf = new YarnConfiguration(conf);
- if (yarnConf.get(RM_HA_ID) == null) {
- // If RM_HA_ID is not configured, use the first of RM_HA_IDS.
- // Any valid RM HA ID should work.
- String[] rmIds = yarnConf.getStrings(RM_HA_IDS);
- Preconditions.checkState((rmIds != null) && (rmIds.length > 0),
- "Not set " + RM_HA_IDS);
- yarnConf.set(RM_HA_ID, rmIds[0]);
- }
-
- hostname = yarnConf.getSocketAddr(
- RM_ADDRESS,
- DEFAULT_RM_ADDRESS,
- DEFAULT_RM_PORT).getHostName();
- } else {
- hostname = conf.getSocketAddr(
- RM_ADDRESS,
- DEFAULT_RM_ADDRESS,
- DEFAULT_RM_PORT).getHostName();
- }
- return SecurityUtil.getServerPrincipal(principal, hostname);
- }
-
- /**
- * Create and add any filesystem delegation tokens with
- * the RM(s) configured to be able to renew them. Returns null
- * on an insecure cluster (i.e. harmless)
- * @param conf configuration
- * @param fs filesystem
- * @param credentials credentials to update
- * @return a list of all added tokens.
- * @throws IOException
- */
- public static Token<?>[] addRMRenewableFSDelegationTokens(Configuration conf,
- FileSystem fs,
- Credentials credentials) throws IOException {
- Preconditions.checkArgument(conf != null);
- Preconditions.checkArgument(credentials != null);
- if (UserGroupInformation.isSecurityEnabled()) {
- return fs.addDelegationTokens(CredentialUtils.getRMPrincipal(conf),
- credentials);
- }
- return null;
- }
-
- /**
- * Add an FS delegation token which can be renewed by the current user
- * @param fs filesystem
- * @param credentials credentials to update
- * @throws IOException problems.
- */
- public static void addSelfRenewableFSDelegationTokens(
- FileSystem fs,
- Credentials credentials) throws IOException {
- Preconditions.checkArgument(fs != null);
- Preconditions.checkArgument(credentials != null);
- fs.addDelegationTokens(
- getSelfRenewer(),
- credentials);
- }
-
- public static String getSelfRenewer() throws IOException {
- return UserGroupInformation.getLoginUser().getShortUserName();
- }
-
- /**
- * Create and add an RM delegation token to the credentials
- * @param yarnClient Yarn Client
- * @param credentials to add token to
- * @return the token which was added
- * @throws IOException
- * @throws YarnException
- */
- public static Token<TokenIdentifier> addRMDelegationToken(YarnClient yarnClient,
- Credentials credentials)
- throws IOException, YarnException {
- Configuration conf = yarnClient.getConfig();
- Text rmPrincipal = new Text(CredentialUtils.getRMPrincipal(conf));
- Text rmDTService = ClientRMProxy.getRMDelegationTokenService(conf);
- Token<TokenIdentifier> rmDelegationToken =
- ConverterUtils.convertFromYarn(
- yarnClient.getRMDelegationToken(rmPrincipal),
- rmDTService);
- credentials.addToken(rmDelegationToken.getService(), rmDelegationToken);
- return rmDelegationToken;
- }
-
- public static Token<TimelineDelegationTokenIdentifier> maybeAddTimelineToken(
- Configuration conf,
- Credentials credentials)
- throws IOException, YarnException {
- if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
- LOG.debug("Timeline service enabled -fetching token");
-
- try(TimelineClient timelineClient = TimelineClient.createTimelineClient()) {
- timelineClient.init(conf);
- timelineClient.start();
- Token<TimelineDelegationTokenIdentifier> token =
- timelineClient.getDelegationToken(
- CredentialUtils.getRMPrincipal(conf));
- credentials.addToken(token.getService(), token);
- return token;
- }
- } else {
- LOG.debug("Timeline service is disabled");
- return null;
- }
- }
-
- /**
- * Filter a list of tokens from a set of credentials
- * @param credentials credential source (a new credential set os re
- * @param filter List of tokens to strip out
- * @return a new, filtered, set of credentials
- */
- public static Credentials filterTokens(Credentials credentials,
- List<Text> filter) {
- Credentials result = new Credentials(credentials);
- Iterator<Token<? extends TokenIdentifier>> iter =
- result.getAllTokens().iterator();
- while (iter.hasNext()) {
- Token<? extends TokenIdentifier> token = iter.next();
- LOG.debug("Token {}", token.getKind());
- if (filter.contains(token.getKind())) {
- LOG.debug("Filtering token {}", token.getKind());
- iter.remove();
- }
- }
- return result;
- }
-
- public static String dumpTokens(Credentials credentials, String separator) {
- ArrayList<Token<? extends TokenIdentifier>> sorted =
- new ArrayList<>(credentials.getAllTokens());
- Collections.sort(sorted, new TokenComparator());
- StringBuilder buffer = new StringBuilder(sorted.size()* 128);
- for (Token<? extends TokenIdentifier> token : sorted) {
- buffer.append(tokenToString(token)).append(separator);
- }
- return buffer.toString();
- }
-
- /**
- * Create a string for people to look at
- * @param token token to convert to a string form
- * @return a printable view of the token
- */
- public static String tokenToString(Token<? extends TokenIdentifier> token) {
- DateFormat df = DateFormat.getDateTimeInstance(
- DateFormat.SHORT, DateFormat.SHORT);
- StringBuilder buffer = new StringBuilder(128);
- buffer.append(token.toString());
- try {
- TokenIdentifier ti = token.decodeIdentifier();
- buffer.append("; ").append(ti);
- if (ti instanceof AbstractDelegationTokenIdentifier) {
- // details in human readable form, and compensate for information HDFS DT omits
- AbstractDelegationTokenIdentifier dt = (AbstractDelegationTokenIdentifier) ti;
- buffer.append("; Renewer: ").append(dt.getRenewer());
- buffer.append("; Issued: ")
- .append(df.format(new Date(dt.getIssueDate())));
- buffer.append("; Max Date: ")
- .append(df.format(new Date(dt.getMaxDate())));
- }
- } catch (IOException e) {
- //marshall problem; not ours
- LOG.debug("Failed to decode {}: {}", token, e, e);
- }
- return buffer.toString();
- }
-
- /**
- * Get the expiry time of a token.
- * @param token token to examine
- * @return the time in milliseconds after which the token is invalid.
- * @throws IOException
- */
- public static long getTokenExpiryTime(Token token) throws IOException {
- TokenIdentifier identifier = token.decodeIdentifier();
- Preconditions.checkState(identifier instanceof AbstractDelegationTokenIdentifier,
- "Token %s of type: %s has an identifier which cannot be examined: %s",
- token, token.getClass(), identifier);
- AbstractDelegationTokenIdentifier id =
- (AbstractDelegationTokenIdentifier) identifier;
- return id.getMaxDate();
- }
-
- private static class TokenComparator
- implements Comparator<Token<? extends TokenIdentifier>>, Serializable {
- @Override
- public int compare(Token<? extends TokenIdentifier> left,
- Token<? extends TokenIdentifier> right) {
- return left.getKind().toString().compareTo(right.getKind().toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
deleted file mode 100644
index cbcb0d6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.containerlaunch;
-
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-
-import java.util.Map;
-
-/**
- * Command line builder purely for the Java CLI.
- * Some of the <code>define</code> methods are designed to work with Hadoop tool and
- * Slider launcher applications.
- */
-public class JavaCommandLineBuilder extends CommandLineBuilder {
-
- public JavaCommandLineBuilder() {
- add(getJavaBinary());
- }
-
- /**
- * Get the java binary. This is called in the constructor so don't try and
- * do anything other than return a constant.
- * @return the path to the Java binary
- */
- protected String getJavaBinary() {
- return ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java";
- }
-
- /**
- * Set the size of the heap if a non-empty heap is passed in.
- * @param heap empty string or something like "128M" ,"1G" etc. The value is
- * trimmed.
- */
- public void setJVMHeap(String heap) {
- if (SliderUtils.isSet(heap)) {
- add("-Xmx" + heap.trim());
- }
- }
-
- /**
- * Turn Java assertions on
- */
- public void enableJavaAssertions() {
- add("-ea");
- add("-esa");
- }
-
- /**
- * Add a system property definition -must be used before setting the main entry point
- * @param property
- * @param value
- */
- public void sysprop(String property, String value) {
- Preconditions.checkArgument(property != null, "null property name");
- Preconditions.checkArgument(value != null, "null value");
- add("-D" + property + "=" + value);
- }
-
- public JavaCommandLineBuilder forceIPv4() {
- sysprop("java.net.preferIPv4Stack", "true");
- return this;
- }
-
- public JavaCommandLineBuilder headless() {
- sysprop("java.awt.headless", "true");
- return this;
- }
-
- public boolean addConfOption(Configuration conf, String key) {
- return defineIfSet(key, conf.get(key));
- }
-
- /**
- * Add a varargs list of configuration parameters —if they are present
- * @param conf configuration source
- * @param keys keys
- */
- public void addConfOptions(Configuration conf, String... keys) {
- for (String key : keys) {
- addConfOption(conf, key);
- }
- }
-
- /**
- * Add all configuration options which match the prefix
- * @param conf configuration
- * @param prefix prefix, e.g {@code "slider."}
- * @return the number of entries copied
- */
- public int addPrefixedConfOptions(Configuration conf, String prefix) {
- int copied = 0;
- for (Map.Entry<String, String> entry : conf) {
- if (entry.getKey().startsWith(prefix)) {
- define(entry.getKey(), entry.getValue());
- copied++;
- }
- }
- return copied;
- }
-
- /**
- * Ass a configuration option to the command line of the application
- * @param conf configuration
- * @param key key
- * @param defVal default value
- * @return the resolved configuration option
- * @throws IllegalArgumentException if key is null or the looked up value
- * is null (that is: the argument is missing and devVal was null.
- */
- public String addConfOptionToCLI(Configuration conf,
- String key,
- String defVal) {
- Preconditions.checkArgument(key != null, "null key");
- String val = conf.get(key, defVal);
- define(key, val);
- return val;
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI. This is very Hadoop API
- * @param key key
- * @param val value
- * @throws IllegalArgumentException if either argument is null
- */
- public void define(String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- Preconditions.checkArgument(val != null, "null value");
- add("-D", key + "=" + val);
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
- * is not null
- * @param key key
- * @param val value
- */
- public boolean defineIfSet(String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- if (val != null) {
- define(key, val);
- return true;
- } else {
- return false;
- }
- }
-
- /**
- * Add a mandatory config option
- * @param conf configuration
- * @param key key
- * @throws BadConfigException if the key is missing
- */
- public void addMandatoryConfOption(Configuration conf,
- String key) throws BadConfigException {
- if (!addConfOption(conf, key)) {
- throw new BadConfigException("Missing configuration option: " + key);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
deleted file mode 100644
index db9de7a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-
-/**
- * The system is in a bad state
- */
-public class BadClusterStateException extends SliderException {
- public BadClusterStateException(String message,
- Object... args) {
- super(EXIT_BAD_STATE, message, args);
- }
-
- public BadClusterStateException(Throwable throwable,
- String message, Object... args) {
- super(EXIT_BAD_STATE, throwable, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
deleted file mode 100644
index 41e3251..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-public class BadCommandArgumentsException extends SliderException {
- public BadCommandArgumentsException(String s, Object... args) {
- super(EXIT_COMMAND_ARGUMENT_ERROR, s, args);
- }
-
- public BadCommandArgumentsException(Throwable throwable, String message,
- Object... args) {
- super(EXIT_COMMAND_ARGUMENT_ERROR, throwable, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
deleted file mode 100644
index 8199c3c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-/**
- * An exception to raise on a bad configuration
- */
-public class BadConfigException extends SliderException {
-
- public BadConfigException(String s) {
- super(EXIT_BAD_CONFIGURATION, s);
- }
-
- public BadConfigException(String message, Object... args) {
- super(EXIT_BAD_CONFIGURATION, message, args);
- }
-
- public BadConfigException(
- Throwable throwable,
- String message, Object... args) {
- super(EXIT_BAD_CONFIGURATION, throwable, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
deleted file mode 100644
index 3577b59..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-public interface ErrorStrings {
- String E_UNSTABLE_CLUSTER = "Unstable Application Instance :";
- String E_CLUSTER_RUNNING = "Application Instance running";
- String E_ALREADY_EXISTS = "already exists";
- String PRINTF_E_INSTANCE_ALREADY_EXISTS = "Application Instance \"%s\" already exists and is defined in %s";
- String PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS = "Application Instance dir already exists: %s";
- String E_MISSING_PATH = "Missing path ";
- String E_INCOMPLETE_CLUSTER_SPEC =
- "Cluster specification is marked as incomplete: ";
- String E_UNKNOWN_INSTANCE = "Unknown application instance ";
- String E_DESTROY_CREATE_RACE_CONDITION =
- "created while it was being destroyed";
- String E_UNKNOWN_ROLE = "Unknown role ";
- /**
- * ERROR Strings
- */
- String ERROR_NO_ACTION = "No action specified";
- String ERROR_UNKNOWN_ACTION = "Unknown command: ";
- String ERROR_NOT_ENOUGH_ARGUMENTS =
- "Not enough arguments for action: ";
- String ERROR_PARSE_FAILURE =
- "Failed to parse ";
- /**
- * All the remaining values after argument processing
- */
- String ERROR_TOO_MANY_ARGUMENTS =
- "Too many arguments";
- String ERROR_DUPLICATE_ENTRY = "Duplicate entry for ";
- String E_APPLICATION_NOT_RUNNING = "Application not running";
- String E_FINISHED_APPLICATION = E_APPLICATION_NOT_RUNNING + ": %s state=%s ";
- String E_NO_IMAGE_OR_HOME_DIR_SPECIFIED =
- "Neither an image path nor binary home directory were specified";
- String E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED =
- "Both application image path and home dir have been provided";
- String E_CONFIGURATION_DIRECTORY_NOT_FOUND =
- "Configuration directory \"%s\" not found";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
deleted file mode 100644
index d66b860..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-/**
- * Get the exit code of an exception. Making it an interface allows
- * us to retrofit exit codes onto existing classes
- */
-public interface ExitCodeProvider {
-
- /**
- * Method to get the exit code
- * @return the exit code
- */
- int getExitCode();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
deleted file mode 100644
index 9657536..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-/*
- * Common Exit codes
- * <p>
- * Exit codes from 64 up are application specific.
- * <p>
- * Many of the exit codes are designed to resemble HTTP error codes,
- * squashed into a single byte. e.g 44 , "not found" is the equivalent
- * of 404
- * <pre>
- * 0-10: general command issues
- * 30-39: equivalent to the 3XX responses, where those responses are
- * considered errors by the application.
- * 40-49: request-related errors
- * 50-59: server-side problems. These may be triggered by the request.
- * 64- : application specific error codes
- * </pre>
- */
-public interface LauncherExitCodes {
-
- /**
- * 0: success
- */
- int EXIT_SUCCESS = 0;
-
- /**
- * -1: generic "false" response. The operation worked but
- * the result was not true
- */
- int EXIT_FALSE = -1;
-
- /**
- * Exit code when a client requested service termination: {@value}
- */
- int EXIT_CLIENT_INITIATED_SHUTDOWN = 1;
-
- /**
- * Exit code when targets could not be launched: {@value}
- */
- int EXIT_TASK_LAUNCH_FAILURE = 2;
-
- /**
- * Exit code when a control-C, kill -3, signal was picked up: {@value}
- */
- int EXIT_INTERRUPTED = 3;
-
- /**
- * Exit code when a usage message was printed: {@value}
- */
- int EXIT_USAGE = 4;
-
- /**
- * Exit code when something happened but we can't be specific: {@value}
- */
- int EXIT_OTHER_FAILURE = 5;
-
- /**
- * Exit code on connectivity problems: {@value}
- */
- int EXIT_MOVED = 31;
-
- /**
- * found: {@value}.
- * <p>
- * This is low value as in HTTP it is normally a success/redirect;
- * whereas on the command line 0 is the sole success code.
- * <p>
- * <code>302 Found</code>
- */
- int EXIT_FOUND = 32;
-
- /**
- * Exit code on a request where the destination has not changed
- * and (somehow) the command specified that this is an error.
- * That is, this exit code is somehow different from a "success"
- * : {@value}
- * <p>
- * <code>304 Not Modified </code>
- */
- int EXIT_NOT_MODIFIED = 34;
-
- /**
- * Exit code when the command line doesn't parse: {@value}, or
- * when it is otherwise invalid.
- * <p>
- * <code>400 BAD REQUEST</code>
- */
- int EXIT_COMMAND_ARGUMENT_ERROR = 40;
-
- /**
- * The request requires user authentication: {@value}
- * <p>
- * <code>401 Unauthorized</code>
- */
- int EXIT_UNAUTHORIZED = 41;
-
- /**
- * Forbidden action: {@value}
- * <p>
- * <code>403: Forbidden</code>
- */
- int EXIT_FORBIDDEN = 43;
-
- /**
- * Something was not found: {@value}
- * <p>
- * <code>404: NOT FOUND</code>
- */
- int EXIT_NOT_FOUND = 44;
-
- /**
- * The operation is not allowed: {@value}
- * <p>
- * <code>405: NOT ALLOWED</code>
- */
- int EXIT_OPERATION_NOT_ALLOWED = 45;
-
- /**
- * The command is somehow not acceptable: {@value}
- * <p>
- * <code>406: NOT ACCEPTABLE</code>
- */
- int EXIT_NOT_ACCEPTABLE = 46;
-
- /**
- * Exit code on connectivity problems: {@value}
- * <p>
- * <code>408: Request Timeout</code>
- */
- int EXIT_CONNECTIVITY_PROBLEM = 48;
-
- /**
- * The request could not be completed due to a conflict with the current
- * state of the resource. {@value}
- * <p>
- * <code>409: conflict</code>
- */
- int EXIT_CONFLICT = 49;
-
- /**
- * internal error: {@value}
- * <p>
- * <code>500 Internal Server Error</code>
- */
- int EXIT_INTERNAL_ERROR = 50;
-
- /**
- * Unimplemented feature: {@value}
- * <p>
- * <code>501: Not Implemented</code>
- */
- int EXIT_UNIMPLEMENTED = 51;
-
- /**
- * Service Unavailable; it may be available later: {@value}
- * <p>
- * <code>503 Service Unavailable</code>
- */
- int EXIT_SERVICE_UNAVAILABLE = 53;
-
- /**
- * The application does not support, or refuses to support this version: {@value}.
- * If raised, this is expected to be raised server-side and likely due
- * to client/server version incompatibilities.
- * <p>
- * <code> 505: Version Not Supported</code>
- */
- int EXIT_UNSUPPORTED_VERSION = 55;
-
- /**
- * Exit code when an exception was thrown from the service: {@value}
- * <p>
- * <code>5XX</code>
- */
- int EXIT_EXCEPTION_THROWN = 56;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
deleted file mode 100644
index 7be23f3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-public interface RestApiErrorMessages {
- String ERROR_APPLICATION_NAME_INVALID =
- "Application name is either empty or not provided";
- String ERROR_APPLICATION_NAME_INVALID_FORMAT =
- "Application name %s is not valid - only lower case letters, digits, " +
- "underscore and hyphen are allowed, and the name must be no more " +
- "than 63 characters";
- String ERROR_COMPONENT_NAME_INVALID =
- "Component name must be no more than %s characters: %s";
- String ERROR_USER_NAME_INVALID =
- "User name must be no more than 63 characters";
-
- String ERROR_APPLICATION_NOT_RUNNING = "Application not running";
- String ERROR_APPLICATION_DOES_NOT_EXIST = "Application not found";
- String ERROR_APPLICATION_IN_USE = "Application already exists in started"
- + " state";
- String ERROR_APPLICATION_INSTANCE_EXISTS = "Application already exists in"
- + " stopped/failed state (either restart with PUT or destroy with DELETE"
- + " before creating a new one)";
-
- String ERROR_SUFFIX_FOR_COMPONENT =
- " for component %s (nor at the global level)";
- String ERROR_ARTIFACT_INVALID = "Artifact is not provided";
- String ERROR_ARTIFACT_FOR_COMP_INVALID =
- ERROR_ARTIFACT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
- String ERROR_ARTIFACT_ID_INVALID =
- "Artifact id (like docker image name) is either empty or not provided";
- String ERROR_ARTIFACT_ID_FOR_COMP_INVALID =
- ERROR_ARTIFACT_ID_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
-
- String ERROR_RESOURCE_INVALID = "Resource is not provided";
- String ERROR_RESOURCE_FOR_COMP_INVALID =
- ERROR_RESOURCE_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
- String ERROR_RESOURCE_MEMORY_INVALID =
- "Application resource or memory not provided";
- String ERROR_RESOURCE_CPUS_INVALID =
- "Application resource or cpus not provided";
- String ERROR_RESOURCE_CPUS_INVALID_RANGE =
- "Unacceptable no of cpus specified, either zero or negative";
- String ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID =
- ERROR_RESOURCE_MEMORY_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
- String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID =
- ERROR_RESOURCE_CPUS_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
- String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE =
- ERROR_RESOURCE_CPUS_INVALID_RANGE
- + " for component %s (or at the global level)";
- String ERROR_CONTAINERS_COUNT_INVALID =
- "Invalid no of containers specified";
- String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID =
- ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
- String ERROR_DEPENDENCY_INVALID = "Dependency %s for component %s is " +
- "invalid, does not exist as a component";
- String ERROR_DEPENDENCY_CYCLE = "Invalid dependencies, a cycle may " +
- "exist: %s";
-
- String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED =
- "Cannot specify" + " cpus/memory along with profile";
- String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED =
- ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED
- + " for component %s";
- String ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET =
- "Resource profile is not " + "supported yet. Please specify cpus/memory.";
-
- String ERROR_NULL_ARTIFACT_ID =
- "Artifact Id can not be null if artifact type is none";
- String ERROR_ABSENT_NUM_OF_INSTANCE =
- "Num of instances should appear either globally or per component";
- String ERROR_ABSENT_LAUNCH_COMMAND =
- "Launch_command is required when type is not DOCKER";
-
- String ERROR_QUICKLINKS_FOR_COMP_INVALID = "Quicklinks specified at"
- + " component level, needs corresponding values set at application level";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
deleted file mode 100644
index e83ccbe..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-
-import org.apache.hadoop.yarn.exceptions.YarnException;
-
-/**
- * A service launch exception that includes an exit code;
- * when caught by the ServiceLauncher, it will convert that
- * into a process exit code.
- */
-public class ServiceLaunchException extends YarnException
- implements ExitCodeProvider, LauncherExitCodes {
-
- private final int exitCode;
-
- /**
- * Create an exception with the specific exit code
- * @param exitCode exit code
- * @param cause cause of the exception
- */
- public ServiceLaunchException(int exitCode, Throwable cause) {
- super(cause);
- this.exitCode = exitCode;
- }
-
- /**
- * Create an exception with the specific exit code and text
- * @param exitCode exit code
- * @param message message to use in exception
- */
- public ServiceLaunchException(int exitCode, String message) {
- super(message);
- this.exitCode = exitCode;
- }
-
- /**
- * Create an exception with the specific exit code, text and cause
- * @param exitCode exit code
- * @param message message to use in exception
- * @param cause cause of the exception
- */
- public ServiceLaunchException(int exitCode, String message, Throwable cause) {
- super(message, cause);
- this.exitCode = exitCode;
- }
-
- /**
- * Get the exit code
- * @return the exit code
- */
- @Override
- public int getExitCode() {
- return exitCode;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
deleted file mode 100644
index 5b74b80..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-
-public class SliderException extends ServiceLaunchException implements
- SliderExitCodes {
- public SliderException() {
- super(EXIT_EXCEPTION_THROWN, "SliderException");
- }
-
- public SliderException(int code, String message) {
- super(code, message);
- }
-
- public SliderException(String s) {
- super(EXIT_EXCEPTION_THROWN, s);
- }
-
- public SliderException(String s, Throwable throwable) {
- super(EXIT_EXCEPTION_THROWN, s, throwable);
- }
-
- /**
- * Format the exception as you create it
- * @param code exit code
- * @param message exception message -sprintf formatted
- * @param args arguments for the formatting
- */
- public SliderException(int code, String message, Object... args) {
- super(code, String.format(message, args));
- }
-
- /**
- * Format the exception, include a throwable.
- * The throwable comes before the message so that it is out of the varargs
- * @param code exit code
- * @param throwable thrown
- * @param message message
- * @param args arguments
- */
- public SliderException(int code,
- Throwable throwable,
- String message,
- Object... args) {
- super(code, String.format(message, args), throwable);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
deleted file mode 100644
index 3a9fa25..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-/**
- * Used to raise a usage exception ... this has the exit code
- * {@link #EXIT_USAGE}
- */
-public class UsageException extends SliderException {
- public UsageException(String s, Object... args) {
- super(EXIT_USAGE, s, args);
- }
-
- public UsageException(Throwable throwable, String message,
- Object... args) {
- super(EXIT_USAGE, throwable, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
deleted file mode 100644
index 33e33a6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.impl.pb.client;
-
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.RPCUtil;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
-import org.apache.hadoop.yarn.service.impl.pb.service.ClientAMProtocolPB;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
-
-public class ClientAMProtocolPBClientImpl
- implements ClientAMProtocol, Closeable {
-
- private ClientAMProtocolPB proxy;
-
- public ClientAMProtocolPBClientImpl(long clientVersion,
- InetSocketAddress addr, Configuration conf) throws IOException {
- RPC.setProtocolEngine(conf, ClientAMProtocolPB.class,
- ProtobufRpcEngine.class);
- proxy = RPC.getProxy(ClientAMProtocolPB.class, clientVersion, addr, conf);
-
- }
-
- @Override public FlexComponentsResponseProto flexComponents(
- FlexComponentsRequestProto request) throws IOException, YarnException {
- try {
- return proxy.flexComponents(null, request);
- } catch (ServiceException e) {
- RPCUtil.unwrapAndThrowException(e);
- }
- return null;
- }
-
- @Override
- public GetStatusResponseProto getStatus(GetStatusRequestProto request)
- throws IOException, YarnException {
- try {
- return proxy.getStatus(null, request);
- } catch (ServiceException e) {
- RPCUtil.unwrapAndThrowException(e);
- }
- return null;
- }
-
- @Override
- public StopResponseProto stop(StopRequestProto requestProto)
- throws IOException, YarnException {
- try {
- return proxy.stop(null, requestProto);
- } catch (ServiceException e) {
- RPCUtil.unwrapAndThrowException(e);
- }
- return null;
- }
-
- @Override public void close() {
- if (this.proxy != null) {
- RPC.stopProxy(this.proxy);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
deleted file mode 100644
index 6a9cd37..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.impl.pb.service;
-
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol;
-
-@ProtocolInfo(
- protocolName = "org.apache.hadoop.yarn.service.ClientAMProtocol",
- protocolVersion = 1)
-public interface ClientAMProtocolPB extends
- ClientAMProtocol.ClientAMProtocolService.BlockingInterface {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
deleted file mode 100644
index 7100781..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.impl.pb.service;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
-
-import java.io.IOException;
-
-public class ClientAMProtocolPBServiceImpl implements ClientAMProtocolPB {
-
- private ClientAMProtocol real;
-
- public ClientAMProtocolPBServiceImpl(ClientAMProtocol impl) {
- this.real = impl;
- }
-
- @Override
- public FlexComponentsResponseProto flexComponents(RpcController controller,
- FlexComponentsRequestProto request) throws ServiceException {
- try {
- return real.flexComponents(request);
- } catch (IOException | YarnException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override public GetStatusResponseProto getStatus(RpcController controller,
- GetStatusRequestProto request) throws ServiceException {
- try {
- return real.getStatus(request);
- } catch (IOException | YarnException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto stop(
- RpcController controller,
- org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto request)
- throws ServiceException {
- try {
- return real.stop(request);
- } catch (IOException | YarnException e) {
- throw new ServiceException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java
deleted file mode 100644
index bfe3bc9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.metrics;
-
-import org.apache.hadoop.metrics2.MetricsCollector;
-import org.apache.hadoop.metrics2.MetricsInfo;
-import org.apache.hadoop.metrics2.MetricsSource;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MetricsRegistry;
-import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
-
-import static org.apache.hadoop.metrics2.lib.Interns.info;
-
-@Metrics(context = "yarn-native-service")
-public class ServiceMetrics implements MetricsSource {
-
- @Metric("containers requested")
- public MutableGaugeInt containersRequested;
-
- @Metric("anti-affinity containers pending")
- public MutableGaugeInt pendingAAContainers;
-
- @Metric("containers running")
- public MutableGaugeInt containersRunning;
-
- @Metric("containers ready")
- public MutableGaugeInt containersReady;
-
- @Metric("containers desired")
- public MutableGaugeInt containersDesired;
-
- @Metric("containers succeeded")
- public MutableGaugeInt containersSucceeded;
-
- @Metric("containers failed")
- public MutableGaugeInt containersFailed;
-
- @Metric("containers preempted")
- public MutableGaugeInt containersPreempted;
-
- @Metric("containers exceeded limits")
- public MutableGaugeInt containersLimitsExceeded;
-
- @Metric("containers surplus")
- public MutableGaugeInt surplusContainers;
-
- @Metric("containers failed due to disk failure")
- public MutableGaugeInt containersDiskFailure;
-
- protected final MetricsRegistry registry;
-
- public ServiceMetrics(MetricsInfo metricsInfo) {
- registry = new MetricsRegistry(metricsInfo);
- }
-
- @Override
- public void getMetrics(MetricsCollector collector, boolean all) {
- registry.snapshot(collector.addRecord(registry.info()), all);
- }
-
- public static ServiceMetrics register(String name, String description) {
- ServiceMetrics metrics = new ServiceMetrics(info(name, description));
- DefaultMetricsSystem.instance().register(name, description, metrics);
- return metrics;
- }
-
- public void tag(String name, String description, String value) {
- registry.tag(name, description, value);
- }
-
- @Override public String toString() {
- return "ServiceMetrics{"
- + "containersRequested=" + containersRequested.value()
- + ", pendingAAContainers=" + pendingAAContainers.value()
- + ", containersRunning=" + containersRunning.value()
- + ", containersDesired=" + containersDesired.value()
- + ", containersSucceeded=" + containersSucceeded.value()
- + ", containersFailed=" + containersFailed.value()
- + ", containersPreempted=" + containersPreempted.value()
- + ", surplusContainers=" + surplusContainers.value() + '}';
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
deleted file mode 100644
index 0d11be2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.provider;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-public abstract class AbstractClientProvider {
-
- public AbstractClientProvider() {
- }
-
- /**
- * Generates a fixed format of application tags given one or more of
- * application name, version and description. This allows subsequent query for
- * an application with a name only, version only or description only or any
- * combination of those as filters.
- *
- * @param appName name of the application
- * @param appVersion version of the application
- * @param appDescription brief description of the application
- * @return
- */
- public static final Set<String> createApplicationTags(String appName,
- String appVersion, String appDescription) {
- Set<String> tags = new HashSet<>();
- tags.add(SliderUtils.createNameTag(appName));
- if (appVersion != null) {
- tags.add(SliderUtils.createVersionTag(appVersion));
- }
- if (appDescription != null) {
- tags.add(SliderUtils.createDescriptionTag(appDescription));
- }
- return tags;
- }
-
- /**
- * Validate the artifact.
- * @param artifact
- */
- public abstract void validateArtifact(Artifact artifact, FileSystem
- fileSystem) throws IOException;
-
- protected abstract void validateConfigFile(ConfigFile configFile, FileSystem
- fileSystem) throws IOException;
-
- /**
- * Validate the config files.
- * @param configFiles config file list
- * @param fs file system
- */
- public void validateConfigFiles(List<ConfigFile> configFiles,
- FileSystem fs) throws IOException {
- Set<String> destFileSet = new HashSet<>();
-
- for (ConfigFile file : configFiles) {
- if (file.getType() == null) {
- throw new IllegalArgumentException("File type is empty");
- }
-
- if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
- .isEmpty(file.getSrcFile())) {
- throw new IllegalArgumentException(
- "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE);
-
- }
- if (!StringUtils.isEmpty(file.getSrcFile())) {
- Path p = new Path(file.getSrcFile());
- if (!fs.exists(p)) {
- throw new IllegalArgumentException(
- "Src_file does not exist for config file: " + file
- .getSrcFile());
- }
- }
-
- if (StringUtils.isEmpty(file.getDestFile())) {
- throw new IllegalArgumentException("Dest_file is empty.");
- }
-
- if (destFileSet.contains(file.getDestFile())) {
- throw new IllegalArgumentException(
- "Duplicated ConfigFile exists: " + file.getDestFile());
- }
- destFileSet.add(file.getDestFile());
-
- java.nio.file.Path destPath = Paths.get(file.getDestFile());
- if (!destPath.isAbsolute() && destPath.getNameCount() > 1) {
- throw new IllegalArgumentException("Non-absolute dest_file has more " +
- "than one path element");
- }
-
- // provider-specific validation
- validateConfigFile(file, fs);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
deleted file mode 100644
index 504680d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
-import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.ServiceContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_RETRY_INTERVAL;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_RETRY_MAX;
-import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
-
-public abstract class AbstractProviderService implements ProviderService,
- YarnServiceConstants {
-
- protected static final Logger log =
- LoggerFactory.getLogger(AbstractProviderService.class);
-
- public abstract void processArtifact(AbstractLauncher launcher,
- ComponentInstance compInstance, SliderFileSystem fileSystem,
- Application application)
- throws IOException;
-
- public void buildContainerLaunchContext(AbstractLauncher launcher,
- Application application, ComponentInstance instance,
- SliderFileSystem fileSystem, Configuration yarnConf)
- throws IOException, SliderException {
- Component component = instance.getComponent().getComponentSpec();;
- processArtifact(launcher, instance, fileSystem, application);
-
- ServiceContext context =
- instance.getComponent().getScheduler().getContext();
- // Generate tokens (key-value pair) for config substitution.
- // Get pre-defined tokens
- Map<String, String> globalTokens =
- instance.getComponent().getScheduler().globalTokens;
- Map<String, String> tokensForSubstitution = ProviderUtils
- .initCompTokensForSubstitute(instance);
- tokensForSubstitution.putAll(globalTokens);
- // Set the environment variables in launcher
- launcher.putEnv(SliderUtils
- .buildEnvMap(component.getConfiguration(), tokensForSubstitution));
- launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
- launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
- if (System.getenv(HADOOP_USER_NAME) != null) {
- launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
- }
- launcher.setEnv("LANG", "en_US.UTF-8");
- launcher.setEnv("LC_ALL", "en_US.UTF-8");
- launcher.setEnv("LANGUAGE", "en_US.UTF-8");
-
- for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
- tokensForSubstitution.put($(entry.getKey()), entry.getValue());
- }
- //TODO add component host tokens?
-// ProviderUtils.addComponentHostTokens(tokensForSubstitution, amState);
-
- // create config file on hdfs and add local resource
- ProviderUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
- component, tokensForSubstitution, instance, context);
-
- // substitute launch command
- String launchCommand = ProviderUtils
- .substituteStrWithTokens(component.getLaunchCommand(),
- tokensForSubstitution);
- CommandLineBuilder operation = new CommandLineBuilder();
- operation.add(launchCommand);
- operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
- launcher.addCommand(operation.build());
-
- // By default retry forever every 30 seconds
- launcher.setRetryContext(YarnServiceConf
- .getInt(CONTAINER_RETRY_MAX, -1, application.getConfiguration(),
- yarnConf), YarnServiceConf
- .getInt(CONTAINER_RETRY_INTERVAL, 30000, application.getConfiguration(),
- yarnConf));
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
deleted file mode 100644
index b28257f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.client;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.exceptions.NoRecordException;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.NotFoundException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.slider.core.registry.retrieve.RegistryRetriever;
-
-import java.io.File;
-import java.io.IOException;
-
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
-
-public class ClientUtils {
- public static ServiceRecord lookupServiceRecord(RegistryOperations rops,
- String user, String name) throws IOException, SliderException {
- return lookupServiceRecord(rops, user, null, name);
- }
-
- public static ServiceRecord lookupServiceRecord(RegistryOperations rops,
- String user, String type, String name) throws IOException,
- SliderException {
- if (StringUtils.isEmpty(user)) {
- user = currentUser();
- } else {
- user = RegistryPathUtils.encodeForRegistry(user);
- }
- if (StringUtils.isEmpty(type)) {
- type = SliderKeys.APP_TYPE;
- }
-
- String path = servicePath(user, type, name);
- return resolve(rops, path);
- }
-
- public static ServiceRecord resolve(RegistryOperations rops, String path)
- throws IOException, SliderException {
- try {
- return rops.resolve(path);
- } catch (PathNotFoundException | NoRecordException e) {
- throw new NotFoundException(e.getPath().toString(), e);
- }
- }
-
- public static PublishedConfiguration getConfigFromRegistry(
- RegistryOperations rops, Configuration configuration,
- String configName, String appName, String user, boolean external)
- throws IOException, SliderException {
- ServiceRecord instance = lookupServiceRecord(rops, user, appName);
-
- RegistryRetriever retriever = new RegistryRetriever(configuration, instance);
- PublishedConfigSet configurations = retriever.getConfigurations(external);
-
- PublishedConfiguration published = retriever.retrieveConfiguration(
- configurations, configName, external);
- return published;
- }
-
- public static String saveOrReturnConfig(PublishedConfiguration published,
- String format, File destPath, String fileName)
- throws BadCommandArgumentsException, IOException {
- ConfigFormat configFormat = ConfigFormat.resolve(format);
- if (configFormat == null) {
- throw new BadCommandArgumentsException(
- "Unknown/Unsupported format %s ", format);
- }
- PublishedConfigurationOutputter outputter =
- PublishedConfigurationOutputter.createOutputter(configFormat,
- published);
- boolean print = destPath == null;
- if (!print) {
- if (destPath.isDirectory()) {
- // creating it under a directory
- destPath = new File(destPath, fileName);
- }
- outputter.save(destPath);
- return null;
- } else {
- return outputter.asString();
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[64/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
new file mode 100644
index 0000000..7baa284
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Build a single command line to include in the container commands;
+ * Special support for JVM command buildup.
+ */
+public class CommandLineBuilder {
+ protected final List<String> argumentList = new ArrayList<>(20);
+
+ /**
+ * Add an entry to the command list
+ * @param args arguments -these will be converted strings
+ */
+ public void add(Object... args) {
+ for (Object arg : args) {
+ argumentList.add(arg.toString());
+ }
+ }
+
+ // Get the number of arguments
+ public int size() {
+ return argumentList.size();
+ }
+
+ /**
+ * Append the output and error files to the tail of the command
+ * @param stdout out
+ * @param stderr error. Set this to null to append into stdout
+ */
+ public void addOutAndErrFiles(String stdout, String stderr) {
+ Preconditions.checkNotNull(stdout, "Null output file");
+ Preconditions.checkState(!stdout.isEmpty(), "output filename invalid");
+ // write out the path output
+ argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
+ stdout);
+ if (stderr != null) {
+ argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
+ stderr);
+ } else {
+ argumentList.add("2>&1");
+ }
+ }
+
+ /**
+ * This just returns the command line
+ * @see #build()
+ * @return the command line
+ */
+ @Override
+ public String toString() {
+ return build();
+ }
+
+ /**
+ * Build the command line
+ * @return the command line
+ */
+ public String build() {
+ return SliderUtils.join(argumentList, " ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
new file mode 100644
index 0000000..0e51a62
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class ContainerLaunchService extends AbstractService{
+
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(ContainerLaunchService.class);
+
+ private ExecutorService executorService;
+ private SliderFileSystem fs;
+
+ public ContainerLaunchService(SliderFileSystem fs) {
+ super(ContainerLaunchService.class.getName());
+ this.fs = fs;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ executorService = Executors.newCachedThreadPool();
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (executorService != null) {
+ executorService.shutdownNow();
+ }
+ super.serviceStop();
+ }
+
+ public void launchCompInstance(Service service,
+ ComponentInstance instance, Container container) {
+ ContainerLauncher launcher =
+ new ContainerLauncher(service, instance, container);
+ executorService.execute(launcher);
+ }
+
+ private class ContainerLauncher implements Runnable {
+ public final Container container;
+ public final Service service;
+ public ComponentInstance instance;
+
+ public ContainerLauncher(
+ Service service,
+ ComponentInstance instance, Container container) {
+ this.container = container;
+ this.service = service;
+ this.instance = instance;
+ }
+
+ @Override public void run() {
+ Component compSpec = instance.getCompSpec();
+ ProviderService provider = ProviderFactory.getProviderService(
+ compSpec.getArtifact());
+ AbstractLauncher launcher = new AbstractLauncher(fs, null);
+ try {
+ provider.buildContainerLaunchContext(launcher, service,
+ instance, fs, getConfig());
+ instance.getComponent().getScheduler().getNmClient()
+ .startContainerAsync(container,
+ launcher.completeContainerLaunch());
+ } catch (Exception e) {
+ LOG.error(instance.getCompInstanceId()
+ + ": Failed to launch container. ", e);
+
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
new file mode 100644
index 0000000..fce58e5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.text.DateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
+
+/**
+ * Utils to work with credentials and tokens.
+ *
+ * Designed to be movable to Hadoop core
+ */
+public final class CredentialUtils {
+
+ private CredentialUtils() {
+ }
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CredentialUtils.class);
+
+ /**
+ * Save credentials to a byte buffer. Returns null if there were no
+ * credentials to save
+ * @param credentials credential set
+ * @return a byte buffer of serialized tokens
+ * @throws IOException if the credentials could not be written to the stream
+ */
+ public static ByteBuffer marshallCredentials(Credentials credentials) throws IOException {
+ ByteBuffer buffer = null;
+ if (!credentials.getAllTokens().isEmpty()) {
+ DataOutputBuffer dob = new DataOutputBuffer();
+ try {
+ credentials.writeTokenStorageToStream(dob);
+ } finally {
+ dob.close();
+ }
+ buffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ }
+ return buffer;
+ }
+
+ /**
+ * Save credentials to a file
+ * @param file file to save to (will be overwritten)
+ * @param credentials credentials to write
+ * @throws IOException
+ */
+ public static void saveTokens(File file,
+ Credentials credentials) throws IOException {
+ try(DataOutputStream daos = new DataOutputStream(
+ new FileOutputStream(file))) {
+ credentials.writeTokenStorageToStream(daos);
+ }
+ }
+
+ /**
+ * Look up and return the resource manager's principal. This method
+ * automatically does the <code>_HOST</code> replacement in the principal and
+ * correctly handles HA resource manager configurations.
+ *
+ * From: YARN-4629
+ * @param conf the {@link Configuration} file from which to read the
+ * principal
+ * @return the resource manager's principal string
+ * @throws IOException thrown if there's an error replacing the host name
+ */
+ public static String getRMPrincipal(Configuration conf) throws IOException {
+ String principal = conf.get(RM_PRINCIPAL, "");
+ String hostname;
+ Preconditions.checkState(!principal.isEmpty(), "Not set: " + RM_PRINCIPAL);
+
+ if (HAUtil.isHAEnabled(conf)) {
+ YarnConfiguration yarnConf = new YarnConfiguration(conf);
+ if (yarnConf.get(RM_HA_ID) == null) {
+ // If RM_HA_ID is not configured, use the first of RM_HA_IDS.
+ // Any valid RM HA ID should work.
+ String[] rmIds = yarnConf.getStrings(RM_HA_IDS);
+ Preconditions.checkState((rmIds != null) && (rmIds.length > 0),
+ "Not set " + RM_HA_IDS);
+ yarnConf.set(RM_HA_ID, rmIds[0]);
+ }
+
+ hostname = yarnConf.getSocketAddr(
+ RM_ADDRESS,
+ DEFAULT_RM_ADDRESS,
+ DEFAULT_RM_PORT).getHostName();
+ } else {
+ hostname = conf.getSocketAddr(
+ RM_ADDRESS,
+ DEFAULT_RM_ADDRESS,
+ DEFAULT_RM_PORT).getHostName();
+ }
+ return SecurityUtil.getServerPrincipal(principal, hostname);
+ }
+
+ /**
+ * Create and add any filesystem delegation tokens with
+ * the RM(s) configured to be able to renew them. Returns null
+ * on an insecure cluster (i.e. harmless)
+ * @param conf configuration
+ * @param fs filesystem
+ * @param credentials credentials to update
+ * @return a list of all added tokens.
+ * @throws IOException
+ */
+ public static Token<?>[] addRMRenewableFSDelegationTokens(Configuration conf,
+ FileSystem fs,
+ Credentials credentials) throws IOException {
+ Preconditions.checkArgument(conf != null);
+ Preconditions.checkArgument(credentials != null);
+ if (UserGroupInformation.isSecurityEnabled()) {
+ return fs.addDelegationTokens(CredentialUtils.getRMPrincipal(conf),
+ credentials);
+ }
+ return null;
+ }
+
+ /**
+ * Add an FS delegation token which can be renewed by the current user
+ * @param fs filesystem
+ * @param credentials credentials to update
+ * @throws IOException problems.
+ */
+ public static void addSelfRenewableFSDelegationTokens(
+ FileSystem fs,
+ Credentials credentials) throws IOException {
+ Preconditions.checkArgument(fs != null);
+ Preconditions.checkArgument(credentials != null);
+ fs.addDelegationTokens(
+ getSelfRenewer(),
+ credentials);
+ }
+
+ public static String getSelfRenewer() throws IOException {
+ return UserGroupInformation.getLoginUser().getShortUserName();
+ }
+
+ /**
+ * Create and add an RM delegation token to the credentials
+ * @param yarnClient Yarn Client
+ * @param credentials to add token to
+ * @return the token which was added
+ * @throws IOException
+ * @throws YarnException
+ */
+ public static Token<TokenIdentifier> addRMDelegationToken(YarnClient yarnClient,
+ Credentials credentials)
+ throws IOException, YarnException {
+ Configuration conf = yarnClient.getConfig();
+ Text rmPrincipal = new Text(CredentialUtils.getRMPrincipal(conf));
+ Text rmDTService = ClientRMProxy.getRMDelegationTokenService(conf);
+ Token<TokenIdentifier> rmDelegationToken =
+ ConverterUtils.convertFromYarn(
+ yarnClient.getRMDelegationToken(rmPrincipal),
+ rmDTService);
+ credentials.addToken(rmDelegationToken.getService(), rmDelegationToken);
+ return rmDelegationToken;
+ }
+
+ public static Token<TimelineDelegationTokenIdentifier> maybeAddTimelineToken(
+ Configuration conf,
+ Credentials credentials)
+ throws IOException, YarnException {
+ if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
+ LOG.debug("Timeline service enabled -fetching token");
+
+ try(TimelineClient timelineClient = TimelineClient.createTimelineClient()) {
+ timelineClient.init(conf);
+ timelineClient.start();
+ Token<TimelineDelegationTokenIdentifier> token =
+ timelineClient.getDelegationToken(
+ CredentialUtils.getRMPrincipal(conf));
+ credentials.addToken(token.getService(), token);
+ return token;
+ }
+ } else {
+ LOG.debug("Timeline service is disabled");
+ return null;
+ }
+ }
+
+ /**
+ * Filter a list of tokens from a set of credentials
+ * @param credentials credential source (a new credential set os re
+ * @param filter List of tokens to strip out
+ * @return a new, filtered, set of credentials
+ */
+ public static Credentials filterTokens(Credentials credentials,
+ List<Text> filter) {
+ Credentials result = new Credentials(credentials);
+ Iterator<Token<? extends TokenIdentifier>> iter =
+ result.getAllTokens().iterator();
+ while (iter.hasNext()) {
+ Token<? extends TokenIdentifier> token = iter.next();
+ LOG.debug("Token {}", token.getKind());
+ if (filter.contains(token.getKind())) {
+ LOG.debug("Filtering token {}", token.getKind());
+ iter.remove();
+ }
+ }
+ return result;
+ }
+
+ public static String dumpTokens(Credentials credentials, String separator) {
+ ArrayList<Token<? extends TokenIdentifier>> sorted =
+ new ArrayList<>(credentials.getAllTokens());
+ Collections.sort(sorted, new TokenComparator());
+ StringBuilder buffer = new StringBuilder(sorted.size()* 128);
+ for (Token<? extends TokenIdentifier> token : sorted) {
+ buffer.append(tokenToString(token)).append(separator);
+ }
+ return buffer.toString();
+ }
+
+ /**
+ * Create a string for people to look at
+ * @param token token to convert to a string form
+ * @return a printable view of the token
+ */
+ public static String tokenToString(Token<? extends TokenIdentifier> token) {
+ DateFormat df = DateFormat.getDateTimeInstance(
+ DateFormat.SHORT, DateFormat.SHORT);
+ StringBuilder buffer = new StringBuilder(128);
+ buffer.append(token.toString());
+ try {
+ TokenIdentifier ti = token.decodeIdentifier();
+ buffer.append("; ").append(ti);
+ if (ti instanceof AbstractDelegationTokenIdentifier) {
+ // details in human readable form, and compensate for information HDFS DT omits
+ AbstractDelegationTokenIdentifier dt = (AbstractDelegationTokenIdentifier) ti;
+ buffer.append("; Renewer: ").append(dt.getRenewer());
+ buffer.append("; Issued: ")
+ .append(df.format(new Date(dt.getIssueDate())));
+ buffer.append("; Max Date: ")
+ .append(df.format(new Date(dt.getMaxDate())));
+ }
+ } catch (IOException e) {
+ //marshall problem; not ours
+ LOG.debug("Failed to decode {}: {}", token, e, e);
+ }
+ return buffer.toString();
+ }
+
+ /**
+ * Get the expiry time of a token.
+ * @param token token to examine
+ * @return the time in milliseconds after which the token is invalid.
+ * @throws IOException
+ */
+ public static long getTokenExpiryTime(Token token) throws IOException {
+ TokenIdentifier identifier = token.decodeIdentifier();
+ Preconditions.checkState(identifier instanceof AbstractDelegationTokenIdentifier,
+ "Token %s of type: %s has an identifier which cannot be examined: %s",
+ token, token.getClass(), identifier);
+ AbstractDelegationTokenIdentifier id =
+ (AbstractDelegationTokenIdentifier) identifier;
+ return id.getMaxDate();
+ }
+
+ private static class TokenComparator
+ implements Comparator<Token<? extends TokenIdentifier>>, Serializable {
+ @Override
+ public int compare(Token<? extends TokenIdentifier> left,
+ Token<? extends TokenIdentifier> right) {
+ return left.getKind().toString().compareTo(right.getKind().toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
new file mode 100644
index 0000000..cbcb0d6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+
+import java.util.Map;
+
+/**
+ * Command line builder purely for the Java CLI.
+ * Some of the <code>define</code> methods are designed to work with Hadoop tool and
+ * Slider launcher applications.
+ */
+public class JavaCommandLineBuilder extends CommandLineBuilder {
+
+ public JavaCommandLineBuilder() {
+ add(getJavaBinary());
+ }
+
+ /**
+ * Get the java binary. This is called in the constructor so don't try and
+ * do anything other than return a constant.
+ * @return the path to the Java binary
+ */
+ protected String getJavaBinary() {
+ return ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java";
+ }
+
+ /**
+ * Set the size of the heap if a non-empty heap is passed in.
+ * @param heap empty string or something like "128M" ,"1G" etc. The value is
+ * trimmed.
+ */
+ public void setJVMHeap(String heap) {
+ if (SliderUtils.isSet(heap)) {
+ add("-Xmx" + heap.trim());
+ }
+ }
+
+ /**
+ * Turn Java assertions on
+ */
+ public void enableJavaAssertions() {
+ add("-ea");
+ add("-esa");
+ }
+
+ /**
+ * Add a system property definition -must be used before setting the main entry point
+ * @param property
+ * @param value
+ */
+ public void sysprop(String property, String value) {
+ Preconditions.checkArgument(property != null, "null property name");
+ Preconditions.checkArgument(value != null, "null value");
+ add("-D" + property + "=" + value);
+ }
+
+ public JavaCommandLineBuilder forceIPv4() {
+ sysprop("java.net.preferIPv4Stack", "true");
+ return this;
+ }
+
+ public JavaCommandLineBuilder headless() {
+ sysprop("java.awt.headless", "true");
+ return this;
+ }
+
+ public boolean addConfOption(Configuration conf, String key) {
+ return defineIfSet(key, conf.get(key));
+ }
+
+ /**
+ * Add a varargs list of configuration parameters —if they are present
+ * @param conf configuration source
+ * @param keys keys
+ */
+ public void addConfOptions(Configuration conf, String... keys) {
+ for (String key : keys) {
+ addConfOption(conf, key);
+ }
+ }
+
+ /**
+ * Add all configuration options which match the prefix
+ * @param conf configuration
+ * @param prefix prefix, e.g {@code "slider."}
+ * @return the number of entries copied
+ */
+ public int addPrefixedConfOptions(Configuration conf, String prefix) {
+ int copied = 0;
+ for (Map.Entry<String, String> entry : conf) {
+ if (entry.getKey().startsWith(prefix)) {
+ define(entry.getKey(), entry.getValue());
+ copied++;
+ }
+ }
+ return copied;
+ }
+
+ /**
+ * Ass a configuration option to the command line of the application
+ * @param conf configuration
+ * @param key key
+ * @param defVal default value
+ * @return the resolved configuration option
+ * @throws IllegalArgumentException if key is null or the looked up value
+ * is null (that is: the argument is missing and devVal was null.
+ */
+ public String addConfOptionToCLI(Configuration conf,
+ String key,
+ String defVal) {
+ Preconditions.checkArgument(key != null, "null key");
+ String val = conf.get(key, defVal);
+ define(key, val);
+ return val;
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI. This is very Hadoop API
+ * @param key key
+ * @param val value
+ * @throws IllegalArgumentException if either argument is null
+ */
+ public void define(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ Preconditions.checkArgument(val != null, "null value");
+ add("-D", key + "=" + val);
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
+ * is not null
+ * @param key key
+ * @param val value
+ */
+ public boolean defineIfSet(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ if (val != null) {
+ define(key, val);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Add a mandatory config option
+ * @param conf configuration
+ * @param key key
+ * @throws BadConfigException if the key is missing
+ */
+ public void addMandatoryConfOption(Configuration conf,
+ String key) throws BadConfigException {
+ if (!addConfOption(conf, key)) {
+ throw new BadConfigException("Missing configuration option: " + key);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
new file mode 100644
index 0000000..db9de7a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+
+/**
+ * The system is in a bad state
+ */
+public class BadClusterStateException extends SliderException {
+ public BadClusterStateException(String message,
+ Object... args) {
+ super(EXIT_BAD_STATE, message, args);
+ }
+
+ public BadClusterStateException(Throwable throwable,
+ String message, Object... args) {
+ super(EXIT_BAD_STATE, throwable, message, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
new file mode 100644
index 0000000..41e3251
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+public class BadCommandArgumentsException extends SliderException {
+ public BadCommandArgumentsException(String s, Object... args) {
+ super(EXIT_COMMAND_ARGUMENT_ERROR, s, args);
+ }
+
+ public BadCommandArgumentsException(Throwable throwable, String message,
+ Object... args) {
+ super(EXIT_COMMAND_ARGUMENT_ERROR, throwable, message, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
new file mode 100644
index 0000000..8199c3c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+/**
+ * An exception to raise on a bad configuration
+ */
+public class BadConfigException extends SliderException {
+
+ public BadConfigException(String s) {
+ super(EXIT_BAD_CONFIGURATION, s);
+ }
+
+ public BadConfigException(String message, Object... args) {
+ super(EXIT_BAD_CONFIGURATION, message, args);
+ }
+
+ public BadConfigException(
+ Throwable throwable,
+ String message, Object... args) {
+ super(EXIT_BAD_CONFIGURATION, throwable, message, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
new file mode 100644
index 0000000..83658c8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+public interface ErrorStrings {
+
+ String PRINTF_E_INSTANCE_ALREADY_EXISTS = "Service Instance \"%s\" already exists and is defined in %s";
+ String PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS = "Service Instance dir already exists: %s";
+
+ /**
+ * ERROR Strings
+ */
+ String ERROR_NO_ACTION = "No action specified";
+ String ERROR_UNKNOWN_ACTION = "Unknown command: ";
+ String ERROR_NOT_ENOUGH_ARGUMENTS =
+ "Not enough arguments for action: ";
+ String ERROR_PARSE_FAILURE =
+ "Failed to parse ";
+ /**
+ * All the remaining values after argument processing
+ */
+ String ERROR_TOO_MANY_ARGUMENTS =
+ "Too many arguments";
+ String ERROR_DUPLICATE_ENTRY = "Duplicate entry for ";
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
new file mode 100644
index 0000000..d66b860
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+/**
+ * Get the exit code of an exception. Making it an interface allows
+ * us to retrofit exit codes onto existing classes
+ */
+public interface ExitCodeProvider {
+
+ /**
+ * Method to get the exit code
+ * @return the exit code
+ */
+ int getExitCode();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
new file mode 100644
index 0000000..483fb48
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+/*
+ * Common Exit codes
+ * <p>
+ * Exit codes from 64 up are service specific.
+ * <p>
+ * Many of the exit codes are designed to resemble HTTP error codes,
+ * squashed into a single byte. e.g 44 , "not found" is the equivalent
+ * of 404
+ * <pre>
+ * 0-10: general command issues
+ * 30-39: equivalent to the 3XX responses, where those responses are
+ * considered errors by the service.
+ * 40-49: request-related errors
+ * 50-59: server-side problems. These may be triggered by the request.
+ * 64- : service specific error codes
+ * </pre>
+ */
+public interface LauncherExitCodes {
+
+ /**
+ * 0: success
+ */
+ int EXIT_SUCCESS = 0;
+
+ /**
+ * -1: generic "false" response. The operation worked but
+ * the result was not true
+ */
+ int EXIT_FALSE = -1;
+
+ /**
+ * Exit code when a client requested service termination: {@value}
+ */
+ int EXIT_CLIENT_INITIATED_SHUTDOWN = 1;
+
+ /**
+ * Exit code when targets could not be launched: {@value}
+ */
+ int EXIT_TASK_LAUNCH_FAILURE = 2;
+
+ /**
+ * Exit code when a control-C, kill -3, signal was picked up: {@value}
+ */
+ int EXIT_INTERRUPTED = 3;
+
+ /**
+ * Exit code when a usage message was printed: {@value}
+ */
+ int EXIT_USAGE = 4;
+
+ /**
+ * Exit code when something happened but we can't be specific: {@value}
+ */
+ int EXIT_OTHER_FAILURE = 5;
+
+ /**
+ * Exit code on connectivity problems: {@value}
+ */
+ int EXIT_MOVED = 31;
+
+ /**
+ * found: {@value}.
+ * <p>
+ * This is low value as in HTTP it is normally a success/redirect;
+ * whereas on the command line 0 is the sole success code.
+ * <p>
+ * <code>302 Found</code>
+ */
+ int EXIT_FOUND = 32;
+
+ /**
+ * Exit code on a request where the destination has not changed
+ * and (somehow) the command specified that this is an error.
+ * That is, this exit code is somehow different from a "success"
+ * : {@value}
+ * <p>
+ * <code>304 Not Modified </code>
+ */
+ int EXIT_NOT_MODIFIED = 34;
+
+ /**
+ * Exit code when the command line doesn't parse: {@value}, or
+ * when it is otherwise invalid.
+ * <p>
+ * <code>400 BAD REQUEST</code>
+ */
+ int EXIT_COMMAND_ARGUMENT_ERROR = 40;
+
+ /**
+ * The request requires user authentication: {@value}
+ * <p>
+ * <code>401 Unauthorized</code>
+ */
+ int EXIT_UNAUTHORIZED = 41;
+
+ /**
+ * Forbidden action: {@value}
+ * <p>
+ * <code>403: Forbidden</code>
+ */
+ int EXIT_FORBIDDEN = 43;
+
+ /**
+ * Something was not found: {@value}
+ * <p>
+ * <code>404: NOT FOUND</code>
+ */
+ int EXIT_NOT_FOUND = 44;
+
+ /**
+ * The operation is not allowed: {@value}
+ * <p>
+ * <code>405: NOT ALLOWED</code>
+ */
+ int EXIT_OPERATION_NOT_ALLOWED = 45;
+
+ /**
+ * The command is somehow not acceptable: {@value}
+ * <p>
+ * <code>406: NOT ACCEPTABLE</code>
+ */
+ int EXIT_NOT_ACCEPTABLE = 46;
+
+ /**
+ * Exit code on connectivity problems: {@value}
+ * <p>
+ * <code>408: Request Timeout</code>
+ */
+ int EXIT_CONNECTIVITY_PROBLEM = 48;
+
+ /**
+ * The request could not be completed due to a conflict with the current
+ * state of the resource. {@value}
+ * <p>
+ * <code>409: conflict</code>
+ */
+ int EXIT_CONFLICT = 49;
+
+ /**
+ * internal error: {@value}
+ * <p>
+ * <code>500 Internal Server Error</code>
+ */
+ int EXIT_INTERNAL_ERROR = 50;
+
+ /**
+ * Unimplemented feature: {@value}
+ * <p>
+ * <code>501: Not Implemented</code>
+ */
+ int EXIT_UNIMPLEMENTED = 51;
+
+ /**
+ * Service Unavailable; it may be available later: {@value}
+ * <p>
+ * <code>503 Service Unavailable</code>
+ */
+ int EXIT_SERVICE_UNAVAILABLE = 53;
+
+ /**
+ * The service does not support, or refuses to support this version: {@value}.
+ * If raised, this is expected to be raised server-side and likely due
+ * to client/server version incompatibilities.
+ * <p>
+ * <code> 505: Version Not Supported</code>
+ */
+ int EXIT_UNSUPPORTED_VERSION = 55;
+
+ /**
+ * Exit code when an exception was thrown from the service: {@value}
+ * <p>
+ * <code>5XX</code>
+ */
+ int EXIT_EXCEPTION_THROWN = 56;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
new file mode 100644
index 0000000..ef22b57
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+public interface RestApiErrorMessages {
+ String ERROR_APPLICATION_NAME_INVALID =
+ "Service name is either empty or not provided";
+ String ERROR_APPLICATION_NAME_INVALID_FORMAT =
+ "Service name %s is not valid - only lower case letters, digits, " +
+ "and hyphen are allowed, and the name must be no more " +
+ "than 63 characters";
+ String ERROR_COMPONENT_NAME_INVALID =
+ "Component name must be no more than %s characters: %s";
+ String ERROR_USER_NAME_INVALID =
+ "User name must be no more than 63 characters";
+
+ String ERROR_APPLICATION_NOT_RUNNING = "Service not running";
+ String ERROR_APPLICATION_DOES_NOT_EXIST = "Service not found";
+ String ERROR_APPLICATION_IN_USE = "Service already exists in started"
+ + " state";
+ String ERROR_APPLICATION_INSTANCE_EXISTS = "Service already exists in"
+ + " stopped/failed state (either restart with PUT or destroy with DELETE"
+ + " before creating a new one)";
+
+ String ERROR_SUFFIX_FOR_COMPONENT =
+ " for component %s (nor at the global level)";
+ String ERROR_ARTIFACT_INVALID = "Artifact is not provided";
+ String ERROR_ARTIFACT_FOR_COMP_INVALID =
+ ERROR_ARTIFACT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_ARTIFACT_ID_INVALID =
+ "Artifact id (like docker image name) is either empty or not provided";
+ String ERROR_ARTIFACT_ID_FOR_COMP_INVALID =
+ ERROR_ARTIFACT_ID_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+
+ String ERROR_RESOURCE_INVALID = "Resource is not provided";
+ String ERROR_RESOURCE_FOR_COMP_INVALID =
+ ERROR_RESOURCE_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_MEMORY_INVALID =
+ "Service resource or memory not provided";
+ String ERROR_RESOURCE_CPUS_INVALID =
+ "Service resource or cpus not provided";
+ String ERROR_RESOURCE_CPUS_INVALID_RANGE =
+ "Unacceptable no of cpus specified, either zero or negative";
+ String ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID =
+ ERROR_RESOURCE_MEMORY_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID =
+ ERROR_RESOURCE_CPUS_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE =
+ ERROR_RESOURCE_CPUS_INVALID_RANGE
+ + " for component %s (or at the global level)";
+ String ERROR_CONTAINERS_COUNT_INVALID =
+ "Invalid no of containers specified";
+ String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID =
+ ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_DEPENDENCY_INVALID = "Dependency %s for component %s is " +
+ "invalid, does not exist as a component";
+ String ERROR_DEPENDENCY_CYCLE = "Invalid dependencies, a cycle may " +
+ "exist: %s";
+
+ String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED =
+ "Cannot specify" + " cpus/memory along with profile";
+ String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED =
+ ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED
+ + " for component %s";
+ String ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET =
+ "Resource profile is not " + "supported yet. Please specify cpus/memory.";
+
+ String ERROR_NULL_ARTIFACT_ID =
+ "Artifact Id can not be null if artifact type is none";
+ String ERROR_ABSENT_NUM_OF_INSTANCE =
+ "Num of instances should appear either globally or per component";
+ String ERROR_ABSENT_LAUNCH_COMMAND =
+ "Launch_command is required when type is not DOCKER";
+
+ String ERROR_QUICKLINKS_FOR_COMP_INVALID = "Quicklinks specified at"
+ + " component level, needs corresponding values set at service level";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
new file mode 100644
index 0000000..e83ccbe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * A service launch exception that includes an exit code;
+ * when caught by the ServiceLauncher, it will convert that
+ * into a process exit code.
+ */
+public class ServiceLaunchException extends YarnException
+ implements ExitCodeProvider, LauncherExitCodes {
+
+ private final int exitCode;
+
+ /**
+ * Create an exception with the specific exit code
+ * @param exitCode exit code
+ * @param cause cause of the exception
+ */
+ public ServiceLaunchException(int exitCode, Throwable cause) {
+ super(cause);
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * Create an exception with the specific exit code and text
+ * @param exitCode exit code
+ * @param message message to use in exception
+ */
+ public ServiceLaunchException(int exitCode, String message) {
+ super(message);
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * Create an exception with the specific exit code, text and cause
+ * @param exitCode exit code
+ * @param message message to use in exception
+ * @param cause cause of the exception
+ */
+ public ServiceLaunchException(int exitCode, String message, Throwable cause) {
+ super(message, cause);
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * Get the exit code
+ * @return the exit code
+ */
+ @Override
+ public int getExitCode() {
+ return exitCode;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
new file mode 100644
index 0000000..5b74b80
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
+
+public class SliderException extends ServiceLaunchException implements
+ SliderExitCodes {
+ public SliderException() {
+ super(EXIT_EXCEPTION_THROWN, "SliderException");
+ }
+
+ public SliderException(int code, String message) {
+ super(code, message);
+ }
+
+ public SliderException(String s) {
+ super(EXIT_EXCEPTION_THROWN, s);
+ }
+
+ public SliderException(String s, Throwable throwable) {
+ super(EXIT_EXCEPTION_THROWN, s, throwable);
+ }
+
+ /**
+ * Format the exception as you create it
+ * @param code exit code
+ * @param message exception message -sprintf formatted
+ * @param args arguments for the formatting
+ */
+ public SliderException(int code, String message, Object... args) {
+ super(code, String.format(message, args));
+ }
+
+ /**
+ * Format the exception, include a throwable.
+ * The throwable comes before the message so that it is out of the varargs
+ * @param code exit code
+ * @param throwable thrown
+ * @param message message
+ * @param args arguments
+ */
+ public SliderException(int code,
+ Throwable throwable,
+ String message,
+ Object... args) {
+ super(code, String.format(message, args), throwable);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
new file mode 100644
index 0000000..3a9fa25
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+/**
+ * Used to raise a usage exception ... this has the exit code
+ * {@link #EXIT_USAGE}
+ */
+public class UsageException extends SliderException {
+ public UsageException(String s, Object... args) {
+ super(EXIT_USAGE, s, args);
+ }
+
+ public UsageException(Throwable throwable, String message,
+ Object... args) {
+ super(EXIT_USAGE, throwable, message, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
new file mode 100644
index 0000000..33e33a6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.impl.pb.client;
+
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.service.impl.pb.service.ClientAMProtocolPB;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+
+public class ClientAMProtocolPBClientImpl
+ implements ClientAMProtocol, Closeable {
+
+ private ClientAMProtocolPB proxy;
+
+ public ClientAMProtocolPBClientImpl(long clientVersion,
+ InetSocketAddress addr, Configuration conf) throws IOException {
+ RPC.setProtocolEngine(conf, ClientAMProtocolPB.class,
+ ProtobufRpcEngine.class);
+ proxy = RPC.getProxy(ClientAMProtocolPB.class, clientVersion, addr, conf);
+
+ }
+
+ @Override public FlexComponentsResponseProto flexComponents(
+ FlexComponentsRequestProto request) throws IOException, YarnException {
+ try {
+ return proxy.flexComponents(null, request);
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ }
+ return null;
+ }
+
+ @Override
+ public GetStatusResponseProto getStatus(GetStatusRequestProto request)
+ throws IOException, YarnException {
+ try {
+ return proxy.getStatus(null, request);
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ }
+ return null;
+ }
+
+ @Override
+ public StopResponseProto stop(StopRequestProto requestProto)
+ throws IOException, YarnException {
+ try {
+ return proxy.stop(null, requestProto);
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ }
+ return null;
+ }
+
+ @Override public void close() {
+ if (this.proxy != null) {
+ RPC.stopProxy(this.proxy);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
new file mode 100644
index 0000000..6a9cd37
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.impl.pb.service;
+
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol;
+
+@ProtocolInfo(
+ protocolName = "org.apache.hadoop.yarn.service.ClientAMProtocol",
+ protocolVersion = 1)
+public interface ClientAMProtocolPB extends
+ ClientAMProtocol.ClientAMProtocolService.BlockingInterface {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
new file mode 100644
index 0000000..7100781
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.impl.pb.service;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+
+import java.io.IOException;
+
+public class ClientAMProtocolPBServiceImpl implements ClientAMProtocolPB {
+
+ private ClientAMProtocol real;
+
+ public ClientAMProtocolPBServiceImpl(ClientAMProtocol impl) {
+ this.real = impl;
+ }
+
+ @Override
+ public FlexComponentsResponseProto flexComponents(RpcController controller,
+ FlexComponentsRequestProto request) throws ServiceException {
+ try {
+ return real.flexComponents(request);
+ } catch (IOException | YarnException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override public GetStatusResponseProto getStatus(RpcController controller,
+ GetStatusRequestProto request) throws ServiceException {
+ try {
+ return real.getStatus(request);
+ } catch (IOException | YarnException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto stop(
+ RpcController controller,
+ org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto request)
+ throws ServiceException {
+ try {
+ return real.stop(request);
+ } catch (IOException | YarnException e) {
+ throw new ServiceException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/ServiceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/ServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/ServiceMonitor.java
new file mode 100644
index 0000000..982448a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/ServiceMonitor.java
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.component.ComponentEvent;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import org.apache.hadoop.yarn.service.component.ComponentState;
+import org.apache.hadoop.yarn.service.monitor.probe.ProbeStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceState.STARTED;
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX;
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.BECOME_NOT_READY;
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.BECOME_READY;
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceState.READY;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_WINDOW;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.DEFAULT_READINESS_CHECK_INTERVAL;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.READINESS_CHECK_INTERVAL;
+
+public class ServiceMonitor extends AbstractService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceMonitor.class);
+
+ public ScheduledExecutorService executorService;
+ private Map<ContainerId, ComponentInstance> liveInstances = null;
+ private ServiceContext context;
+ private Configuration conf;
+
+ public ServiceMonitor(String name, ServiceContext context) {
+ super(name);
+ liveInstances = context.scheduler.getLiveInstances();
+ this.context = context;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ executorService = Executors.newScheduledThreadPool(1);
+ this.conf = conf;
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ long readinessCheckInterval = YarnServiceConf
+ .getLong(READINESS_CHECK_INTERVAL, DEFAULT_READINESS_CHECK_INTERVAL,
+ context.service.getConfiguration(), conf);
+
+ executorService
+ .scheduleAtFixedRate(new ReadinessChecker(), readinessCheckInterval,
+ readinessCheckInterval, TimeUnit.SECONDS);
+
+ // Default 6 hours.
+ long failureResetInterval = YarnServiceConf
+ .getLong(CONTAINER_FAILURE_WINDOW, 21600,
+ context.service.getConfiguration(), conf);
+
+ executorService
+ .scheduleAtFixedRate(new ContainerFailureReset(), failureResetInterval,
+ failureResetInterval, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void serviceStop() throws Exception {
+ if (executorService != null) {
+ executorService.shutdownNow();
+ }
+ }
+
+ private class ReadinessChecker implements Runnable {
+
+ @Override
+ public void run() {
+
+ // check if the comp instance are ready
+ for (Map.Entry<ContainerId, ComponentInstance> entry : liveInstances
+ .entrySet()) {
+ ComponentInstance instance = entry.getValue();
+
+ ProbeStatus status = instance.ping();
+ if (status.isSuccess()) {
+ if (instance.getState() == STARTED) {
+ // synchronously update the state.
+ instance.handle(
+ new ComponentInstanceEvent(entry.getKey(), BECOME_READY));
+ }
+ } else {
+ if (instance.getState() == READY) {
+ instance.handle(
+ new ComponentInstanceEvent(entry.getKey(), BECOME_NOT_READY));
+ }
+ }
+ }
+
+ for (Component component : context.scheduler.getAllComponents()
+ .values()) {
+ // If comp hasn't started yet and its dependencies are satisfied
+ if (component.getState() == ComponentState.INIT && component
+ .areDependenciesReady()) {
+ LOG.info("[COMPONENT {}]: Dependencies satisfied, ramping up.",
+ component.getName());
+ ComponentEvent event = new ComponentEvent(component.getName(), FLEX)
+ .setDesired(component.getComponentSpec().getNumberOfContainers());
+ component.handle(event);
+ }
+ }
+ }
+ }
+
+ private class ContainerFailureReset implements Runnable {
+ @Override
+ public void run() {
+ for (Component component : context.scheduler.getAllComponents().values()) {
+ component.resetCompFailureCount();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
new file mode 100644
index 0000000..1923086
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Map;
+
+public class HttpProbe extends Probe {
+ protected static final Logger log = LoggerFactory.getLogger(HttpProbe.class);
+
+ private static final String HOST_TOKEN = "${THIS_HOST}";
+
+ private final String urlString;
+ private final int timeout;
+ private final int min, max;
+
+
+ public HttpProbe(String url, int timeout, int min, int max, Configuration
+ conf) {
+ super("Http probe of " + url + " [" + min + "-" + max + "]", conf);
+ this.urlString = url;
+ this.timeout = timeout;
+ this.min = min;
+ this.max = max;
+ }
+
+ public static HttpProbe create(Map<String, String> props)
+ throws IOException {
+ String urlString = getProperty(props, WEB_PROBE_URL, null);
+ new URL(urlString);
+ int timeout = getPropertyInt(props, WEB_PROBE_CONNECT_TIMEOUT,
+ WEB_PROBE_CONNECT_TIMEOUT_DEFAULT);
+ int minSuccess = getPropertyInt(props, WEB_PROBE_MIN_SUCCESS,
+ WEB_PROBE_MIN_SUCCESS_DEFAULT);
+ int maxSuccess = getPropertyInt(props, WEB_PROBE_MAX_SUCCESS,
+ WEB_PROBE_MAX_SUCCESS_DEFAULT);
+ return new HttpProbe(urlString, timeout, minSuccess, maxSuccess, null);
+ }
+
+
+ private static HttpURLConnection getConnection(URL url, int timeout) throws
+ IOException {
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setInstanceFollowRedirects(true);
+ connection.setConnectTimeout(timeout);
+ return connection;
+ }
+
+ @Override
+ public ProbeStatus ping(ComponentInstance instance) {
+ ProbeStatus status = new ProbeStatus();
+ ContainerStatus containerStatus = instance.getContainerStatus();
+ if (containerStatus == null || SliderUtils.isEmpty(containerStatus.getIPs())
+ || StringUtils.isEmpty(containerStatus.getHost())) {
+ status.fail(this, new IOException("IP is not available yet"));
+ return status;
+ }
+
+ String ip = containerStatus.getIPs().get(0);
+ HttpURLConnection connection = null;
+ try {
+ URL url = new URL(urlString.replace(HOST_TOKEN, ip));
+ connection = getConnection(url, this.timeout);
+ int rc = connection.getResponseCode();
+ if (rc < min || rc > max) {
+ String error = "Probe " + url + " error code: " + rc;
+ log.info(error);
+ status.fail(this,
+ new IOException(error));
+ } else {
+ status.succeed(this);
+ }
+ } catch (Throwable e) {
+ String error = "Probe " + urlString + " failed for IP " + ip + ": " + e;
+ log.info(error, e);
+ status.fail(this,
+ new IOException(error, e));
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
+ }
+ }
+ return status;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/LogEntryBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/LogEntryBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/LogEntryBuilder.java
new file mode 100644
index 0000000..9ad86fe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/LogEntryBuilder.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+/**
+ * Build up log entries for ease of splunk
+ */
+public class LogEntryBuilder {
+
+ private final StringBuilder builder = new StringBuilder();
+
+ public LogEntryBuilder() {
+ }
+
+ public LogEntryBuilder(String text) {
+ elt(text);
+ }
+
+
+ public LogEntryBuilder(String name, Object value) {
+ entry(name, value);
+ }
+
+ public LogEntryBuilder elt(String text) {
+ addComma();
+ builder.append(text);
+ return this;
+ }
+
+ public LogEntryBuilder elt(String name, Object value) {
+ addComma();
+ entry(name, value);
+ return this;
+ }
+
+ private void addComma() {
+ if (!isEmpty()) {
+ builder.append(", ");
+ }
+ }
+
+ private void entry(String name, Object value) {
+ builder.append(name).append('=');
+ if (value != null) {
+ builder.append('"').append(value.toString()).append('"');
+ } else {
+ builder.append("null");
+ }
+ }
+
+ @Override
+ public String toString() {
+ return builder.toString();
+ }
+
+ private boolean isEmpty() {
+ return builder.length() == 0;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorKeys.java
new file mode 100644
index 0000000..55b55f6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorKeys.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+/**
+ * Config keys for monitoring
+ */
+public interface MonitorKeys {
+
+ /**
+ * Port probing key : port to attempt to create a TCP connection to {@value}.
+ */
+ String PORT_PROBE_PORT = "port";
+ /**
+ * Port probing key : timeout for the the connection attempt {@value}.
+ */
+ String PORT_PROBE_CONNECT_TIMEOUT = "timeout";
+ /**
+ * Port probing default : timeout for the connection attempt {@value}.
+ */
+ int PORT_PROBE_CONNECT_TIMEOUT_DEFAULT = 1000;
+
+ /**
+ * Web probing key : URL {@value}.
+ */
+ String WEB_PROBE_URL = "url";
+ /**
+ * Web probing key : min success code {@value}.
+ */
+ String WEB_PROBE_MIN_SUCCESS = "min.success";
+ /**
+ * Web probing key : max success code {@value}.
+ */
+ String WEB_PROBE_MAX_SUCCESS = "max.success";
+ /**
+ * Web probing default : min successful response code {@value}.
+ */
+ int WEB_PROBE_MIN_SUCCESS_DEFAULT = 200;
+ /**
+ * Web probing default : max successful response code {@value}.
+ */
+ int WEB_PROBE_MAX_SUCCESS_DEFAULT = 299;
+ /**
+ * Web probing key : timeout for the connection attempt {@value}
+ */
+ String WEB_PROBE_CONNECT_TIMEOUT = "timeout";
+ /**
+ * Port probing default : timeout for the connection attempt {@value}.
+ */
+ int WEB_PROBE_CONNECT_TIMEOUT_DEFAULT = 1000;
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
deleted file mode 100644
index 3e9b764..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.publisher;
-
-import org.apache.hadoop.yarn.webapp.NotFoundException;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.slider.core.registry.docstore.PublishedExports;
-import org.apache.slider.core.registry.docstore.PublishedExportsSet;
-import org.apache.slider.core.registry.docstore.UriMap;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.PUBLISHED_CONFIGURATION_REGEXP;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.PUBLISHED_CONFIGURATION_SET_REGEXP;
-
-/**
- * This publishes configuration sets
- */
-public class PublisherResource extends AbstractSliderResource {
- protected static final Logger log =
- LoggerFactory.getLogger(PublisherResource.class);
- public static final String EXPORTS_NAME = "exports";
- public static final String EXPORTS_RESOURCES_PATH = "/" + EXPORTS_NAME;
- public static final String EXPORT_RESOURCE_PATH = EXPORTS_RESOURCES_PATH + "/{exportname}" ;
- public static final String SET_NAME =
- "{setname: " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}";
- public static final String SETNAME = "setname";
- public static final String CLASSPATH = "/classpath";
- public static final String CONFIG = "config";
-
- public static final String SETNAME_PATTERN =
- "{"+ SETNAME+": " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}";
- private static final String CONFIG_PATTERN =
- SETNAME_PATTERN + "/{"+ CONFIG +": " + PUBLISHED_CONFIGURATION_REGEXP + "}";
- private final StateAccessForProviders appState;
-
- public PublisherResource(WebAppApi slider) {
- super(slider);
- appState = slider.getAppState();
- }
-
- private void init(HttpServletResponse res, UriInfo uriInfo) {
- res.setContentType(null);
- log.debug(uriInfo.getRequestUri().toString());
- }
-
- /**
- * Get a named config set
- * @param setname name of the config set
- * @return the config set
- * @throws NotFoundException if there was no matching set
- */
- private PublishedConfigSet getConfigSet(String setname) {
- PublishedConfigSet configSet =
- appState.getPublishedConfigSet(setname);
- if (configSet == null) {
- throw new NotFoundException("Not found: " + setname);
- }
- return configSet;
- }
-
- @GET
- @Path("/")
- @Produces({MediaType.APPLICATION_JSON})
- public UriMap enumConfigSets(
- @Context UriInfo uriInfo,
- @Context HttpServletResponse res) {
- init(res, uriInfo);
- String baseURL = uriInfo.getRequestUri().toString();
- if (!baseURL.endsWith("/")) {
- baseURL += "/";
- }
- UriMap uriMap = new UriMap();
- for (String name : appState.listConfigSets()) {
- uriMap.put(name, baseURL + name);
- log.info("registering config set {} at {}", name, baseURL);
- }
- uriMap.put(EXPORTS_NAME, baseURL + EXPORTS_NAME);
- return uriMap;
- }
-
- @GET
- @Path(CLASSPATH)
- @Produces({MediaType.APPLICATION_JSON})
- public List<URL> getAMClassPath() {
- URL[] urls = ((URLClassLoader) getClass().getClassLoader()).getURLs();
- return Arrays.asList(urls);
- }
-
- @GET
- @Path(EXPORTS_RESOURCES_PATH)
- @Produces({MediaType.APPLICATION_JSON})
- public PublishedExportsSet gePublishedExports() {
-
- return appState.getPublishedExportsSet();
- }
-
- @GET
- @Path(EXPORT_RESOURCE_PATH)
- @Produces({MediaType.APPLICATION_JSON})
- public PublishedExports getAMExports2(@PathParam("exportname") String exportname,
- @Context UriInfo uriInfo,
- @Context HttpServletResponse res) {
- init(res, uriInfo);
- PublishedExportsSet set = appState.getPublishedExportsSet();
- return set.get(exportname);
- }
-
- @GET
- @Path("/"+ SETNAME_PATTERN)
- @Produces({MediaType.APPLICATION_JSON})
- public PublishedConfigSet getPublishedConfiguration(
- @PathParam(SETNAME) String setname,
- @Context UriInfo uriInfo,
- @Context HttpServletResponse res) {
- init(res, uriInfo);
-
- logRequest(uriInfo);
- PublishedConfigSet publishedConfigSet = getConfigSet(setname);
- log.debug("Number of configurations: {}", publishedConfigSet.size());
- return publishedConfigSet.shallowCopy();
- }
-
- private void logRequest(UriInfo uriInfo) {
- log.info(uriInfo.getRequestUri().toString());
- }
-
- @GET
- @Path("/" + CONFIG_PATTERN)
- @Produces({MediaType.APPLICATION_JSON})
- public PublishedConfiguration getConfigurationInstance(
- @PathParam(SETNAME) String setname,
- @PathParam(CONFIG) String config,
- @Context UriInfo uriInfo,
- @Context HttpServletResponse res) {
- init(res, uriInfo);
-
- PublishedConfiguration publishedConfig =
- getPublishedConfiguration(setname, config);
- if (publishedConfig == null) {
- log.info("Configuration {} not found", config);
- throw new NotFoundException("Not found: " + uriInfo.getAbsolutePath());
- }
- return publishedConfig;
- }
-
- /**
- * Get a configuration
- * @param setname name of the config set
- * @param config config
- * @return null if there was a config, but not a set
- * @throws NotFoundException if there was no matching set
- */
- public PublishedConfiguration getPublishedConfiguration(String setname,
- String config) {
- return getConfigSet(setname).get(config);
- }
-
- @GET
- @Path("/" + CONFIG_PATTERN + ".json")
- @Produces({MediaType.APPLICATION_JSON})
- public String getConfigurationContentJson(
- @PathParam(SETNAME) String setname,
-
- @PathParam(CONFIG) String config,
- @Context UriInfo uriInfo,
- @Context HttpServletResponse res) throws IOException {
- return getStringRepresentation(setname, config, uriInfo, res,
- ConfigFormat.JSON);
- }
-
- @GET
- @Path("/" + CONFIG_PATTERN + ".xml")
- @Produces({MediaType.APPLICATION_XML})
- public String getConfigurationContentXML(
- @PathParam(SETNAME) String setname,
- @PathParam(CONFIG) String config,
- @Context UriInfo uriInfo,
- @Context HttpServletResponse res) throws IOException {
- return getStringRepresentation(setname, config, uriInfo, res,
- ConfigFormat.XML);
- }
-
- @GET
- @Path("/" + CONFIG_PATTERN + ".properties")
- @Produces({MediaType.APPLICATION_XML})
- public String getConfigurationContentProperties(
- @PathParam(SETNAME) String setname,
-
- @PathParam(CONFIG) String config,
- @Context UriInfo uriInfo,
- @Context HttpServletResponse res) throws IOException {
-
- return getStringRepresentation(setname, config, uriInfo, res,
- ConfigFormat.PROPERTIES);
- }
-
- public String getStringRepresentation(String setname,
- String config,
- UriInfo uriInfo,
- HttpServletResponse res, ConfigFormat format) throws IOException {
- // delegate (including init)
- PublishedConfiguration publishedConfig =
- getConfigurationInstance(setname, config, uriInfo, res);
- PublishedConfigurationOutputter outputter =
- publishedConfig.createOutputter(format);
- return outputter.asString();
- }
-
- @GET
- @Path("/" + CONFIG_PATTERN +"/{propertyName}")
- @Produces({MediaType.APPLICATION_JSON})
- public Map<String,String> getConfigurationProperty(
- @PathParam(SETNAME) String setname,
- @PathParam(CONFIG) String config,
- @PathParam("propertyName") String propertyName,
- @Context UriInfo uriInfo,
- @Context HttpServletResponse res) {
- PublishedConfiguration publishedConfig =
- getConfigurationInstance(setname, config, uriInfo, res);
- String propVal = publishedConfig.entries.get(propertyName);
- if (propVal == null) {
- log.debug("Configuration property {} not found in configuration {}",
- propertyName, config);
- throw new NotFoundException("Property not found: " + propertyName);
- }
- Map<String, String> rtnVal = new HashMap<>();
- rtnVal.put(propertyName, propVal);
-
- return rtnVal;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
deleted file mode 100644
index efb09a8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.registry;
-
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.List;
-
-/**
- * Representation of a path entry
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class PathEntryResource {
-
- /**
- * Child nodes: as the short path to each element
- */
- public List<String> nodes;
-
- /**
- * Service record: if null —there is no resolvable service
- * record at this node.
- */
- public ServiceRecord service;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
deleted file mode 100644
index c824848..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.registry;
-
-import com.google.inject.Singleton;
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.exceptions.AuthenticationFailedException;
-import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
-import org.apache.hadoop.registry.client.exceptions.NoPathPermissionsException;
-import org.apache.hadoop.registry.client.exceptions.NoRecordException;
-import org.apache.hadoop.yarn.webapp.ForbiddenException;
-import org.apache.hadoop.yarn.webapp.NotFoundException;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-
-/**
- * This is the read-only view of the YARN registry.
- *
- * Model:
- * <ol>
- * <li>a tree of nodes</li>
- * <li>Default view is of children + record</li>
- * </ol>
- *
- */
-@Singleton
-public class RegistryResource extends AbstractSliderResource {
- protected static final Logger log =
- LoggerFactory.getLogger(RegistryResource.class);
- public static final String SERVICE_PATH =
- "/{path:.*}";
-
- private final RegistryOperations registry;
-
- /**
- * Construct an instance bonded to a registry
- * @param slider slider API
- */
- public RegistryResource(WebAppApi slider) {
- super(slider);
- this.registry = slider.getRegistryOperations();
- }
-
-
- /**
- * Internal init code, per request
- * @param request incoming request
- * @param uriInfo URI details
- */
- private void init(HttpServletRequest request, UriInfo uriInfo) {
- log.debug(uriInfo.getRequestUri().toString());
- }
-
- @GET
- @Produces({MediaType.APPLICATION_JSON})
- public PathEntryResource getRoot(@Context HttpServletRequest request,
- @Context UriInfo uriInfo) {
- return lookup("/", request, uriInfo);
- }
-
-// {path:.*}
-
- @Path(SERVICE_PATH)
- @GET
- @Produces({MediaType.APPLICATION_JSON})
- public PathEntryResource lookup(
- @PathParam("path") String path,
- @Context HttpServletRequest request,
- @Context UriInfo uriInfo) {
- init(request, uriInfo);
- return resolvePath(path);
- }
-
- /**
- * Do the actual processing of requests to responses; can be directly
- * invoked for testing.
- * @param path path to query
- * @return the entry
- * @throws WebApplicationException on any failure.
- */
- public PathEntryResource resolvePath(String path) throws
- WebApplicationException {
- try {
- PathEntryResource pathEntry =
- fromRegistry(path);
- if (log.isDebugEnabled()) {
- log.debug("Resolved:\n{}", pathEntry);
- }
- return pathEntry;
-
- } catch (Exception e) {
- throw buildException(path, e);
- }
- }
-
-
- /**
- * Build from the registry, filling up the children and service records.
- * If there is no service record at the end of the path, that entry is
- * null
- * @param path path to query
- * @return the built up record
- * @throws IOException problems
- *
- */
- private PathEntryResource fromRegistry(String path) throws IOException {
- PathEntryResource entry = new PathEntryResource();
- try {
- entry.service = registry.resolve(path);
- } catch (NoRecordException e) {
- // ignoring
- log.debug("No record at {}", path);
- } catch (InvalidRecordException e) {
- // swallowing this exception, the sign of "no entry present"
- // "nothing parseable"
- log.warn("Failed to resolve {}: {}", path, e, e);
- }
- entry.nodes = registry.list(path);
- return entry;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java
deleted file mode 100644
index 79b687f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.view;
-
-import com.google.inject.Inject;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-
-/**
- *
- */
-public class ClusterSpecificationBlock extends SliderHamletBlock {
-
- @Inject
- public ClusterSpecificationBlock(WebAppApi slider) {
- super(slider);
- }
-
- @Override
- protected void render(Block html) {
- doRender(html);
- }
-
- // An extra method to make testing easier since you can't make an instance of Block
- protected void doRender(Hamlet html) {
- html.
- div("cluster_json").
- h2("JSON Cluster Specification").
- pre().
- _(getJson())._()._();
- }
-
- /**
- * Get the JSON, catching any exceptions and returning error text instead
- * @return
- */
- private String getJson() {
- return appState.getApplication().toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
deleted file mode 100644
index 4796d6c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.view;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
-import com.google.inject.Inject;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.types.ComponentInformation;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-
-import javax.annotation.Nonnull;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-/**
- *
- */
-public class ContainerStatsBlock extends SliderHamletBlock {
-
- private static final String EVEN = "even", ODD = "odd", BOLD = "bold", SCHEME = "http://", PATH = "/node/container/";
-
- // Some functions that help transform the data into an object we can use to abstract presentation specifics
- protected static final Function<Entry<String,Integer>,Entry<TableContent,Integer>> stringIntPairFunc = toTableContentFunction();
- protected static final Function<Entry<String,Long>,Entry<TableContent,Long>> stringLongPairFunc = toTableContentFunction();
- protected static final Function<Entry<String,String>,Entry<TableContent,String>> stringStringPairFunc = toTableContentFunction();
-
- @Inject
- public ContainerStatsBlock(WebAppApi slider) {
- super(slider);
- }
-
- /**
- * Sort a collection of ClusterNodes by name
- */
- protected static class ClusterNodeNameComparator implements Comparator<ClusterNode>,
- Serializable {
-
- @Override
- public int compare(ClusterNode node1, ClusterNode node2) {
- if (null == node1 && null != node2) {
- return -1;
- } else if (null != node1 && null == node2) {
- return 1;
- } else if (null == node1) {
- return 0;
- }
-
- final String name1 = node1.name, name2 = node2.name;
- if (null == name1 && null != name2) {
- return -1;
- } else if (null != name1 && null == name2) {
- return 1;
- } else if (null == name1) {
- return 0;
- }
-
- return name1.compareTo(name2);
- }
-
- }
-
- @Override
- protected void render(Block html) {
- final Map<String,RoleInstance> containerInstances = getContainerInstances(
- appState.cloneOwnedContainerList());
-
- Map<String, Map<String, ClusterNode>> clusterNodeMap =
- appState.getRoleClusterNodeMapping();
- Map<String, ComponentInformation> componentInfoMap = appState.getComponentInfoSnapshot();
-
- for (Entry<String, Map<String, ClusterNode>> entry : clusterNodeMap.entrySet()) {
- final String name = entry.getKey();
- Map<String, ClusterNode> clusterNodesInRole = entry.getValue();
- //final RoleStatus roleStatus = entry.getValue();
-
- DIV<Hamlet> div = html.div("role-info ui-widget-content ui-corner-all");
-
- List<ClusterNode> nodesInRole =
- new ArrayList<>(clusterNodesInRole.values());
-
- div.h2(BOLD, StringUtils.capitalize(name));
-
- // Generate the details on this role
- ComponentInformation componentInfo = componentInfoMap.get(name);
- if (componentInfo != null) {
- Iterable<Entry<String,Integer>> stats = componentInfo.buildStatistics().entrySet();
- generateRoleDetails(div,"role-stats-wrap", "Specifications",
- Iterables.transform(stats, stringIntPairFunc));
- }
-
- // Sort the ClusterNodes by their name (containerid)
- Collections.sort(nodesInRole, new ClusterNodeNameComparator());
-
- // Generate the containers running this role
- generateRoleDetails(div, "role-stats-containers", "Containers",
- Iterables.transform(nodesInRole, new Function<ClusterNode,Entry<TableContent,String>>() {
-
- @Override
- public Entry<TableContent,String> apply(ClusterNode input) {
- final String containerId = input.name;
-
- if (containerInstances.containsKey(containerId)) {
- RoleInstance roleInst = containerInstances.get(containerId);
- if (roleInst.container.getNodeHttpAddress() != null) {
- return Maps.<TableContent,String> immutableEntry(
- new TableAnchorContent(containerId,
- buildNodeUrlForContainer(roleInst.container.getNodeHttpAddress(), containerId)), null);
- }
- }
- return Maps.immutableEntry(new TableContent(input.name), null);
- }
-
- }));
-
- Application application = appState.getApplication();
- Iterable<Entry<TableContent, String>> tableContent;
- tableContent = Collections.emptySet();
-
- // Generate the options used by this role
- generateRoleDetails(div, "role-options-wrap", "Role Options", tableContent);
-
- // Close the div for this role
- div._();
- }
- }
-
- protected static <T> Function<Entry<String,T>,Entry<TableContent,T>> toTableContentFunction() {
- return new Function<Entry<String,T>,Entry<TableContent,T>>() {
- @Override
- public Entry<TableContent,T> apply(@Nonnull Entry<String,T> input) {
- return Maps.immutableEntry(new TableContent(input.getKey()), input.getValue());
- }
- };
- }
-
- protected Map<String,RoleInstance> getContainerInstances(List<RoleInstance> roleInstances) {
- Map<String,RoleInstance> map = Maps.newHashMapWithExpectedSize(roleInstances.size());
- for (RoleInstance roleInstance : roleInstances) {
- // UUID is the containerId
- map.put(roleInstance.id, roleInstance);
- }
- return map;
- }
-
- /**
- * Given a div, a name for this data, and some pairs of data, generate a nice HTML table. If contents is empty (of size zero), then a mesage will be printed
- * that there were no items instead of an empty table.
- *
- */
- protected <T1 extends TableContent,T2> void generateRoleDetails(DIV<Hamlet> parent, String divSelector, String detailsName, Iterable<Entry<T1,T2>> contents) {
- final DIV<DIV<Hamlet>> div = parent.div(divSelector).h3(BOLD, detailsName);
-
- int offset = 0;
- TABLE<DIV<DIV<Hamlet>>> table = null;
- TBODY<TABLE<DIV<DIV<Hamlet>>>> tbody = null;
- for (Entry<T1,T2> content : contents) {
- if (null == table) {
- table = div.table("ui-widget-content ui-corner-bottom");
- tbody = table.tbody();
- }
-
- TR<TBODY<TABLE<DIV<DIV<Hamlet>>>>> row = tbody.tr(offset % 2 == 0 ? EVEN : ODD);
-
- // Defer to the implementation of the TableContent for what the cell should contain
- content.getKey().printCell(row);
-
- // Only add the second column if the element is non-null
- // This also lets us avoid making a second method if we're only making a one-column table
- if (null != content.getValue()) {
- row.td(content.getValue().toString());
- }
-
- row._();
-
- offset++;
- }
-
- // If we made a table, close it out
- if (null != table) {
- tbody._()._();
- } else {
- // Otherwise, throw in a nice "no content" message
- div.p("no-table-contents")._("None")._();
- }
-
- // Close out the initial div
- div._();
- }
-
- /**
- * Build a URL from the address:port and container ID directly to the NodeManager service
- * @param nodeAddress
- * @param containerId
- * @return
- */
- protected String buildNodeUrlForContainer(String nodeAddress, String containerId) {
- StringBuilder sb = new StringBuilder(SCHEME.length() + nodeAddress.length() + PATH.length() + containerId.length());
-
- sb.append(SCHEME).append(nodeAddress).append(PATH).append(containerId);
-
- return sb.toString();
- }
-
- /**
- * Creates a table cell with the provided String as content.
- */
- protected static class TableContent {
- private String cell;
-
- public TableContent(String cell) {
- this.cell = cell;
- }
-
- public String getCell() {
- return cell;
- }
-
- /**
- * Adds a td to the given tr. The tr is not closed
- * @param tableRow
- */
- public void printCell(TR<?> tableRow) {
- tableRow.td(this.cell);
- }
- }
-
- /**
- * Creates a table cell with an anchor to the given URL with the provided String as content.
- */
- protected static class TableAnchorContent extends TableContent {
- private String anchorUrl;
-
- public TableAnchorContent(String cell, String anchorUrl) {
- super(cell);
- this.anchorUrl = anchorUrl;
- }
-
- /* (non-javadoc)
- * @see org.apache.slider.server.appmaster.web.view.ContainerStatsBlock$TableContent#printCell()
- */
- @Override
- public void printCell(TR<?> tableRow) {
- tableRow.td().a(anchorUrl, getCell())._();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
deleted file mode 100644
index c0a120d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
+++ /dev/null
@@ -1,273 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.view;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.inject.Inject;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.LI;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL;
-import org.apache.slider.api.types.ApplicationLivenessInformation;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.registry.docstore.ExportEntry;
-import org.apache.slider.core.registry.docstore.PublishedExports;
-import org.apache.slider.core.registry.docstore.PublishedExportsSet;
-import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_COMPONENTS;
-
-/**
- * The main content on the Slider AM web page
- */
-public class IndexBlock extends SliderHamletBlock {
- private static final Logger log = LoggerFactory.getLogger(IndexBlock.class);
-
- /**
- * Message printed when application is at full size.
- *
- * {@value}
- */
- public static final String ALL_CONTAINERS_ALLOCATED = "all containers allocated";
-
- @Inject
- public IndexBlock(WebAppApi slider) {
- super(slider);
- }
-
- @Override
- protected void render(Block html) {
- doIndex(html, getProviderName());
- }
-
- // An extra method to make testing easier since you can't make an instance of Block
- @VisibleForTesting
- protected void doIndex(Hamlet html, String providerName) {
- String name = appState.getApplicationName();
- if (name != null && (name.startsWith(" ") || name.endsWith(" "))) {
- name = "'" + name + "'";
- }
- DIV<Hamlet> div = html.div("general_info")
- .h1("index_header",
- "Application: " + name);
-
- ApplicationLivenessInformation liveness =
- appState.getApplicationLivenessInformation();
- String livestatus = liveness.allRequestsSatisfied
- ? ALL_CONTAINERS_ALLOCATED
- : String.format("Awaiting %d containers", liveness.requestsOutstanding);
- Hamlet.TABLE<DIV<Hamlet>> table1 = div.table();
- table1.tr()
- .td("Status")
- .td(livestatus)
- ._();
- table1.tr()
- .td("Total number of containers")
- .td(Integer.toString(appState.getNumOwnedContainers()))
- ._();
- table1.tr()
- .td("Create time: ")
- .td("N/A")
- ._();
- table1.tr()
- .td("Running since: ")
- .td("N/A")
- ._();
- table1.tr()
- .td("Time last flexed: ")
- .td("N/A")
- ._();
- table1.tr()
- .td("Application storage path: ")
- .td("N/A")
- ._();
- table1.tr()
- .td("Application configuration path: ")
- .td("N/A")
- ._();
- table1._();
- div._();
- div = null;
-
- DIV<Hamlet> containers = html.div("container_instances")
- .h3("Component Instances");
-
- int aaRoleWithNoSuitableLocations = 0;
- int aaRoleWithOpenRequest = 0;
- int roleWithOpenRequest = 0;
-
- Hamlet.TABLE<DIV<Hamlet>> table = containers.table();
- Hamlet.TR<Hamlet.THEAD<Hamlet.TABLE<DIV<Hamlet>>>> header = table.thead().tr();
- trb(header, "Component");
- trb(header, "Desired");
- trb(header, "Actual");
- trb(header, "Outstanding Requests");
- trb(header, "Failed");
- trb(header, "Failed to start");
- trb(header, "Placement");
- header._()._(); // tr & thead
-
- List<RoleStatus> roleStatuses =
- new ArrayList<>(appState.getRoleStatusMap().values());
- Collections.sort(roleStatuses, new RoleStatus.CompareByName());
- for (RoleStatus status : roleStatuses) {
- String roleName = status.getName();
- String nameUrl = apiPath(LIVE_COMPONENTS) + "/" + roleName;
- String aatext;
- if (status.isAntiAffinePlacement()) {
- boolean aaRequestOutstanding = status.isAARequestOutstanding();
- int pending = (int)status.getAAPending();
- aatext = buildAADetails(aaRequestOutstanding, pending);
- if (SliderUtils.isSet(status.getLabelExpression())) {
- aatext += " (label: " + status.getLabelExpression() + ")";
- }
- if (pending > 0 && !aaRequestOutstanding) {
- aaRoleWithNoSuitableLocations ++;
- } else if (aaRequestOutstanding) {
- aaRoleWithOpenRequest++;
- }
- } else {
- if (SliderUtils.isSet(status.getLabelExpression())) {
- aatext = "label: " + status.getLabelExpression();
- } else {
- aatext = "";
- }
- if (status.getRequested() > 0) {
- roleWithOpenRequest ++;
- }
- }
- ServiceMetrics metrics = status.getComponentMetrics();
- table.tr()
- .td().a(nameUrl, roleName)._()
- .td(String.format("%d", metrics.containersDesired.value()))
- .td(String.format("%d", metrics.containersRunning.value()))
- .td(String.format("%d", metrics.containersRequested.value()))
- .td(String.format("%d", metrics.containersFailed.value()))
- .td(aatext)
- ._();
- }
-
- // empty row for some more spacing
- table.tr()._();
- // close table
- table._();
-
- containers._();
- containers = null;
-
- // some spacing
- html.div()._();
- html.div()._();
-
- DIV<Hamlet> diagnostics = html.div("diagnostics");
-
- List<String> statusEntries = new ArrayList<>(0);
- if (roleWithOpenRequest > 0) {
- statusEntries.add(String.format("%d %s with requests unsatisfiable by cluster",
- roleWithOpenRequest, plural(roleWithOpenRequest, "component")));
- }
- if (aaRoleWithNoSuitableLocations > 0) {
- statusEntries.add(String.format("%d anti-affinity %s no suitable nodes in the cluster",
- aaRoleWithNoSuitableLocations,
- plural(aaRoleWithNoSuitableLocations, "component has", "components have")));
- }
- if (aaRoleWithOpenRequest > 0) {
- statusEntries.add(String.format("%d anti-affinity %s with requests unsatisfiable by cluster",
- aaRoleWithOpenRequest,
- plural(aaRoleWithOpenRequest, "component has", "components have")));
-
- }
- if (!statusEntries.isEmpty()) {
- diagnostics.h3("Diagnostics");
- Hamlet.TABLE<DIV<Hamlet>> diagnosticsTable = diagnostics.table();
- for (String entry : statusEntries) {
- diagnosticsTable.tr().td(entry)._();
- }
- diagnosticsTable._();
- }
- diagnostics._();
-
- DIV<Hamlet> provider_info = html.div("provider_info");
- provider_info.h3(providerName + " information");
- UL<Hamlet> ul = html.ul();
- //TODO render app/cluster status
- ul._();
- provider_info._();
-
- DIV<Hamlet> exports = html.div("exports");
- exports.h3("Exports");
- ul = html.ul();
- enumeratePublishedExports(appState.getPublishedExportsSet(), ul);
- ul._();
- exports._();
- }
-
- @VisibleForTesting
- String buildAADetails(boolean outstanding, int pending) {
- return String.format("Anti-affinity:%s %d pending %s",
- (outstanding ? " 1 active request and" : ""),
- pending, plural(pending, "request"));
- }
-
- private String plural(int n, String singular) {
- return plural(n, singular, singular + "s");
- }
- private String plural(int n, String singular, String plural) {
- return n == 1 ? singular : plural;
- }
-
- private void trb(Hamlet.TR tr,
- String text) {
- tr.td().b(text)._();
- }
-
- private String getProviderName() {
- return "docker";
- }
-
-
- protected void enumeratePublishedExports(PublishedExportsSet exports, UL<Hamlet> ul) {
- for(String key : exports.keys()) {
- PublishedExports export = exports.get(key);
- LI<UL<Hamlet>> item = ul.li();
- item.span().$class("bold")._(export.description)._();
- UL sublist = item.ul();
- for (Entry<String, Set<ExportEntry>> entry : export.sortedEntries()
- .entrySet()) {
- if (SliderUtils.isNotEmpty(entry.getValue())) {
- LI sublistItem = sublist.li()._(entry.getKey());
- for (ExportEntry exportEntry : entry.getValue()) {
- sublistItem._(exportEntry.getValue());
- }
- sublistItem._();
- }
- }
- sublist._();
- item._();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
deleted file mode 100644
index 069d386..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.view;
-
-import com.google.inject.Inject;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-
-import static org.apache.slider.server.appmaster.web.SliderAMWebApp.*;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
-
-/**
- *
- */
-public class NavBlock extends SliderHamletBlock {
-
- @Inject
- public NavBlock(WebAppApi slider) {
- super(slider);
- }
-
- @Override
- protected void render(Block html) {
- html.
- div("#nav").
- h3("Slider").
- ul().
- li().a(this.prefix(), "Overview")._().
- li().a(relPath(CONTAINER_STATS), "Statistics")._().
- li().a(relPath(CLUSTER_SPEC), "Specification")._().
- li().a(rootPath(SYSTEM_METRICS_JSON), "Metrics")._().
- li().a(rootPath(SYSTEM_HEALTHCHECK), "Health")._().
- li().a(rootPath(SYSTEM_THREADS), "Threads")._().
- _()
- .h3("REST API").
- ul().
- li().a(apiPath(MODEL_DESIRED), "Specified")._().
- li().a(apiPath(MODEL_RESOLVED), "Resolved")._().
- li().a(apiPath(LIVE_RESOURCES), "Resources")._().
- li().a(apiPath(LIVE_COMPONENTS), "Components")._().
- li().a(apiPath(LIVE_CONTAINERS), "Containers")._().
- li().a(apiPath(LIVE_NODES), "Nodes")._().
- li().a(apiPath(LIVE_STATISTICS), "Statistics")._().
- li().a(apiPath(LIVE_LIVENESS), "Liveness")._()
- ._()
- ._();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java
deleted file mode 100644
index 5f44bda..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.view;
-
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.rest.RestPaths;
-
-import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_APPLICATION;
-
-/**
- * Anything we want to share across slider hamlet blocks
- */
-public abstract class SliderHamletBlock extends HtmlBlock {
-
- protected final StateAccessForProviders appState;
- protected final RestPaths restPaths = new RestPaths();
-
- public SliderHamletBlock(WebAppApi slider) {
- this.appState = slider.getAppState();
- }
-
- protected String rootPath(String absolutePath) {
- return root_url(absolutePath);
- }
-
- protected String relPath(String... args) {
- return ujoin(this.prefix(), args);
- }
-
- protected String apiPath(String api) {
- return root_url(SLIDER_PATH_APPLICATION, api);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java
deleted file mode 100644
index 77408a5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.avro;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.common.tools.SliderUtils;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * The role history
- */
-public class LoadedRoleHistory {
-
- private RoleHistoryHeader header;
-
- private Path path;
-
- public final Map<String, Integer> roleMap = new HashMap<>();
-
- public final List<NodeEntryRecord> records = new ArrayList<>();
-
- /**
- * Add a record
- * @param record
- */
- public void add(NodeEntryRecord record) {
- records.add(record);
- }
-
- /**
- * Number of loaded records
- * @return
- */
- public int size() {
- return records.size();
- }
-
- public RoleHistoryHeader getHeader() {
- return header;
- }
-
- public void setHeader(RoleHistoryHeader header) {
- this.header = header;
- }
-
- public Path getPath() {
- return path;
- }
-
- public void setPath(Path path) {
- this.path = path;
- }
-
- public void buildMapping(Map<CharSequence, Integer> source) {
- roleMap.clear();
- for (Map.Entry<CharSequence, Integer> entry : source.entrySet()) {
- roleMap.put(SliderUtils.sequenceToString(entry.getKey()),
- entry.getValue());
- }
- }
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder(
- "LoadedRoleHistory{");
- sb.append("path=").append(path);
- sb.append("; number of roles=").append(roleMap.size());
- sb.append("; size=").append(size());
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/NewerFilesFirst.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/NewerFilesFirst.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/NewerFilesFirst.java
deleted file mode 100644
index 2e049cb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/NewerFilesFirst.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.avro;
-
-import org.apache.hadoop.fs.Path;
-
-import java.io.Serializable;
-import java.util.Comparator;
-
-/**
- * Compare two filenames by name; the more recent one comes first
- */
-public class NewerFilesFirst implements Comparator<Path>, Serializable {
-
- /**
- * Takes the ordering of path names from the normal string comparison
- * and negates it, so that names that come after other names in
- * the string sort come before here
- * @param o1 leftmost
- * @param o2 rightmost
- * @return positive if o1 > o2
- */
- @Override
- public int compare(Path o1, Path o2) {
- return (o2.getName().compareTo(o1.getName()));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/OlderFilesFirst.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/OlderFilesFirst.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/OlderFilesFirst.java
deleted file mode 100644
index 407aaa6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/OlderFilesFirst.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.avro;
-
-import org.apache.hadoop.fs.Path;
-
-import java.io.Serializable;
-import java.util.Comparator;
-
-/**
- * Compare two filenames by name; the older ones comes first
- */
-public class OlderFilesFirst implements Comparator<Path>, Serializable {
-
- /**
- * Takes the ordering of path names from the normal string comparison
- * and negates it, so that names that come after other names in
- * the string sort come before here
- * @param o1 leftmost
- * @param o2 rightmost
- * @return positive if o1 > o2
- */
- @Override
- public int compare(Path o1, Path o2) {
- return (o1.getName().compareTo(o2.getName()));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
deleted file mode 100644
index 52553d0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
+++ /dev/null
@@ -1,449 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.avro;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.avro.AvroTypeException;
-import org.apache.avro.Schema;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.io.DatumWriter;
-import org.apache.avro.io.Decoder;
-import org.apache.avro.io.DecoderFactory;
-import org.apache.avro.io.Encoder;
-import org.apache.avro.io.EncoderFactory;
-import org.apache.avro.specific.SpecificDatumReader;
-import org.apache.avro.specific.SpecificDatumWriter;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.GlobFilter;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.server.appmaster.state.NodeEntry;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.RoleHistory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Locale;
-import java.util.Map;
-
-/**
- * Write out the role history to an output stream.
- */
-public class RoleHistoryWriter {
- protected static final Logger log =
- LoggerFactory.getLogger(RoleHistoryWriter.class);
-
- /**
- * Although Avro is designed to handle some changes, we still keep a version
- * marker in the file to catch changes that are fundamentally incompatible
- * at the semantic level -changes that require either a different
- * parser or get rejected outright.
- */
- public static final int ROLE_HISTORY_VERSION = 0x01;
-
- /**
- * Write out the history.
- * This does not update the history's dirty/savetime fields
- *
- * @param out outstream
- * @param history history
- * @param savetime time in millis for the save time to go in as a record
- * @return no of records written
- * @throws IOException IO failures
- */
- public long write(OutputStream out, RoleHistory history, long savetime)
- throws IOException {
- try {
- DatumWriter<RoleHistoryRecord> writer =
- new SpecificDatumWriter<>(RoleHistoryRecord.class);
-
- RoleHistoryRecord record = createHeaderRecord(savetime, history);
- int roles = history.getRoleSize();
- Schema schema = record.getSchema();
- Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out);
- writer.write(record, encoder);
- // now write the rolemap record
- writer.write(createRolemapRecord(history), encoder);
- long count = 0;
- //now for every role history entry, write out its record
- Collection<NodeInstance> instances = history.cloneNodemap().values();
- for (NodeInstance instance : instances) {
- for (int role = 0; role < roles; role++) {
- NodeEntry nodeEntry = instance.get(role);
-
- if (nodeEntry != null) {
- NodeEntryRecord ner = build(nodeEntry, role, instance.hostname);
- record = new RoleHistoryRecord(ner);
- writer.write(record, encoder);
- count++;
- }
- }
- }
- // footer
- RoleHistoryFooter footer = new RoleHistoryFooter();
- footer.setCount(count);
- writer.write(new RoleHistoryRecord(footer), encoder);
- encoder.flush();
- return count;
- } finally {
- out.close();
- }
- }
-
- /**
- * Create the header record
- * @param savetime time of save
- * @param history history
- * @return a record to place at the head of the file
- */
- private RoleHistoryRecord createHeaderRecord(long savetime, RoleHistory history) {
- RoleHistoryHeader header = new RoleHistoryHeader();
- header.setVersion(ROLE_HISTORY_VERSION);
- header.setSaved(savetime);
- header.setSavedx(Long.toHexString(savetime));
- header.setSavedate(SliderUtils.toGMTString(savetime));
- header.setRoles(history.getRoleSize());
- return new RoleHistoryRecord(header);
- }
-
- /**
- * Create the rolemap record
- * @param history history
- * @return a record to insert into the file
- */
- private RoleHistoryRecord createRolemapRecord(RoleHistory history) {
- RoleHistoryMapping entry = new RoleHistoryMapping();
- Map<CharSequence, Integer> mapping = history.buildMappingForHistoryFile();
- entry.setRolemap(mapping);
- return new RoleHistoryRecord(entry);
- }
-
- /**
- * Write the history information to a file
- *
- * @param fs filesystem
- * @param path path
- * @param overwrite overwrite flag
- * @param history history
- * @param savetime time in millis for the save time to go in as a record
- * @return no of records written
- * @throws IOException IO failures
- */
- public long write(FileSystem fs,
- Path path,
- boolean overwrite,
- RoleHistory history,
- long savetime)
- throws IOException {
- FSDataOutputStream out = fs.create(path, overwrite);
- return write(out, history, savetime);
- }
-
-
- /**
- * Create the filename for a history file
- * @param time time value
- * @return a filename such that later filenames sort later in the directory
- */
- public Path createHistoryFilename(Path historyPath, long time) {
- String filename = String.format(Locale.ENGLISH,
- SliderKeys.HISTORY_FILENAME_CREATION_PATTERN,
- time);
- Path path = new Path(historyPath, filename);
- return path;
- }
-
- /**
- * Build a {@link NodeEntryRecord} from a node entry; include whether
- * the node is in use and when it was last used.
- * @param entry entry count
- * @param role role index
- * @param hostname name
- * @return the record
- */
- private NodeEntryRecord build(NodeEntry entry, int role, String hostname) {
- NodeEntryRecord record = new NodeEntryRecord(
- hostname, role, entry.getLive() > 0, entry.getLastUsed()
- );
- return record;
- }
-
- /**
- * Read a history, returning one that is ready to have its onThaw()
- * method called
- * @param in input source
- * @return no. of entries read
- * @throws IOException problems
- */
- public LoadedRoleHistory read(InputStream in) throws
- IOException,
- BadConfigException {
- try {
- LoadedRoleHistory loadedRoleHistory = new LoadedRoleHistory();
- DatumReader<RoleHistoryRecord> reader =
- new SpecificDatumReader<>(RoleHistoryRecord.class);
- Decoder decoder =
- DecoderFactory.get().jsonDecoder(RoleHistoryRecord.getClassSchema(),
- in);
-
- //read header : no entry -> EOF
- RoleHistoryRecord record = reader.read(null, decoder);
- if (record == null) {
- throw new IOException("Role History Header not found at start of file.");
- }
- Object entry = record.getEntry();
- if (!(entry instanceof RoleHistoryHeader)) {
- throw new IOException("Role History Header not found at start of file");
- }
- RoleHistoryHeader header = (RoleHistoryHeader) entry;
- if (header.getVersion() != ROLE_HISTORY_VERSION) {
- throw new IOException(
- String.format("Can't read role file version %04x -need %04x",
- header.getVersion(),
- ROLE_HISTORY_VERSION));
- }
- loadedRoleHistory.setHeader(header);
- RoleHistoryFooter footer = null;
- int records = 0;
- //go through reading data
- try {
- while (footer == null) {
- record = reader.read(null, decoder);
- if (record == null) {
- throw new IOException("Null record after " + records + " records");
- }
- entry = record.getEntry();
-
- if (entry instanceof RoleHistoryHeader) {
- throw new IOException("Duplicate Role History Header found");
- } else if (entry instanceof RoleHistoryMapping) {
- // role history mapping entry
- if (!loadedRoleHistory.roleMap.isEmpty()) {
- // duplicate role maps are viewed as something to warn over, rather than fail
- log.warn("Duplicate role map; ignoring");
- } else {
- RoleHistoryMapping historyMapping = (RoleHistoryMapping) entry;
- loadedRoleHistory.buildMapping(historyMapping.getRolemap());
- }
- } else if (entry instanceof NodeEntryRecord) {
- // normal record
- records++;
- NodeEntryRecord nodeEntryRecord = (NodeEntryRecord) entry;
- loadedRoleHistory.add(nodeEntryRecord);
- } else if (entry instanceof RoleHistoryFooter) {
- //tail end of the file
- footer = (RoleHistoryFooter) entry;
- } else {
- // this is to handle future versions, such as when rolling back
- // from a later version of slider
- log.warn("Discarding unknown record {}", entry);
- }
- }
- } catch (EOFException e) {
- EOFException ex = new EOFException(
- "End of file reached after " + records + " records");
- ex.initCause(e);
- throw ex;
- }
- // at this point there should be no data left.
- // check by reading and expecting a -1
- if (in.read() > 0) {
- // footer is in stream before the last record
- throw new EOFException(
- "File footer reached before end of file -after " + records +
- " records");
- }
- if (records != footer.getCount()) {
- log.warn("mismatch between no of records saved {} and number read {}",
- footer.getCount(), records);
- }
- return loadedRoleHistory;
- } finally {
- in.close();
- }
-
- }
-
- /**
- * Read a role history from a path in a filesystem
- * @param fs filesystem
- * @param path path to the file
- * @return the records read
- * @throws IOException any problem
- */
- public LoadedRoleHistory read(FileSystem fs, Path path)
- throws IOException, BadConfigException {
- FSDataInputStream instream = fs.open(path);
- return read(instream);
- }
-
- /**
- * Read from a resource in the classpath -used for testing
- * @param resource resource
- * @return the records read
- * @throws IOException any problem
- */
- public LoadedRoleHistory read(String resource)
- throws IOException, BadConfigException {
-
- return read(this.getClass().getClassLoader().getResourceAsStream(resource));
- }
-
-
- /**
- * Find all history entries in a dir. The dir is created if it is
- * not already defined.
- *
- * The scan uses the match pattern {@link SliderKeys#HISTORY_FILENAME_MATCH_PATTERN}
- * while dropping empty files and directories which match the pattern.
- * The list is then sorted with a comparator that sorts on filename,
- * relying on the filename of newer created files being later than the old ones.
- *
- *
- *
- * @param fs filesystem
- * @param dir dir to scan
- * @param includeEmptyFiles should empty files be included in the result?
- * @return a possibly empty list
- * @throws IOException IO problems
- * @throws FileNotFoundException if the target dir is actually a path
- */
- public List<Path> findAllHistoryEntries(FileSystem fs,
- Path dir,
- boolean includeEmptyFiles) throws IOException {
- assert fs != null;
- assert dir != null;
- if (!fs.exists(dir)) {
- fs.mkdirs(dir);
- } else if (!fs.isDirectory(dir)) {
- throw new FileNotFoundException("Not a directory " + dir.toString());
- }
-
- PathFilter filter = new GlobFilter(SliderKeys.HISTORY_FILENAME_GLOB_PATTERN);
- FileStatus[] stats = fs.listStatus(dir, filter);
- List<Path> paths = new ArrayList<Path>(stats.length);
- for (FileStatus stat : stats) {
- log.debug("Possible entry: {}", stat.toString());
- if (stat.isFile() && (includeEmptyFiles || stat.getLen() > 0)) {
- paths.add(stat.getPath());
- }
- }
- sortHistoryPaths(paths);
- return paths;
- }
-
- @VisibleForTesting
- public static void sortHistoryPaths(List<Path> paths) {
- Collections.sort(paths, new NewerFilesFirst());
- }
-
- /**
- * Iterate through the paths until one can be loaded
- * @param paths paths to load
- * @return the loaded history including the path -or null if all failed to load
- */
- public LoadedRoleHistory attemptToReadHistory(FileSystem fileSystem,
- List<Path> paths)
- throws BadConfigException {
- ListIterator<Path> pathIterator = paths.listIterator();
- boolean success = false;
- LoadedRoleHistory history = null;
- while (!success && pathIterator.hasNext()) {
- Path path = pathIterator.next();
- try {
- history = read(fileSystem, path);
- //success
- success = true;
- history.setPath(path);
- } catch (IOException e) {
- log.info("Failed to read {}", path, e);
- } catch (AvroTypeException e) {
- log.warn("Failed to parse {}", path, e);
- } catch (Exception e) {
- // low level event logged @ warn level
- log.warn("Exception while reading {}", path, e);
- }
- }
- return history;
- }
-
- /**
- * Try to load the history from a directory -a failure to load a specific
- * file is downgraded to a log and the next older path attempted instead
- * @param fs filesystem
- * @param dir dir to load from
- * @return the history loaded, including the path
- * @throws IOException if indexing the history directory fails.
- */
- public LoadedRoleHistory loadFromHistoryDir(FileSystem fs, Path dir)
- throws IOException, BadConfigException {
- assert fs != null: "null filesystem";
- List<Path> entries = findAllHistoryEntries(fs, dir, false);
- return attemptToReadHistory(fs, entries);
- }
-
- /**
- * Delete all old history entries older than the one we want to keep. This
- * uses the filename ordering to determine age, not timestamps
- * @param fileSystem filesystem
- * @param keep path to keep -used in thresholding the files
- * @return the number of files deleted
- * @throws FileNotFoundException if the path to keep is not present (safety
- * check to stop the entire dir being purged)
- * @throws IOException IO problems
- */
- public int purgeOlderHistoryEntries(FileSystem fileSystem, Path keep)
- throws IOException { assert fileSystem != null : "null filesystem";
- if (!fileSystem.exists(keep)) {
- throw new FileNotFoundException(keep.toString());
- }
- Path dir = keep.getParent();
- log.debug("Purging entries in {} up to {}", dir, keep);
- List<Path> paths = findAllHistoryEntries(fileSystem, dir, true);
- Collections.sort(paths, new OlderFilesFirst());
- int deleteCount = 0;
- for (Path path : paths) {
- if (path.equals(keep)) {
- break;
- } else {
- log.debug("Deleting {}", path);
- deleteCount++;
- fileSystem.delete(path, false);
- }
- }
- return deleteCount;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/HttpProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/HttpProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/HttpProbe.java
deleted file mode 100644
index 5eba622..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/HttpProbe.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.servicemonitor;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.slider.common.tools.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.Map;
-
-public class HttpProbe extends Probe {
- protected static final Logger log = LoggerFactory.getLogger(HttpProbe.class);
-
- private static final String HOST_TOKEN = "${THIS_HOST}";
-
- private final String urlString;
- private final int timeout;
- private final int min, max;
-
-
- public HttpProbe(String url, int timeout, int min, int max, Configuration
- conf) {
- super("Http probe of " + url + " [" + min + "-" + max + "]", conf);
- this.urlString = url;
- this.timeout = timeout;
- this.min = min;
- this.max = max;
- }
-
- public static HttpProbe create(Map<String, String> props)
- throws IOException {
- String urlString = getProperty(props, WEB_PROBE_URL, null);
- new URL(urlString);
- int timeout = getPropertyInt(props, WEB_PROBE_CONNECT_TIMEOUT,
- WEB_PROBE_CONNECT_TIMEOUT_DEFAULT);
- int minSuccess = getPropertyInt(props, WEB_PROBE_MIN_SUCCESS,
- WEB_PROBE_MIN_SUCCESS_DEFAULT);
- int maxSuccess = getPropertyInt(props, WEB_PROBE_MAX_SUCCESS,
- WEB_PROBE_MAX_SUCCESS_DEFAULT);
- return new HttpProbe(urlString, timeout, minSuccess, maxSuccess, null);
- }
-
-
- private static HttpURLConnection getConnection(URL url, int timeout) throws
- IOException {
- HttpURLConnection connection = (HttpURLConnection) url.openConnection();
- connection.setInstanceFollowRedirects(true);
- connection.setConnectTimeout(timeout);
- return connection;
- }
-
- @Override
- public ProbeStatus ping(ComponentInstance instance) {
- ProbeStatus status = new ProbeStatus();
- ContainerStatus containerStatus = instance.getContainerStatus();
- if (containerStatus == null || SliderUtils.isEmpty(containerStatus.getIPs())
- || StringUtils.isEmpty(containerStatus.getHost())) {
- status.fail(this, new IOException("IP is not available yet"));
- return status;
- }
-
- String ip = containerStatus.getIPs().get(0);
- HttpURLConnection connection = null;
- try {
- URL url = new URL(urlString.replace(HOST_TOKEN, ip));
- connection = getConnection(url, this.timeout);
- int rc = connection.getResponseCode();
- if (rc < min || rc > max) {
- String error = "Probe " + url + " error code: " + rc;
- log.info(error);
- status.fail(this,
- new IOException(error));
- } else {
- status.succeed(this);
- }
- } catch (Throwable e) {
- String error = "Probe " + urlString + " failed for IP " + ip + ": " + e;
- log.info(error, e);
- status.fail(this,
- new IOException(error, e));
- } finally {
- if (connection != null) {
- connection.disconnect();
- }
- }
- return status;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/LogEntryBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/LogEntryBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/LogEntryBuilder.java
deleted file mode 100644
index a1ad44f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/LogEntryBuilder.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.servicemonitor;
-
-/**
- * Build up log entries for ease of splunk
- */
-public class LogEntryBuilder {
-
- private final StringBuilder builder = new StringBuilder();
-
- public LogEntryBuilder() {
- }
-
- public LogEntryBuilder(String text) {
- elt(text);
- }
-
-
- public LogEntryBuilder(String name, Object value) {
- entry(name, value);
- }
-
- public LogEntryBuilder elt(String text) {
- addComma();
- builder.append(text);
- return this;
- }
-
- public LogEntryBuilder elt(String name, Object value) {
- addComma();
- entry(name, value);
- return this;
- }
-
- private void addComma() {
- if (!isEmpty()) {
- builder.append(", ");
- }
- }
-
- private void entry(String name, Object value) {
- builder.append(name).append('=');
- if (value != null) {
- builder.append('"').append(value.toString()).append('"');
- } else {
- builder.append("null");
- }
- }
-
- @Override
- public String toString() {
- return builder.toString();
- }
-
- private boolean isEmpty() {
- return builder.length() == 0;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorKeys.java
deleted file mode 100644
index e97ab43..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorKeys.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.servicemonitor;
-
-/**
- * Config keys for monitoring
- */
-public interface MonitorKeys {
-
- /**
- * Port probing key : port to attempt to create a TCP connection to {@value}.
- */
- String PORT_PROBE_PORT = "port";
- /**
- * Port probing key : timeout for the the connection attempt {@value}.
- */
- String PORT_PROBE_CONNECT_TIMEOUT = "timeout";
- /**
- * Port probing default : timeout for the connection attempt {@value}.
- */
- int PORT_PROBE_CONNECT_TIMEOUT_DEFAULT = 1000;
-
- /**
- * Web probing key : URL {@value}.
- */
- String WEB_PROBE_URL = "url";
- /**
- * Web probing key : min success code {@value}.
- */
- String WEB_PROBE_MIN_SUCCESS = "min.success";
- /**
- * Web probing key : max success code {@value}.
- */
- String WEB_PROBE_MAX_SUCCESS = "max.success";
- /**
- * Web probing default : min successful response code {@value}.
- */
- int WEB_PROBE_MIN_SUCCESS_DEFAULT = 200;
- /**
- * Web probing default : max successful response code {@value}.
- */
- int WEB_PROBE_MAX_SUCCESS_DEFAULT = 299;
- /**
- * Web probing key : timeout for the connection attempt {@value}
- */
- String WEB_PROBE_CONNECT_TIMEOUT = "timeout";
- /**
- * Port probing default : timeout for the connection attempt {@value}.
- */
- int WEB_PROBE_CONNECT_TIMEOUT_DEFAULT = 1000;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java
deleted file mode 100644
index 1e5c94c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.servicemonitor;
-
-import org.apache.slider.api.resource.ReadinessCheck;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Formatter;
-import java.util.Locale;
-
-/**
- * Various utils to work with the monitor
- */
-public final class MonitorUtils {
- protected static final Logger LOG = LoggerFactory.getLogger(MonitorUtils
- .class);
-
- private MonitorUtils() {
- }
-
- public static String toPlural(int val) {
- return val != 1 ? "s" : "";
- }
-
- /**
- * Convert milliseconds to human time -the exact format is unspecified
- * @param milliseconds a time in milliseconds
- * @return a time that is converted to human intervals
- */
- public static String millisToHumanTime(long milliseconds) {
- StringBuilder sb = new StringBuilder();
- // Send all output to the Appendable object sb
- Formatter formatter = new Formatter(sb, Locale.US);
-
- long s = Math.abs(milliseconds / 1000);
- long m = Math.abs(milliseconds % 1000);
- if (milliseconds > 0) {
- formatter.format("%d.%03ds", s, m);
- } else if (milliseconds == 0) {
- formatter.format("0");
- } else {
- formatter.format("-%d.%03ds", s, m);
- }
- return sb.toString();
- }
-
- public static Probe getProbe(ReadinessCheck readinessCheck) {
- if (readinessCheck == null) {
- return null;
- }
- if (readinessCheck.getType() == null) {
- return null;
- }
- try {
- switch (readinessCheck.getType()) {
- case HTTP:
- return HttpProbe.create(readinessCheck.getProps());
- case PORT:
- return PortProbe.create(readinessCheck.getProps());
- default:
- return null;
- }
- } catch (Throwable t) {
- throw new IllegalArgumentException("Error creating readiness check " +
- t);
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[70/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index 88f74ef..17f8c95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -22,7 +22,7 @@ info:
Bringing a new service on YARN today is not a simple experience. The APIs of existing frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs) or writing a complex spec (for declarative frameworks). In addition to building critical building blocks inside YARN (as part of other efforts at link:https://issues.apache.org/jira/browse/YARN-4692[YARN-4692]), there is a need for simplifying the user facing story for building services. Experience of projects like Apache Slider running real-life services like HBase, Storm, Accumulo, Solr etc, gives us some very good insights on how simplified APIs for services should look like.
- To this end, we should look at a new simple-services API layer backed by REST interfaces. This API can be used to create and manage the lifecycle of YARN services. Services here can range from simple single-component apps to complex multi-component assemblies needing orchestration.
+ To this end, we should look at a new simple-services API layer backed by REST interfaces. This API can be used to create and manage the lifecycle of YARN services. Services here can range from simple single-component service to complex multi-component assemblies needing orchestration.
We should also look at making this a unified REST based entry point for other important features like resource-profile management (link:https://issues.apache.org/jira/browse/YARN-3926[YARN-3926]), package-definitions' lifecycle-management and service-discovery (link:https://issues.apache.org/jira/browse/YARN-913[YARN-913]/link:https://issues.apache.org/jira/browse/YARN-4757[YARN-4757]). We also need to flesh out its relation to our present much lower level REST APIs (link:https://issues.apache.org/jira/browse/YARN-1695[YARN-1695]) in YARN for application-submission and management.
@@ -41,177 +41,177 @@ schemes:
- http
- https
# will be prefixed to all paths
-basePath: /services/v1/
+basePath: /ws/v1/
consumes:
- application/json
produces:
- application/json
paths:
- /applications:
+ /services:
get:
- summary: List of applications/services running in the cluster
- description: Get a list of all currently running applications (response includes a minimal projection of the application info). For more details do a GET on a specific application name.
+ summary: List of services running in the cluster
+ description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name.
responses:
200:
- description: An array of applications
+ description: An array of services
schema:
type: array
items:
- $ref: '#/definitions/Application'
+ $ref: '#/definitions/Service'
default:
description: Unexpected error
schema:
- $ref: '#/definitions/ApplicationStatus'
+ $ref: '#/definitions/ServiceStatus'
post:
- summary: Create an application/service
- description: Create an application. The request JSON is an Application object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the application creation request. There is no guarantee that the application will actually reach a RUNNING state. Resource availability and several other factors determines if the application will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the application and determine its state.
+ summary: Create a service
+ description: Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state.
parameters:
- - name: Application
+ - name: Service
in: body
- description: Application request object
+ description: Service request object
required: true
schema:
- $ref: '#/definitions/Application'
+ $ref: '#/definitions/Service'
responses:
202:
description: Request accepted
default:
description: Unexpected error
schema:
- $ref: '#/definitions/ApplicationStatus'
+ $ref: '#/definitions/ServiceStatus'
- /applications/{app_name}:
+ /services/{service_name}:
get:
- summary: Get an application/service details
- description: Return the details (including containers) of a running application
+ summary: Get service details
+ description: Return the details (including containers) of a running service
parameters:
- - name: app_name
+ - name: service_name
in: path
- description: Application name
+ description: Service name
required: true
type: string
responses:
200:
- description: An application object
+ description: a service object
schema:
type: object
items:
- $ref: '#/definitions/Application'
+ $ref: '#/definitions/Service'
examples:
- app_name: logsearch
+ service_name: logsearch
artifact:
id: logsearch:latest
type: docker
404:
- description: Application does not exist
+ description: Service does not exist
default:
description: Unexpected error
schema:
- $ref: '#/definitions/ApplicationStatus'
+ $ref: '#/definitions/ServiceStatus'
put:
- summary: Update an application/service or upgrade the binary version of the components of a running application
- description: Update the runtime properties of an application. As of now, only update of lifetime and number of instances (flexing) of the components of an application is supported. The PUT operation is also used to orchestrate an upgrade of the application containers to a newer version of their artifacts.
+ summary: Update a service or upgrade the binary version of the components of a running service
+ description: Update the runtime properties of a service. As of now, only update of lifetime and number of instances (flexing) of the components of a service is supported. The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts.
parameters:
- - name: app_name
+ - name: service_name
in: path
- description: Application name
+ description: Service name
required: true
type: string
responses:
204:
description: Update or upgrade was successful
404:
- description: Application does not exist
+ description: Service does not exist
default:
description: Unexpected error
schema:
- $ref: '#/definitions/ApplicationStatus'
+ $ref: '#/definitions/ServiceStatus'
delete:
- summary: Destroy application/service
- description: Destroy an application and release all resources. This API might have to return JSON data providing location of logs, etc. Not finalized yet.
+ summary: Destroy service
+ description: Destroy a service and release all resources. This API might have to return JSON data providing location of logs, etc. Not finalized yet.
parameters:
- - name: app_name
+ - name: service_name
in: path
- description: Application name
+ description: Service name
required: true
type: string
responses:
204:
description: Destroy was successful
404:
- description: Application does not exist
+ description: Service does not exist
default:
description: Unexpected error
schema:
- $ref: '#/definitions/ApplicationStatus'
+ $ref: '#/definitions/ServiceStatus'
definitions:
- Application:
- description: An Application resource has the following attributes.
+ Service:
+ description: a service resource has the following attributes.
required:
- name
properties:
name:
type: string
- description: A unique application name. If Registry DNS is enabled, the max length is 63 characters.
+ description: A unique service name. If Registry DNS is enabled, the max length is 63 characters.
id:
type: string
- description: A unique application id.
+ description: A unique service id.
artifact:
- description: Artifact of single-component applications.
+ description: Artifact of single-component service.
$ref: '#/definitions/Artifact'
resource:
- description: Resource of single-component applications or the global default for multi-component applications. Mandatory if it is a single-component application and if cpus and memory are not specified at the Application level.
+ description: Resource of single-component service or the global default for multi-component services. Mandatory if it is a single-component service and if cpus and memory are not specified at the Service level.
$ref: '#/definitions/Resource'
launch_command:
type: string
- description: The custom launch command of an application component (optional). If not specified for applications with docker images say, it will default to the default start command of the image. If there is a single component in this application, you can specify this without the need to have a 'components' section.
+ description: The custom launch command of a service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section.
launch_time:
type: string
format: date
- description: The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.
+ description: The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.
number_of_containers:
type: integer
format: int64
- description: Number of containers for each app-component in the application. Each app-component can further override this app-level global default.
+ description: Number of containers for each component in the service. Each component can further override this service-level global default.
number_of_running_containers:
type: integer
format: int64
- description: In get response this provides the total number of running containers for this application (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.
+ description: In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.
lifetime:
type: integer
format: int64
- description: Life time (in seconds) of the application from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.
+ description: Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.
placement_policy:
- description: Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the app owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to application owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.
+ description: Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.
$ref: '#/definitions/PlacementPolicy'
components:
- description: Components of an application.
+ description: Components of a service.
type: array
items:
$ref: '#/definitions/Component'
configuration:
- description: Config properties of an application. Configurations provided at the application/global level are available to all the components. Specific properties can be overridden at the component level.
+ description: Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.
$ref: '#/definitions/Configuration'
containers:
- description: Containers of a started application. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.
+ description: Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.
type: array
items:
$ref: '#/definitions/Container'
state:
- description: State of the application. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started application.
- $ref: '#/definitions/ApplicationState'
+ description: State of the service. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started service.
+ $ref: '#/definitions/ServiceState'
quicklinks:
type: object
- description: A blob of key-value pairs of quicklinks to be exported for an application.
+ description: A blob of key-value pairs of quicklinks to be exported for a service.
additionalProperties:
type: string
queue:
type: string
- description: The YARN queue that this application should be submitted to.
+ description: The YARN queue that this service should be submitted to.
Resource:
description:
- Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise.
+ Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise.
properties:
profile:
type: string
@@ -224,75 +224,75 @@ definitions:
type: string
description: Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.
PlacementPolicy:
- description: Placement policy of an instance of an application. This feature is in the works in YARN-4902.
+ description: Placement policy of an instance of a service. This feature is in the works in YARN-6592.
properties:
label:
type: string
- description: Assigns an app to a named partition of the cluster where the application desires to run (optional). If not specified all apps are submitted to a default label of the app owner. One or more labels can be setup for each application owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.
+ description: Assigns a service to a named partition of the cluster where the service desires to run (optional). If not specified all services are submitted to a default label of the service owner. One or more labels can be setup for each service owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.
Artifact:
- description: Artifact of an application component. If not specified, component will just run the bare launch command and no artifact will be localized.
+ description: Artifact of a service component. If not specified, component will just run the bare launch command and no artifact will be localized.
required:
- id
properties:
id:
type: string
- description: Artifact id. Examples are package location uri for tarball based apps, image name for docker, name of application, etc.
+ description: Artifact id. Examples are package location uri for tarball based services, image name for docker, name of service, etc.
type:
type: string
- description: Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For APPLICATION type, the application specified will be read and its components will be added into this application. The original component with artifact type APPLICATION will be removed (any properties specified in the original component will be ignored).
+ description: Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For SERVICE type, the service specified will be read and its components will be added into this service. The original component with artifact type SERVICE will be removed (any properties specified in the original component will be ignored).
enum:
- DOCKER
- TARBALL
- - APPLICATION
+ - SERVICE
default: DOCKER
uri:
type: string
description: Artifact location to support multiple artifact stores (optional).
Component:
- description: One or more components of the application. If the application is HBase say, then the component can be a simple role like master or regionserver. If the application is a complex business webapp then a component can be other applications say Kafka or Storm. Thereby it opens up the support for complex and nested applications.
+ description: One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.
required:
- name
properties:
name:
type: string
- description: Name of the application component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.
+ description: Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.
dependencies:
type: array
items:
type: string
- description: An array of application components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an application should be represented as a DAG.
+ description: An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of a service should be represented as a DAG.
readiness_check:
- description: Readiness check for this app-component.
+ description: Readiness check for this component.
$ref: '#/definitions/ReadinessCheck'
artifact:
- description: Artifact of the component (optional). If not specified, the application level global artifact takes effect.
+ description: Artifact of the component (optional). If not specified, the service level global artifact takes effect.
$ref: '#/definitions/Artifact'
launch_command:
type: string
description: The custom launch command of this component (optional for DOCKER component, required otherwise). When specified at the component level, it overrides the value specified at the global level (if any).
resource:
- description: Resource of this component (optional). If not specified, the application level global resource takes effect.
+ description: Resource of this component (optional). If not specified, the service level global resource takes effect.
$ref: '#/definitions/Resource'
number_of_containers:
type: integer
format: int64
- description: Number of containers for this app-component (optional). If not specified, the application level global number_of_containers takes effect.
+ description: Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.
run_privileged_container:
type: boolean
description: Run all containers of this component in privileged mode (YARN-4262).
placement_policy:
- description: Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the app level placement_policy takes effect. Refer to the description at the global level for more details.
+ description: Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.
$ref: '#/definitions/PlacementPolicy'
configuration:
- description: Config properties for this app-component.
+ description: Config properties for this component.
$ref: '#/definitions/Configuration'
quicklinks:
type: array
items:
type: string
- description: A list of quicklink keys defined at the application level, and to be resolved by this component.
+ description: A list of quicklink keys defined at the service level, and to be resolved by this component.
ReadinessCheck:
- description: A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every application is different. Hence the need for a simple interface, with scope to support advanced usecases.
+ description: A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.
required:
- type
properties:
@@ -308,28 +308,28 @@ definitions:
additionalProperties:
type: string
artifact:
- description: Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits application owners who can run applications without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET
+ description: Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET
$ref: '#/definitions/Artifact'
Configuration:
- description: Set of configuration properties that can be injected into the application components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.
+ description: Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.
properties:
properties:
type: object
- description: A blob of key-value pairs of common application properties.
+ description: A blob of key-value pairs of common service properties.
additionalProperties:
type: string
env:
type: object
- description: A blob of key-value pairs which will be appended to the default system properties and handed off to the application at start time. All placeholder references to properties will be substituted before injection.
+ description: A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.
additionalProperties:
type: string
files:
- description: Array of list of files that needs to be created and made available as volumes in the application component containers.
+ description: Array of list of files that needs to be created and made available as volumes in the service component containers.
type: array
items:
$ref: '#/definitions/ConfigFile'
ConfigFile:
- description: A config file that needs to be created and made available as a volume in an application component container.
+ description: A config file that needs to be created and made available as a volume in a service component container.
properties:
type:
type: string
@@ -352,11 +352,11 @@ definitions:
type: object
description: A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.
Container:
- description: An instance of a running application container.
+ description: An instance of a running service container.
properties:
id:
type: string
- description: Unique container id of a running application, e.g. container_e3751_1458061340047_0008_01_000002.
+ description: Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002.
launch_time:
type: string
format: date
@@ -371,7 +371,7 @@ definitions:
type: string
description: The bare node or host in which the container is running, e.g. cn008.example.com.
state:
- description: State of the container of an application.
+ description: State of the container of a service.
$ref: '#/definitions/ContainerState'
component_name:
type: string
@@ -385,12 +385,12 @@ definitions:
privileged_container:
type: boolean
description: Container running in privileged mode or not.
- ApplicationState:
- description: The current state of an application.
+ ServiceState:
+ description: The current state of a service.
properties:
state:
type: string
- description: enum of the state of the application
+ description: enum of the state of the service
enum:
- ACCEPTED
- STARTED
@@ -398,25 +398,26 @@ definitions:
- STOPPED
- FAILED
ContainerState:
- description: The current state of the container of an application.
+ description: The current state of the container of a service.
properties:
state:
type: string
description: enum of the state of the container
enum:
- INIT
+ - STARTED
- READY
- ApplicationStatus:
- description: The current status of a submitted application, returned as a response to the GET API.
+ ServiceStatus:
+ description: The current status of a submitted service, returned as a response to the GET API.
properties:
diagnostics:
type: string
- description: Diagnostic information (if any) for the reason of the current state of the application. It typically has a non-null value, if the application is in a non-running state.
+ description: Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state.
state:
- description: Application state.
- $ref: '#/definitions/ApplicationState'
+ description: Service state.
+ $ref: '#/definitions/ServiceState'
code:
type: integer
format: int32
- description: An error code specific to a scenario which app owners should be able to use to understand the failure in addition to the diagnostic information.
+ description: An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh
deleted file mode 100644
index 9f15b7e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-export SLIDER_VERSION=${project.version}
-export HDP_VERSION=${HDP_VERSION:-$SLIDER_VERSION}
-export SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
-export LIB_PARENT_DIR=`dirname $SCRIPT_DIR`
-export JAVA_HOME=${JAVA_HOME:-/usr/jdk64/jdk1.8.0_40}
-export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
-export REST_SERVICE_PORT=${REST_SERVICE_PORT:-9191}
-export APP_RUNAS_USER=${APP_RUNAS_USER:-root}
-export REST_SERVICE_LOG_DIR=${REST_SERVICE_LOG_DIR:-/tmp/}
-export JAVA_OPTS="-Xms256m -Xmx1024m -XX:+PrintGC -Xloggc:$REST_SERVICE_LOG_DIR/gc.log"
-$JAVA_HOME/bin/java $JAVA_OPTS -cp .:$HADOOP_CONF_DIR:$LIB_PARENT_DIR/services-api/*:$LIB_PARENT_DIR/slider/* -DREST_SERVICE_LOG_DIR=$REST_SERVICE_LOG_DIR -Dlog4j.configuration=log4j-server.properties -Dslider.libdir=$LIB_PARENT_DIR/slider org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp 1>>$REST_SERVICE_LOG_DIR/restservice-out.log 2>&1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/dev-support/findbugs-exclude.xml
new file mode 100644
index 0000000..2814cca
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/dev-support/findbugs-exclude.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+ <Match>
+ <Package name="org.apache.hadoop.yarn.proto" />
+ </Match>
+ <Match>
+ <class name="org.apache.hadoop.yarn.service.utils.ServiceApiUtil" />
+ <Bug pattern="MS_CANNOT_BE_FINAL" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.service.utils.JsonSerDeser" />
+ <Bug pattern="OBL_UNSATISFIED_OBLIGATION" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.service.utils.JsonSerDeser" />
+ <Bug pattern="UI_INHERITANCE_UNSAFE_GETRESOURCE" />
+ </Match>
+ <Match>
+ <Package name="org.apache.hadoop.yarn.service.client.params"/>
+ <Bug pattern="UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD"/>
+ </Match>
+ <Match>
+ <Package name="org.apache.hadoop.yarn.service.client.params"/>
+ <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"/>
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.service.client.ServiceClient"/>
+ <Field name="registryClient" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC"/>
+ </Match>
+
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf/yarnservice-log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf/yarnservice-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf/yarnservice-log4j.properties
new file mode 100644
index 0000000..58c8e27
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf/yarnservice-log4j.properties
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is the log4j configuration for Slider Application Master
+
+# Log rotation based on size (256MB) with a max of 20 backup files
+log4j.rootLogger=INFO, amlog
+log4j.threshhold=ALL
+log4j.appender.amlog=org.apache.log4j.RollingFileAppender
+log4j.appender.amlog.layout=org.apache.log4j.PatternLayout
+log4j.appender.amlog.File=${LOG_DIR}/serviceam.log
+log4j.appender.amlog.MaxFileSize=256MB
+log4j.appender.amlog.MaxBackupIndex=20
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# configure stderr
+# set the conversion pattern of stderr
+# Print the date in ISO 8601 format
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+
+# for debugging yarn-service framework
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.curator.framework.state=ERROR
+log4j.logger.org.apache.curator.framework.imps=WARN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
new file mode 100644
index 0000000..1f8a408
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -0,0 +1,408 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-services</artifactId>
+ <version>3.0.0-beta1-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-yarn-services-core</artifactId>
+ <packaging>jar</packaging>
+ <name>Apache Hadoop YARN Services Core</name>
+
+ <properties>
+ <!-- Needed for generating FindBugs warnings using parent pom -->
+ <yarn.basedir>${project.parent.basedir}</yarn.basedir>
+ </properties>
+
+ <build>
+ <!-- resources are filtered for dynamic updates. This gets build info in-->
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+
+ <plugins>
+ <plugin>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <executions>
+ <execution>
+ <id>compile-protoc</id>
+ <goals>
+ <goal>protoc</goal>
+ </goals>
+ <configuration>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <protocCommand>${protoc.path}</protocCommand>
+ <imports>
+ <param>${basedir}/src/main/proto</param>
+ </imports>
+ <source>
+ <directory>${basedir}/src/main/proto</directory>
+ <includes>
+ <include>ClientAMProtocol.proto</include>
+ </includes>
+ </source>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <environmentVariables>
+ <JAVA_HOME>${java.home}</JAVA_HOME>
+ </environmentVariables>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>com.beust</groupId>
+ <artifactId>jcommander</artifactId>
+ <version>1.30</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-xc</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-registry</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-digester</groupId>
+ <artifactId>commons-digester</artifactId>
+ <version>1.8</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-servlets</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+
+ <!-- ======================================================== -->
+ <!-- service registry -->
+ <!-- ======================================================== -->
+
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </dependency>
+
+ <!-- ======================================================== -->
+ <!-- Jersey and webapp support -->
+ <!-- ======================================================== -->
+
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-servlet</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-guice</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <version>3.1</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.objenesis</groupId>
+ <artifactId>objenesis</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-easymock</artifactId>
+ <version>1.6.5</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>1.6.5</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.javassist</groupId>
+ <artifactId>javassist</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.objenesis</groupId>
+ <artifactId>objenesis</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jettison</groupId>
+ <artifactId>jettison</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>1.16</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.swagger</groupId>
+ <artifactId>swagger-annotations</artifactId>
+ <version>1.5.4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+
+ <profiles>
+ <profile>
+ <id>dist</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-assemblies</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>dist</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <descriptorRefs>
+ <descriptorRef>hadoop-yarn-services-dist</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ </profiles>
+
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
new file mode 100644
index 0000000..516d23d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+
+import java.io.IOException;
+
+public interface ClientAMProtocol {
+ FlexComponentsResponseProto flexComponents(FlexComponentsRequestProto request)
+ throws IOException, YarnException;
+
+ GetStatusResponseProto getStatus(GetStatusRequestProto requestProto)
+ throws IOException, YarnException;
+
+ StopResponseProto stop(StopRequestProto requestProto)
+ throws IOException, YarnException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
new file mode 100644
index 0000000..8e4c34d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
+import org.apache.hadoop.yarn.service.component.ComponentEvent;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX;
+
+public class ClientAMService extends AbstractService
+ implements ClientAMProtocol {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ClientAMService.class);
+
+ private ServiceContext context;
+ private Server server;
+
+ private InetSocketAddress bindAddress;
+
+ public ClientAMService(ServiceContext context) {
+ super("Client AM Service");
+ this.context = context;
+ }
+
+ @Override protected void serviceStart() throws Exception {
+ Configuration conf = getConfig();
+ YarnRPC rpc = YarnRPC.create(conf);
+ InetSocketAddress address = new InetSocketAddress(0);
+ server = rpc.getServer(ClientAMProtocol.class, this, address, conf,
+ context.secretManager, 1);
+ server.start();
+
+ String nodeHostString =
+ System.getenv(ApplicationConstants.Environment.NM_HOST.name());
+
+ bindAddress = NetUtils.createSocketAddrForHost(nodeHostString,
+ server.getListenerAddress().getPort());
+
+ LOG.info("Instantiated ClientAMService at " + bindAddress);
+ super.serviceStart();
+ }
+
+ @Override protected void serviceStop() throws Exception {
+ if (server != null) {
+ server.stop();
+ }
+ super.serviceStop();
+ }
+
+ @Override public FlexComponentsResponseProto flexComponents(
+ FlexComponentsRequestProto request) throws IOException {
+ if (!request.getComponentsList().isEmpty()) {
+ for (ComponentCountProto component : request.getComponentsList()) {
+ ComponentEvent event = new ComponentEvent(component.getName(), FLEX)
+ .setDesired(component.getNumberOfContainers());
+ context.scheduler.getDispatcher().getEventHandler().handle(event);
+ LOG.info("Flexing component {} to {}", component.getName(),
+ component.getNumberOfContainers());
+ }
+ }
+ return FlexComponentsResponseProto.newBuilder().build();
+ }
+
+ @Override
+ public GetStatusResponseProto getStatus(GetStatusRequestProto request)
+ throws IOException, YarnException {
+ String stat = ServiceApiUtil.jsonSerDeser.toJson(context.service);
+ return GetStatusResponseProto.newBuilder().setStatus(stat).build();
+ }
+
+ @Override
+ public StopResponseProto stop(StopRequestProto requestProto)
+ throws IOException, YarnException {
+ LOG.info("Stop the service.");
+ // Stop the service in 2 seconds delay to make sure this rpc call is completed.
+ // shutdown hook will be executed which will stop AM gracefully.
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(2000);
+ ExitUtil.terminate(0);
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted while stopping", e);
+ }
+ }
+ };
+ thread.start();
+ return StopResponseProto.newBuilder().build();
+ }
+
+ public InetSocketAddress getBindAddress() {
+ return bindAddress;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
new file mode 100644
index 0000000..4743f28
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.yarn.service.component.Component;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.NODE_BLACKLIST_THRESHOLD;
+
+/**
+ * This tracks the container failures per node. If the failure counter exceeds
+ * the maxFailurePerNode limit, it'll blacklist that node.
+ *
+ */
+public class ContainerFailureTracker {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerFailureTracker.class);
+
+ // Host -> num container failures
+ private Map<String, Integer> failureCountPerNode = new HashMap<>();
+ private Set<String> blackListedNodes = new HashSet<>();
+ private ServiceContext context;
+ private int maxFailurePerNode;
+ private Component component;
+
+ public ContainerFailureTracker(ServiceContext context, Component component) {
+ this.context = context;
+ this.component = component;
+ maxFailurePerNode = component.getComponentSpec().getConfiguration()
+ .getPropertyInt(NODE_BLACKLIST_THRESHOLD, 3);
+ }
+
+
+ public synchronized void incNodeFailure(String host) {
+ int num = 0;
+ if (failureCountPerNode.containsKey(host)) {
+ num = failureCountPerNode.get(host);
+ }
+ num++;
+ failureCountPerNode.put(host, num);
+
+ // black list the node if exceed max failure
+ if (num > maxFailurePerNode && !blackListedNodes.contains(host)) {
+ List<String> blacklists = new ArrayList<>();
+ blacklists.add(host);
+ blackListedNodes.add(host);
+ context.scheduler.getAmRMClient().updateBlacklist(blacklists, null);
+ LOG.info("[COMPONENT {}]: Failed {} times on this host, blacklisted {}."
+ + " Current list of blacklisted nodes: {}",
+ component.getName(), num, host, blackListedNodes);
+ }
+ }
+
+ public synchronized void resetContainerFailures() {
+ // reset container failure counter per node
+ failureCountPerNode.clear();
+ context.scheduler.getAmRMClient()
+ .updateBlacklist(null, new ArrayList<>(blackListedNodes));
+ LOG.info("[COMPONENT {}]: Clearing blacklisted nodes {} ",
+ component.getName(), blackListedNodes);
+ blackListedNodes.clear();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
new file mode 100644
index 0000000..94dbc6e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import com.google.common.cache.LoadingCache;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+
+public class ServiceContext {
+ public Service service = null;
+ public SliderFileSystem fs;
+ public String serviceHdfsDir = "";
+ public ApplicationAttemptId attemptId;
+ public LoadingCache<ConfigFile, Object> configCache;
+ public ServiceScheduler scheduler;
+ public ClientToAMTokenSecretManager secretManager;
+ public ClientAMService clientAMService;
+
+ public ServiceContext() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
new file mode 100644
index 0000000..e700961
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
+import org.apache.hadoop.yarn.service.client.params.SliderAMArgs;
+import org.apache.hadoop.yarn.service.monitor.ServiceMonitor;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class ServiceMaster extends CompositeService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceMaster.class);
+
+ private static SliderAMArgs amArgs;
+ protected ServiceContext context;
+
+ public ServiceMaster(String name) {
+ super(name);
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ //TODO Deprecate slider conf, make sure works with yarn conf
+ printSystemEnv();
+ if (UserGroupInformation.isSecurityEnabled()) {
+ UserGroupInformation.setConfiguration(conf);
+ }
+ LOG.info("Login user is {}", UserGroupInformation.getLoginUser());
+
+ context = new ServiceContext();
+ Path appDir = getAppDir();
+ context.serviceHdfsDir = appDir.toString();
+ SliderFileSystem fs = new SliderFileSystem(conf);
+ context.fs = fs;
+ fs.setAppDir(appDir);
+ loadApplicationJson(context, fs);
+
+ ContainerId amContainerId = getAMContainerId();
+
+ ApplicationAttemptId attemptId = amContainerId.getApplicationAttemptId();
+ LOG.info("Service AppAttemptId: " + attemptId);
+ context.attemptId = attemptId;
+
+ // configure AM to wait forever for RM
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, -1);
+ conf.unset(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS);
+
+ DefaultMetricsSystem.initialize("ServiceAppMaster");
+
+ context.secretManager = new ClientToAMTokenSecretManager(attemptId, null);
+ ClientAMService clientAMService = new ClientAMService(context);
+ context.clientAMService = clientAMService;
+ addService(clientAMService);
+
+ ServiceScheduler scheduler = createServiceScheduler(context);
+ addService(scheduler);
+ context.scheduler = scheduler;
+
+ ServiceMonitor monitor = new ServiceMonitor("Service Monitor", context);
+ addService(monitor);
+
+ super.serviceInit(conf);
+ }
+
+ protected ContainerId getAMContainerId() throws BadClusterStateException {
+ return ContainerId.fromString(SliderUtils.mandatoryEnvVariable(
+ ApplicationConstants.Environment.CONTAINER_ID.name()));
+ }
+
+ protected Path getAppDir() {
+ return new Path(amArgs.getAppDefPath()).getParent();
+ }
+
+ protected ServiceScheduler createServiceScheduler(ServiceContext context)
+ throws IOException, YarnException {
+ return new ServiceScheduler(context);
+ }
+
+ protected void loadApplicationJson(ServiceContext context,
+ SliderFileSystem fs) throws IOException {
+ context.service = ServiceApiUtil
+ .loadServiceFrom(fs, new Path(amArgs.getAppDefPath()));
+ LOG.info(context.service.toString());
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ LOG.info("Stopping app master");
+ super.serviceStop();
+ }
+
+ private void printSystemEnv() {
+ for (Map.Entry<String, String> envs : System.getenv().entrySet()) {
+ LOG.info("{} = {}", envs.getKey(), envs.getValue());
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
+ StringUtils.startupShutdownMessage(ServiceMaster.class, args, LOG);
+ amArgs = new SliderAMArgs(args);
+ amArgs.parse();
+ try {
+ ServiceMaster serviceMaster = new ServiceMaster("Service Master");
+ ShutdownHookManager.get()
+ .addShutdownHook(new CompositeServiceShutdownHook(serviceMaster), 30);
+ YarnConfiguration conf = new YarnConfiguration();
+ new GenericOptionsParser(conf, args);
+ serviceMaster.init(conf);
+ serviceMaster.start();
+ } catch (Throwable t) {
+ LOG.error("Error starting service master", t);
+ ExitUtil.terminate(1, "Error starting service master");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java
new file mode 100644
index 0000000..9fc886e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+
+import static org.apache.hadoop.metrics2.lib.Interns.info;
+
+@Metrics(context = "yarn-native-service")
+public class ServiceMetrics implements MetricsSource {
+
+ @Metric("containers requested")
+ public MutableGaugeInt containersRequested;
+
+ @Metric("anti-affinity containers pending")
+ public MutableGaugeInt pendingAAContainers;
+
+ @Metric("containers running")
+ public MutableGaugeInt containersRunning;
+
+ @Metric("containers ready")
+ public MutableGaugeInt containersReady;
+
+ @Metric("containers desired")
+ public MutableGaugeInt containersDesired;
+
+ @Metric("containers succeeded")
+ public MutableGaugeInt containersSucceeded;
+
+ @Metric("containers failed")
+ public MutableGaugeInt containersFailed;
+
+ @Metric("containers preempted")
+ public MutableGaugeInt containersPreempted;
+
+ @Metric("containers surplus")
+ public MutableGaugeInt surplusContainers;
+
+ @Metric("containers failed due to disk failure")
+ public MutableGaugeInt containersDiskFailure;
+
+ protected final MetricsRegistry registry;
+
+ public ServiceMetrics(MetricsInfo metricsInfo) {
+ registry = new MetricsRegistry(metricsInfo);
+ }
+
+ @Override
+ public void getMetrics(MetricsCollector collector, boolean all) {
+ registry.snapshot(collector.addRecord(registry.info()), all);
+ }
+
+ public static ServiceMetrics register(String name, String description) {
+ ServiceMetrics metrics = new ServiceMetrics(info(name, description));
+ DefaultMetricsSystem.instance().register(name, description, metrics);
+ return metrics;
+ }
+
+ public void tag(String name, String description, String value) {
+ registry.tag(name, description, value);
+ }
+
+ @Override public String toString() {
+ return "ServiceMetrics{"
+ + "containersRequested=" + containersRequested.value()
+ + ", pendingAAContainers=" + pendingAAContainers.value()
+ + ", containersRunning=" + containersRunning.value()
+ + ", containersDesired=" + containersDesired.value()
+ + ", containersSucceeded=" + containersSucceeded.value()
+ + ", containersFailed=" + containersFailed.value()
+ + ", containersPreempted=" + containersPreempted.value()
+ + ", surplusContainers=" + surplusContainers.value() + '}';
+ }
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
deleted file mode 100644
index 8310530..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
-import static org.easymock.EasyMock.*;
-
-/**
- * Test loading example resources.
- */
-@RunWith(value = Parameterized.class)
-public class TestLoadExampleAppJson extends Assert {
- private String resource;
-
- public TestLoadExampleAppJson(String resource) {
- this.resource = resource;
- }
-
- @Parameterized.Parameters
- public static Collection<String[]> filenames() {
- String[][] stringArray = new String[ExampleAppJson
- .ALL_EXAMPLE_RESOURCES.size()][1];
- int i = 0;
- for (String s : ExampleAppJson.ALL_EXAMPLE_RESOURCES) {
- stringArray[i++][0] = s;
- }
- return Arrays.asList(stringArray);
- }
-
- @Test
- public void testLoadResource() throws Throwable {
- try {
- Application application = JSON_SER_DESER.fromResource(resource);
-
- SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
- FileSystem mockFs = createNiceMock(FileSystem.class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- replay(sfs, mockFs);
-
- ServiceApiUtil.validateAndResolveApplication(application, sfs,
- new YarnConfiguration());
- } catch (Exception e) {
- throw new Exception("exception loading " + resource + ":" + e.toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
deleted file mode 100644
index 98c78d3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Test cluster name validation.
- */
-public class TestValidateServiceNames {
-
- void assertValidName(String name) {
- boolean valid = SliderUtils.isClusternameValid(name);
- Assert.assertTrue("Clustername '" + name + "' mistakenly declared invalid",
- valid);
- }
-
- void assertInvalidName(String name) {
- boolean valid = SliderUtils.isClusternameValid(name);
- Assert.assertFalse("Clustername '\" + name + \"' mistakenly declared valid",
- valid);
- }
-
- void assertInvalid(List<String> names) {
- for (String name : names) {
- assertInvalidName(name);
- }
- }
-
- void assertValid(List<String> names) {
- for (String name : names) {
- assertValidName(name);
- }
- }
-
- @Test
- public void testEmptyName() throws Throwable {
- assertInvalidName("");
- }
-
- @Test
- public void testSpaceName() throws Throwable {
- assertInvalidName(" ");
- }
-
-
- @Test
- public void testLeadingHyphen() throws Throwable {
- assertInvalidName("-hyphen");
- }
-
- @Test
- public void testTitleLetters() throws Throwable {
- assertInvalidName("Title");
- }
-
- @Test
- public void testCapitalLetters() throws Throwable {
- assertInvalidName("UPPER-CASE-CLUSTER");
- }
-
- @Test
- public void testInnerBraced() throws Throwable {
- assertInvalidName("a[a");
- }
-
- @Test
- public void testLeadingBrace() throws Throwable {
- assertInvalidName("[");
- }
-
- @Test
- public void testNonalphaLeadingChars() throws Throwable {
- assertInvalid(Arrays.asList(
- "[a", "#", "@", "=", "*", "."
- ));
- }
-
- @Test
- public void testNonalphaInnerChars() throws Throwable {
- assertInvalid(Arrays.asList(
- "a[a", "b#", "c@", "d=", "e*", "f.", "g ", "h i"
- ));
- }
-
- @Test
- public void testClusterValid() throws Throwable {
- assertValidName("cluster");
- }
-
- @Test
- public void testValidNames() throws Throwable {
- assertValid(Arrays.asList(
- "cluster",
- "cluster1",
- "very-very-very-long-cluster-name",
- "c1234567890"
- ));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
deleted file mode 100644
index 5b24a1d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.providers;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.easymock.EasyMock.*;
-
-/**
- * Test the AbstractClientProvider shared methods.
- */
-public class TestAbstractClientProvider {
- private static final String EXCEPTION_PREFIX = "Should have thrown " +
- "exception: ";
- private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
- "exception: ";
-
- private static class ClientProvider extends AbstractClientProvider {
- @Override
- public void validateArtifact(Artifact artifact, FileSystem fileSystem)
- throws IOException {
- }
-
- @Override
- protected void validateConfigFile(ConfigFile configFile,
- FileSystem fileSystem) throws IOException {
- }
- }
-
- @Test
- public void testConfigFiles() throws IOException {
- ClientProvider clientProvider = new ClientProvider();
- FileSystem mockFs = createNiceMock(FileSystem.class);
- expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes();
- replay(mockFs);
-
- ConfigFile configFile = new ConfigFile();
- List<ConfigFile> configFiles = new ArrayList<>();
- configFiles.add(configFile);
-
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- Assert.fail(EXCEPTION_PREFIX + "null file type");
- } catch (IllegalArgumentException e) {
- }
-
- configFile.setType(ConfigFile.TypeEnum.TEMPLATE);
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- Assert.fail(EXCEPTION_PREFIX + "empty src_file for type template");
- } catch (IllegalArgumentException e) {
- }
-
- configFile.setSrcFile("srcfile");
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- Assert.fail(EXCEPTION_PREFIX + "empty dest file");
- } catch (IllegalArgumentException e) {
- }
-
- configFile.setDestFile("destfile");
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- configFile = new ConfigFile();
- configFile.setType(ConfigFile.TypeEnum.JSON);
- configFile.setSrcFile(null);
- configFile.setDestFile("path/destfile2");
- configFiles.add(configFile);
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- Assert.fail(EXCEPTION_PREFIX + "dest file with multiple path elements");
- } catch (IllegalArgumentException e) {
- }
-
- configFile.setDestFile("/path/destfile2");
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- configFile.setDestFile("destfile");
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- Assert.fail(EXCEPTION_PREFIX + "duplicate dest file");
- } catch (IllegalArgumentException e) {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
deleted file mode 100644
index 489578d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.providers;
-
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.Artifact.TypeEnum;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultClientProvider;
-import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
-import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderService;
-import org.apache.hadoop.yarn.service.provider.docker.DockerClientProvider;
-import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
-import org.apache.hadoop.yarn.service.provider.docker.DockerProviderService;
-import org.apache.hadoop.yarn.service.provider.tarball.TarballClientProvider;
-import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
-import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderService;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test provider factories.
- */
-public class TestProviderFactory {
- @Test
- public void testDockerFactory() throws Throwable {
- ProviderFactory factory = ProviderFactory
- .createSliderProviderFactory(new Artifact().type(TypeEnum.DOCKER));
- assertTrue(factory instanceof DockerProviderFactory);
- assertTrue(factory.createClientProvider() instanceof DockerClientProvider);
- assertTrue(factory.createServerProvider() instanceof DockerProviderService);
- assertTrue(ProviderFactory.getProviderService(new Artifact()
- .type(TypeEnum.DOCKER)) instanceof DockerProviderService);
- }
-
- @Test
- public void testTarballFactory() throws Throwable {
- ProviderFactory factory = ProviderFactory
- .createSliderProviderFactory(new Artifact().type(TypeEnum.TARBALL));
- assertTrue(factory instanceof TarballProviderFactory);
- assertTrue(factory.createClientProvider() instanceof TarballClientProvider);
- assertTrue(factory.createServerProvider() instanceof
- TarballProviderService);
- assertTrue(ProviderFactory.getProviderService(new Artifact()
- .type(TypeEnum.TARBALL)) instanceof TarballProviderService);
- }
-
- @Test
- public void testDefaultFactory() throws Throwable {
- ProviderFactory factory = ProviderFactory
- .createSliderProviderFactory(null);
- assertTrue(factory instanceof DefaultProviderFactory);
- assertTrue(factory.createClientProvider() instanceof DefaultClientProvider);
- assertTrue(factory.createServerProvider() instanceof DefaultProviderService);
- assertTrue(ProviderFactory.getProviderService(null) instanceof
- DefaultProviderService);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/servicemonitor/TestServiceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/servicemonitor/TestServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/servicemonitor/TestServiceMonitor.java
deleted file mode 100644
index 6f5653f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/servicemonitor/TestServiceMonitor.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.hadoop.yarn.service.servicemonitor;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.MockServiceAM;
-import org.apache.hadoop.yarn.service.ServiceTestUtils;
-
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-
-public class TestServiceMonitor extends ServiceTestUtils {
-
- private File basedir;
- YarnConfiguration conf = new YarnConfiguration();
-
- @Before
- public void setup() throws Exception {
- basedir = new File("target", "apps");
- if (basedir.exists()) {
- FileUtils.deleteDirectory(basedir);
- } else {
- basedir.mkdirs();
- }
- conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 2);
- }
-
- @After
- public void tearDown() throws IOException {
- if (basedir != null) {
- FileUtils.deleteDirectory(basedir);
- }
- }
-
- // Create compa with 1 container
- // Create compb with 1 container
- // Verify compb dependency satisfied
- // Increase compa to 2 containers
- // Verify compb dependency becomes unsatisfied.
- @Test
- public void testComponentDependency() throws Exception{
- ApplicationId applicationId = ApplicationId.newInstance(123456, 1);
- Application exampleApp = new Application();
- exampleApp.setId(applicationId.toString());
- exampleApp.setName("testComponentDependency");
- exampleApp.addComponent(createComponent("compa", 1, "sleep 1000"));
- Component compb = createComponent("compb", 1, "sleep 1000");
-
- // Let compb depends on compa;
- compb.setDependencies(Collections.singletonList("compa"));
- exampleApp.addComponent(compb);
-
- MockServiceAM am = new MockServiceAM(exampleApp);
- am.init(conf);
- am.start();
-
- // compa ready
- Assert.assertTrue(am.getComponent("compa").areDependenciesReady());
- //compb not ready
- Assert.assertFalse(am.getComponent("compb").areDependenciesReady());
-
- // feed 1 container to compa,
- am.feedContainerToComp(exampleApp, 1, "compa");
- // waiting for compb's dependencies are satisfied
- am.waitForDependenciesSatisfied("compb");
-
- // feed 1 container to compb
- am.feedContainerToComp(exampleApp, 2, "compb");
- am.flexComponent("compa", 2);
- am.waitForNumDesiredContainers("compa", 2);
-
- // compb dependencies not satisfied again.
- Assert.assertFalse(am.getComponent("compb").areDependenciesReady());
- am.stop();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
deleted file mode 100644
index a891df8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.timelineservice;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
-import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ApplicationState;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.ContainerState;
-import org.apache.hadoop.yarn.service.api.records.PlacementPolicy;
-import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceId;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Test class for ServiceTimelinePublisher.
- */
-public class TestServiceTimelinePublisher {
- private TimelineV2Client timelineClient;
- private Configuration config;
- private ServiceTimelinePublisher serviceTimelinePublisher;
- private static String SERVICE_NAME = "HBASE";
- private static String SERVICEID = "application_1490093646524_0005";
- private static String ARTIFACTID = "ARTIFACTID";
- private static String COMPONENT_NAME = "DEFAULT";
- private static String CONTAINER_ID =
- "container_e02_1490093646524_0005_01_000001";
- private static String CONTAINER_IP =
- "localhost";
- private static String CONTAINER_HOSTNAME =
- "cnl124-localhost.site";
- private static String CONTAINER_BAREHOST =
- "localhost.com";
-
- @Before
- public void setUp() throws Exception {
- config = new Configuration();
- config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
- timelineClient =
- new DummyTimelineClient(ApplicationId.fromString(SERVICEID));
- serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
- timelineClient.init(config);
- serviceTimelinePublisher.init(config);
- timelineClient.start();
- serviceTimelinePublisher.start();
- }
-
- @After
- public void tearDown() throws Exception {
- if (serviceTimelinePublisher != null) {
- serviceTimelinePublisher.stop();
- }
- if (timelineClient != null) {
- timelineClient.stop();
- }
- }
-
- @Test
- public void testServiceAttemptEntity() {
- Application application = createMockApplication();
- serviceTimelinePublisher
- .serviceAttemptRegistered(application, new YarnConfiguration());
-
- Collection<TimelineEntity> lastPublishedEntities =
- ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
- // 2 entities because during registration component also registered.
- assertEquals(2, lastPublishedEntities.size());
- for (TimelineEntity timelineEntity : lastPublishedEntities) {
- if (timelineEntity.getType() == ServiceTimelineEntityType.COMPONENT
- .toString()) {
- verifyComponentTimelineEntity(timelineEntity);
- } else {
- verifyServiceAttemptTimelineEntity(timelineEntity, null, true);
- }
- }
-
- ServiceContext context = new ServiceContext();
- context.attemptId = ApplicationAttemptId
- .newInstance(ApplicationId.fromString(application.getId()), 1);
- String exitDiags = "service killed";
- serviceTimelinePublisher.serviceAttemptUnregistered(context, exitDiags);
- lastPublishedEntities =
- ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
- for (TimelineEntity timelineEntity : lastPublishedEntities) {
- if (timelineEntity.getType() == ServiceTimelineEntityType.SERVICE_ATTEMPT
- .toString()) {
- verifyServiceAttemptTimelineEntity(timelineEntity, exitDiags,
- false);
- }
- }
- }
-
- @Test
- public void testComponentInstanceEntity() {
- Container container = new Container();
- container.id(CONTAINER_ID).ip(CONTAINER_IP).bareHost(CONTAINER_BAREHOST)
- .hostname(CONTAINER_HOSTNAME).state(ContainerState.RUNNING_BUT_UNREADY)
- .launchTime(new Date());
- ComponentInstanceId id = new ComponentInstanceId(0, COMPONENT_NAME);
- ComponentInstance instance = mock(ComponentInstance.class);
- when(instance.getCompName()).thenReturn(COMPONENT_NAME);
- when(instance.getCompInstanceName()).thenReturn("comp_instance_name");
- serviceTimelinePublisher.componentInstanceStarted(container,
- instance);
-
- Collection<TimelineEntity> lastPublishedEntities =
- ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
- assertEquals(1, lastPublishedEntities.size());
- TimelineEntity entity = lastPublishedEntities.iterator().next();
-
- assertEquals(1, entity.getEvents().size());
- assertEquals(CONTAINER_ID, entity.getId());
- assertEquals(CONTAINER_BAREHOST,
- entity.getInfo().get(ServiceTimelineMetricsConstants.BARE_HOST));
- assertEquals(COMPONENT_NAME,
- entity.getInfo().get(ServiceTimelineMetricsConstants.COMPONENT_NAME));
- assertEquals(ContainerState.RUNNING_BUT_UNREADY.toString(),
- entity.getInfo().get(ServiceTimelineMetricsConstants.STATE));
-
- // updated container state
- container.setState(ContainerState.READY);
- serviceTimelinePublisher.componentInstanceUpdated(container);
- lastPublishedEntities =
- ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
- assertEquals(1, lastPublishedEntities.size());
- entity = lastPublishedEntities.iterator().next();
- assertEquals(2, entity.getEvents().size());
- assertEquals(ContainerState.READY.toString(),
- entity.getInfo().get(ServiceTimelineMetricsConstants.STATE));
-
- }
-
- private void verifyServiceAttemptTimelineEntity(TimelineEntity timelineEntity,
- String message, boolean isRegistedEntity) {
- assertEquals(SERVICEID, timelineEntity.getId());
- assertEquals(SERVICE_NAME,
- timelineEntity.getInfo().get(ServiceTimelineMetricsConstants.NAME));
- if (isRegistedEntity) {
- assertEquals(ApplicationState.STARTED.toString(),
- timelineEntity.getInfo().get(ServiceTimelineMetricsConstants.STATE));
- assertEquals(ServiceTimelineEvent.SERVICE_ATTEMPT_REGISTERED.toString(),
- timelineEntity.getEvents().iterator().next().getId());
- } else {
- assertEquals("ENDED",
- timelineEntity.getInfo().get(ServiceTimelineMetricsConstants.STATE).toString());
- assertEquals(message, timelineEntity.getInfo()
- .get(ServiceTimelineMetricsConstants.DIAGNOSTICS_INFO));
- assertEquals(2, timelineEntity.getEvents().size());
- assertEquals(ServiceTimelineEvent.SERVICE_ATTEMPT_UNREGISTERED.toString(),
- timelineEntity.getEvents().iterator().next().getId());
- }
- }
-
- private void verifyComponentTimelineEntity(TimelineEntity entity) {
- Map<String, Object> info = entity.getInfo();
- assertEquals("DEFAULT", entity.getId());
- assertEquals(ARTIFACTID,
- info.get(ServiceTimelineMetricsConstants.ARTIFACT_ID));
- assertEquals("DOCKER",
- info.get(ServiceTimelineMetricsConstants.ARTIFACT_TYPE));
- assertEquals("medium",
- info.get(ServiceTimelineMetricsConstants.RESOURCE_PROFILE));
- assertEquals(1, info.get(ServiceTimelineMetricsConstants.RESOURCE_CPU));
- assertEquals("1024",
- info.get(ServiceTimelineMetricsConstants.RESOURCE_MEMORY));
- assertEquals("sleep 1",
- info.get(ServiceTimelineMetricsConstants.LAUNCH_COMMAND));
- assertEquals("false",
- info.get(ServiceTimelineMetricsConstants.RUN_PRIVILEGED_CONTAINER));
- assertEquals("label",
- info.get(ServiceTimelineMetricsConstants.PLACEMENT_POLICY));
- }
-
- private static Application createMockApplication() {
- Application application = mock(Application.class);
-
- when(application.getId()).thenReturn(SERVICEID);
- when(application.getLaunchTime()).thenReturn(new Date());
- when(application.getState()).thenReturn(ApplicationState.STARTED);
- when(application.getName()).thenReturn(SERVICE_NAME);
- when(application.getConfiguration()).thenReturn(
- new org.apache.hadoop.yarn.service.api.records.Configuration());
-
- Component component = mock(Component.class);
- Artifact artifact = new Artifact();
- artifact.setId(ARTIFACTID);
- Resource resource = new Resource();
- resource.setCpus(1);
- resource.setMemory(1024 + "");
- resource.setProfile("medium");
- when(component.getArtifact()).thenReturn(artifact);
- when(component.getName()).thenReturn(COMPONENT_NAME);
- when(component.getResource()).thenReturn(resource);
- when(component.getLaunchCommand()).thenReturn("sleep 1");
- PlacementPolicy placementPolicy = new PlacementPolicy();
- placementPolicy.setLabel("label");
- when(component.getPlacementPolicy()).thenReturn(placementPolicy);
- when(component.getConfiguration()).thenReturn(
- new org.apache.hadoop.yarn.service.api.records.Configuration());
- List<Component> components = new ArrayList<Component>();
- components.add(component);
-
- when(application.getComponents()).thenReturn(components);
- return application;
- }
-
- protected static class DummyTimelineClient extends TimelineV2ClientImpl {
- private Map<Identifier, TimelineEntity> lastPublishedEntities =
- new HashMap<>();
-
- public DummyTimelineClient(ApplicationId appId) {
- super(appId);
- }
-
- @Override
- public void putEntitiesAsync(TimelineEntity... entities)
- throws IOException, YarnException {
- putEntities(entities);
- }
-
- @Override
- public void putEntities(TimelineEntity... entities)
- throws IOException, YarnException {
- for (TimelineEntity timelineEntity : entities) {
- TimelineEntity entity =
- lastPublishedEntities.get(timelineEntity.getIdentifier());
- if (entity == null) {
- lastPublishedEntities.put(timelineEntity.getIdentifier(),
- timelineEntity);
- } else {
- entity.addMetrics(timelineEntity.getMetrics());
- entity.addEvents(timelineEntity.getEvents());
- entity.addInfo(timelineEntity.getInfo());
- entity.addConfigs(timelineEntity.getConfigs());
- entity.addRelatesToEntities(timelineEntity.getRelatesToEntities());
- entity
- .addIsRelatedToEntities(timelineEntity.getIsRelatedToEntities());
- }
- }
- }
-
- public Collection<TimelineEntity> getLastPublishedEntities() {
- return lastPublishedEntities.values();
- }
-
- public void reset() {
- lastPublishedEntities = null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/example-app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/example-app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/example-app.json
deleted file mode 100644
index 5dfbd64..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/example-app.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "name": "example-app",
- "components" :
- [
- {
- "name": "simple",
- "number_of_containers": 1,
- "launch_command": "sleep 2",
- "resource": {
- "cpus": 1,
- "memory": "128"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/log4j.properties
deleted file mode 100644
index 3adbaa4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,66 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# log4j configuration used during build and unit tests
-
-log4j.rootLogger=INFO,stdout
-log4j.threshhold=ALL
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
-
-log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
-log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
-log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
-#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
-
-log4j.logger.org.apache.slider=DEBUG
-log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
-log4j.logger.org.apache.hadoop.yarn.registry=DEBUG
-
-#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
-#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
-#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
-
-#crank back on some noise
-log4j.logger.org.apache.hadoop.ipc.CallQueueManager=WARN
-
-log4j.logger.org.apache.hadoop.util.Shell=ERROR
-log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-log4j.logger.org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager=FATAL
-org.apache.hadoop.security.authentication.server.AuthenticationFilter=WARN
-log4j.logger.org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner=WARN
-log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
-log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
-log4j.logger.org.apache.hadoop.hdfs=WARN
-log4j.logger.BlockStateChange=WARN
-
-log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
-log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
-log4j.logger.org.apache.zookeeper=WARN
-log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
-
-log4j.logger.org.apache.hadoop.yarn.factories.impl.pb.RpcServerFactoryPBImpl=WARN
-log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeResourceMonitorImpl=ERROR
-log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.security=WARN
-log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher=WARN
-log4j.logger.org.apache.hadoop.metrics2=ERROR
-log4j.logger.org.apache.hadoop.util.HostsFileReader=WARN
-log4j.logger.org.apache.hadoop.yarn.event.AsyncDispatcher=WARN
-log4j.logger.org.apache.hadoop.security.token.delegation=WARN
-log4j.logger.org.apache.hadoop.yarn.util.AbstractLivelinessMonitor=WARN
-log4j.logger.org.apache.hadoop.yarn.server.nodemanager.security=WARN
-log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMNMInfo=WARN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
deleted file mode 100644
index d7e2fd0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
+++ /dev/null
@@ -1,72 +0,0 @@
-{
- "name": "app-1",
- "lifetime": "3600",
- "launch_command": "sleep 3600",
- "configuration": {
- "properties": {
- "g1": "a",
- "g2": "b"
- },
- "files": [
- {
- "type": "PROPERTIES",
- "dest_file": "file1",
- "props": {
- "k1": "v1",
- "k2": "v2"
- }
- },
- {
- "type": "XML",
- "dest_file": "file2",
- "props": {
- "k3": "v3"
- }
- }
- ]
- },
- "resource": {
- "cpus": 1,
- "memory": "512"
- },
- "number_of_containers": 2,
- "components": [
- {
- "name": "simple",
- "configuration": {
- "files": [
- {
- "type": "PROPERTIES",
- "dest_file": "file1",
- "props": {
- "k1": "overridden"
- }
- }
- ]
- }
- },
- {
- "name": "master",
- "configuration": {
- "properties": {
- "name": "m",
- "g1": "overridden"
- }
- }
- },
- {
- "name": "worker",
- "resource": {
- "cpus": 1,
- "memory": "1024"
- },
- "configuration": {
- "properties": {
- "name": "worker",
- "g1": "overridden-by-worker",
- "timeout": "1000"
- }
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
deleted file mode 100644
index a163b33..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
- "name": "app-1",
- "lifetime": "3600",
- "launch_command": "sleep 3600",
- "configuration": {
- "properties": {
- "g1": "a",
- "g2": "b",
- "yarn.service.failure-count-reset.window": "60"
- }
- },
- "resource": {
- "cpus": 1,
- "memory": "512"
- },
- "number_of_containers": 2,
- "components": [
- {
- "name": "simple"
- },
- {
- "name": "master",
- "number_of_containers": 1,
- "configuration": {
- "properties": {
- "g1": "overridden",
- "g3": "will-be-overridden",
- "jvm.heapsize": "512M"
- }
- }
- },
- {
- "name": "worker",
- "number_of_containers": 5,
- "resource": {
- "cpus": 1,
- "memory": "1024"
- },
- "configuration": {
- "properties": {
- "g1": "overridden-by-worker",
- "jvm.heapsize": "512M"
- }
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json
deleted file mode 100644
index 73d4e7b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "default-app-1",
- "lifetime": "3600",
- "components" :
- [
- {
- "name": "sleep",
- "number_of_containers": 1,
- "launch_command": "sleep 3600",
- "resource": {
- "cpus": 2,
- "memory": "256"
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
deleted file mode 100644
index 1f9dfeb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "name": "external-0",
- "lifetime": "3600",
- "artifact": {
- "type": "APPLICATION",
- "id": "app-1"
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json
deleted file mode 100644
index 03ebce5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
- "name": "external-1",
- "lifetime": "3600",
- "components": [
- {
- "name": "simple",
- "artifact": {
- "type": "APPLICATION",
- "id": "app-1"
- }
- },
- {
- "name": "master",
- "configuration": {
- "properties": {
- "g3": "is-overridden"
- }
- }
- },
- {
- "name": "other",
- "launch_command": "sleep 3600",
- "number_of_containers": 2,
- "resource": {
- "cpus": 1,
- "memory": "512"
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json
deleted file mode 100644
index 9e61fba..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "name": "external-2",
- "lifetime": "3600",
- "components": [
- {
- "name": "ext",
- "artifact": {
- "type": "APPLICATION",
- "id": "external-1"
- }
- },
- {
- "name": "another",
- "launch_command": "sleep 3600",
- "number_of_containers": 1,
- "resource": {
- "cpus": 1,
- "memory": "512"
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/appConfig.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/appConfig.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/appConfig.json
deleted file mode 100644
index c87f77c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/appConfig.json
+++ /dev/null
@@ -1,42 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {},
- "global": {
- "am.config.generation": "true",
- "component.unique.names": "true",
-
- "export.app.monitor": "${COMPONENT1_HOST} : ${@//site/test-xml/xmlkey}",
- "export.other.key": "exportvalue",
-
- "docker.image": "docker.io/centos:centos6",
- "docker.startCommand": "sleep 600",
-
- "conf.test-json.type": "json",
- "conf.test-json.name": "/tmp/test.json",
- "conf.test-xml.type": "xml",
- "conf.test-xml.name": "/tmp/test.xml",
- "conf.test-properties.type": "properties",
- "conf.test-properties.name": "/tmp/test.xml",
- "conf.test-yaml.type": "yaml",
- "conf.test-yaml.name": "/tmp/test.yaml",
- "conf.test-env.type": "env",
- "conf.test-env.name": "/tmp/testenv",
- "conf.test-template.type": "template",
- "conf.test-template.name": "/tmp/test.template",
- "conf.test-hadoop-xml.type": "hadoop-xml",
- "conf.test-hadoop-xml.name": "/tmp/test-hadoop.xml",
-
- "site.test-json.jsonkey": "val1",
- "site.test-xml.xmlkey": "val2",
- "site.test-hadoop-xml.xmlkey": "val3",
- "site.test-properties.propkey": "val4",
- "site.test-yaml.yamlkey": "val5",
- "site.test-env.content": "test ${envkey1} {{envkey2}} content",
- "site.test-env.envkey1": "envval1",
- "site.test-env.envkey2": "envval2",
- "site.test-template.templatekey1": "templateval1",
- "site.test-template.templatekey2": "templateval2"
- },
- "components": {
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/resources.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/resources.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/resources.json
deleted file mode 100644
index 1b06224..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/resources.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {},
- "global": {},
- "components": {
- "slider-appmaster": {
- "yarn.memory": "384"
- },
- "COMPONENT": {
- "yarn.role.priority": "1",
- "yarn.component.instances": 2,
- "yarn.memory": "512",
- "yarn.vcores": "2"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/test.template
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/test.template b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/test.template
deleted file mode 100644
index 2922655..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/test.template
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-test ${templatekey1} {{templatekey2}} content
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/yarn-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/yarn-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/yarn-site.xml
deleted file mode 100644
index 266caa9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/yarn-site.xml
+++ /dev/null
@@ -1,19 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-<configuration>
- <!-- Dummy (invalid) config file to be overwriten by TestYarnNativeServices with MiniCluster configuration. -->
-</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
deleted file mode 100644
index 6a208d8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
- http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>hadoop-yarn-applications</artifactId>
- <groupId>org.apache.hadoop</groupId>
- <version>3.0.0-beta1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-slider</artifactId>
- <name>Apache Hadoop YARN Slider</name>
- <packaging>pom</packaging>
-
- <properties>
- <hadoop.common.build.dir>${basedir}/../../../../hadoop-common-project/hadoop-common/target</hadoop.common.build.dir>
- </properties>
-
- <!-- Do not add dependencies here, add them to the POM of the leaf module -->
-
- <modules>
- <module>hadoop-yarn-slider-core</module>
- </modules>
-</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index 0fc5ceb..4fb579c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -36,7 +36,7 @@
<modules>
<module>hadoop-yarn-applications-distributedshell</module>
<module>hadoop-yarn-applications-unmanaged-am-launcher</module>
- <module>hadoop-yarn-slider</module>
+ <module>hadoop-yarn-services</module>
<module>hadoop-yarn-services-api</module>
</modules>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
index 4a048af..fa54a09 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
@@ -1,3 +1,17 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
# YARN DNS Server
## Introduction
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
index e69de29..89fefe9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
@@ -0,0 +1,13 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index 5440738..e4e611b 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -75,7 +75,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
- <excludePackageNames>org.apache.hadoop.yarn.proto:org.apache.hadoop.yarn.federation.proto:org.apache.slider</excludePackageNames>
+ <excludePackageNames>org.apache.hadoop.yarn.proto:org.apache.hadoop.yarn.federation.proto:org.apache.hadoop.yarn.service</excludePackageNames>
</configuration>
</plugin>
</plugins>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[62/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
new file mode 100644
index 0000000..add2475
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.registry;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.registry.client.api.BindFlags;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join;
+
+/**
+ * Registry view for providers. This tracks where the service
+ * is registered, offers access to the record and other things.
+ */
+public class YarnRegistryViewForProviders {
+ private static final Log LOG =
+ LogFactory.getLog(YarnRegistryViewForProviders.class);
+
+ private final RegistryOperations registryOperations;
+ private final String user;
+ private final String sliderServiceClass;
+ private final String instanceName;
+ /**
+ * Record used where the service registered itself.
+ * Null until the service is registered
+ */
+ private ServiceRecord selfRegistration;
+
+ /**
+ * Path where record was registered
+ * Null until the service is registered
+ */
+ private String selfRegistrationPath;
+
+ public YarnRegistryViewForProviders(RegistryOperations registryOperations,
+ String user,
+ String sliderServiceClass,
+ String instanceName,
+ ApplicationAttemptId applicationAttemptId) {
+ Preconditions.checkArgument(registryOperations != null,
+ "null registry operations");
+ Preconditions.checkArgument(user != null, "null user");
+ Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass),
+ "unset service class");
+ Preconditions.checkArgument(SliderUtils.isSet(instanceName),
+ "instanceName");
+ Preconditions.checkArgument(applicationAttemptId != null,
+ "null applicationAttemptId");
+ this.registryOperations = registryOperations;
+ this.user = user;
+ this.sliderServiceClass = sliderServiceClass;
+ this.instanceName = instanceName;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+
+ private void setSelfRegistration(ServiceRecord selfRegistration) {
+ this.selfRegistration = selfRegistration;
+ }
+
+ /**
+ * Get the path to where the service has registered itself.
+ * Null until the service is registered
+ * @return the service registration path.
+ */
+ public String getSelfRegistrationPath() {
+ return selfRegistrationPath;
+ }
+
+ /**
+ * Get the absolute path to where the service has registered itself.
+ * This includes the base registry path
+ * Null until the service is registered
+ * @return the service registration path.
+ */
+ public String getAbsoluteSelfRegistrationPath() {
+ if (selfRegistrationPath == null) {
+ return null;
+ }
+ String root = registryOperations.getConfig().getTrimmed(
+ RegistryConstants.KEY_REGISTRY_ZK_ROOT,
+ RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
+ return RegistryPathUtils.join(root, selfRegistrationPath);
+ }
+
+ /**
+ * Add a component under the slider name/entry
+ * @param componentName component name
+ * @param record record to put
+ * @throws IOException
+ */
+ public void putComponent(String componentName,
+ ServiceRecord record) throws
+ IOException {
+ putComponent(sliderServiceClass, instanceName,
+ componentName,
+ record);
+ }
+
+ /**
+ * Add a component
+ * @param serviceClass service class to use under ~user
+ * @param componentName component name
+ * @param record record to put
+ * @throws IOException
+ */
+ public void putComponent(String serviceClass,
+ String serviceName,
+ String componentName,
+ ServiceRecord record) throws IOException {
+ String path = RegistryUtils.componentPath(
+ user, serviceClass, serviceName, componentName);
+ registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
+ registryOperations.bind(path, record, BindFlags.OVERWRITE);
+ }
+
+ /**
+ * Add a service under a path, optionally purging any history
+ * @param username user
+ * @param serviceClass service class to use under ~user
+ * @param serviceName name of the service
+ * @param record service record
+ * @param deleteTreeFirst perform recursive delete of the path first.
+ * @return the path the service was created at
+ * @throws IOException
+ */
+ public String putService(String username,
+ String serviceClass,
+ String serviceName,
+ ServiceRecord record,
+ boolean deleteTreeFirst) throws IOException {
+ String path = RegistryUtils.servicePath(
+ username, serviceClass, serviceName);
+ if (deleteTreeFirst) {
+ registryOperations.delete(path, true);
+ }
+ registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
+ registryOperations.bind(path, record, BindFlags.OVERWRITE);
+ return path;
+ }
+
+ /**
+ * Add a service under a path for the current user
+ * @param record service record
+ * @param deleteTreeFirst perform recursive delete of the path first
+ * @return the path the service was created at
+ * @throws IOException
+ */
+ public String registerSelf(
+ ServiceRecord record,
+ boolean deleteTreeFirst) throws IOException {
+ selfRegistrationPath =
+ putService(user, sliderServiceClass, instanceName, record, deleteTreeFirst);
+ setSelfRegistration(record);
+ return selfRegistrationPath;
+ }
+
+ /**
+ * Delete a component
+ * @param containerId component name
+ * @throws IOException
+ */
+ public void deleteComponent(ComponentInstanceId instanceId,
+ String containerId) throws IOException {
+ String path = RegistryUtils.componentPath(
+ user, sliderServiceClass, instanceName,
+ containerId);
+ LOG.info(instanceId + ": Deleting registry path " + path);
+ registryOperations.delete(path, false);
+ }
+
+ /**
+ * Delete the children of a path -but not the path itself.
+ * It is not an error if the path does not exist
+ * @param path path to delete
+ * @param recursive flag to request recursive deletes
+ * @throws IOException IO problems
+ */
+ public void deleteChildren(String path, boolean recursive) throws IOException {
+ List<String> childNames = null;
+ try {
+ childNames = registryOperations.list(path);
+ } catch (PathNotFoundException e) {
+ return;
+ }
+ for (String childName : childNames) {
+ String child = join(path, childName);
+ registryOperations.delete(child, recursive);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java
new file mode 100644
index 0000000..cf4e836
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+import org.apache.commons.configuration2.SubsetConfiguration;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.MetricsSink;
+import org.apache.hadoop.metrics2.MetricsTag;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Write the metrics to a ATSv2. Generally, this class is instantiated via
+ * hadoop-metrics2 property files. Specifically, you would create this class by
+ * adding the following to by This would actually be set as: <code>
+ * [prefix].sink.[some instance name].class
+ * =org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink
+ * </code>, where <tt>prefix</tt> is "atsv2": and <tt>some instance name</tt> is
+ * just any unique name, so properties can be differentiated if there are
+ * multiple sinks of the same type created
+ */
+public class ServiceMetricsSink implements MetricsSink {
+
+ private static final Logger log =
+ LoggerFactory.getLogger(ServiceMetricsSink.class);
+
+ private ServiceTimelinePublisher serviceTimelinePublisher;
+
+ public ServiceMetricsSink() {
+
+ }
+
+ public ServiceMetricsSink(ServiceTimelinePublisher publisher) {
+ serviceTimelinePublisher = publisher;
+ }
+
+ /**
+ * Publishes service and component metrics to ATS.
+ */
+ @Override
+ public void putMetrics(MetricsRecord record) {
+ if (serviceTimelinePublisher.isStopped()) {
+ log.warn("ServiceTimelinePublisher has stopped. "
+ + "Not publishing any more metrics to ATS.");
+ return;
+ }
+
+ boolean isServiceMetrics = false;
+ boolean isComponentMetrics = false;
+ String appId = null;
+ for (MetricsTag tag : record.tags()) {
+ if (tag.name().equals("type") && tag.value().equals("service")) {
+ isServiceMetrics = true;
+ } else if (tag.name().equals("type") && tag.value().equals("component")) {
+ isComponentMetrics = true;
+ break; // if component metrics, no more information required from tag so
+ // break the loop
+ } else if (tag.name().equals("appId")) {
+ appId = tag.value();
+ }
+ }
+
+ if (isServiceMetrics && appId != null) {
+ if (log.isDebugEnabled()) {
+ log.debug("Publishing service metrics. " + record);
+ }
+ serviceTimelinePublisher.publishMetrics(record.metrics(), appId,
+ ServiceTimelineEntityType.SERVICE_ATTEMPT.toString(),
+ record.timestamp());
+ } else if (isComponentMetrics) {
+ if (log.isDebugEnabled()) {
+ log.debug("Publishing Component metrics. " + record);
+ }
+ serviceTimelinePublisher.publishMetrics(record.metrics(), record.name(),
+ ServiceTimelineEntityType.COMPONENT.toString(), record.timestamp());
+ }
+ }
+
+ @Override
+ public void init(SubsetConfiguration conf) {
+ }
+
+ @Override
+ public void flush() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java
new file mode 100644
index 0000000..d5c9539
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+/**
+ * Slider entities that are published to ATS.
+ */
+public enum ServiceTimelineEntityType {
+ /**
+ * Used for publishing service entity information.
+ */
+ SERVICE_ATTEMPT,
+
+ /**
+ * Used for publishing component entity information.
+ */
+ COMPONENT,
+
+ /**
+ * Used for publishing component instance entity information.
+ */
+ COMPONENT_INSTANCE
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
new file mode 100644
index 0000000..7f7f9a1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+/**
+ * Events that are used to store in ATS.
+ */
+public enum ServiceTimelineEvent {
+ SERVICE_ATTEMPT_REGISTERED,
+
+ SERVICE_ATTEMPT_UNREGISTERED,
+
+ COMPONENT_INSTANCE_REGISTERED,
+
+ COMPONENT_INSTANCE_UNREGISTERED,
+
+ COMPONENT_INSTANCE_UPDATED
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
new file mode 100644
index 0000000..78a7171
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+/**
+ * Constants which are stored as key in ATS
+ */
+public final class ServiceTimelineMetricsConstants {
+
+ public static final String URI = "URI";
+
+ public static final String NAME = "NAME";
+
+ public static final String STATE = "STATE";
+
+ public static final String EXIT_STATUS_CODE = "EXIT_STATUS_CODE";
+
+ public static final String EXIT_REASON = "EXIT_REASON";
+
+ public static final String DIAGNOSTICS_INFO = "DIAGNOSTICS_INFO";
+
+ public static final String LAUNCH_TIME = "LAUNCH_TIME";
+
+ public static final String QUICK_LINKS = "QUICK_LINKS";
+
+ public static final String LAUNCH_COMMAND = "LAUNCH_COMMAND";
+
+ public static final String TOTAL_CONTAINERS = "NUMBER_OF_CONTAINERS";
+
+ public static final String RUNNING_CONTAINERS =
+ "NUMBER_OF_RUNNING_CONTAINERS";
+
+ /**
+ * Artifacts constants.
+ */
+ public static final String ARTIFACT_ID = "ARTIFACT_ID";
+
+ public static final String ARTIFACT_TYPE = "ARTIFACT_TYPE";
+
+ public static final String ARTIFACT_URI = "ARTIFACT_URI";
+
+ /**
+ * Resource constants.
+ */
+ public static final String RESOURCE_CPU = "RESOURCE_CPU";
+
+ public static final String RESOURCE_MEMORY = "RESOURCE_MEMORY";
+
+ public static final String RESOURCE_PROFILE = "RESOURCE_PROFILE";
+
+ /**
+ * component instance constants.
+ */
+ public static final String IP = "IP";
+
+ public static final String HOSTNAME = "HOSTNAME";
+
+ public static final String BARE_HOST = "BARE_HOST";
+
+ public static final String COMPONENT_NAME = "COMPONENT_NAME";
+
+ public static final String COMPONENT_INSTANCE_NAME = "COMPONENT_INSTANCE_NAME";
+
+ /**
+ * component constants.
+ */
+ public static final String DEPENDENCIES = "DEPENDENCIES";
+
+ public static final String DESCRIPTION = "DESCRIPTION";
+
+ public static final String RUN_PRIVILEGED_CONTAINER =
+ "RUN_PRIVILEGED_CONTAINER";
+
+ public static final String PLACEMENT_POLICY = "PLACEMENT_POLICY";
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
new file mode 100644
index 0000000..5e65ad9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+import org.apache.hadoop.metrics2.AbstractMetric;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.client.api.TimelineV2Client;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.service.timelineservice.ServiceTimelineMetricsConstants.DIAGNOSTICS_INFO;
+
+/**
+ * A single service that publishes all the Timeline Entities.
+ */
+public class ServiceTimelinePublisher extends CompositeService {
+
+ // Number of bytes of config which can be published in one shot to ATSv2.
+ public static final int ATS_CONFIG_PUBLISH_SIZE_BYTES = 10 * 1024;
+
+ private TimelineV2Client timelineClient;
+
+ private volatile boolean stopped = false;
+
+ private static final Logger log =
+ LoggerFactory.getLogger(ServiceTimelinePublisher.class);
+
+ @Override
+ protected void serviceInit(org.apache.hadoop.conf.Configuration configuration)
+ throws Exception {
+ addService(timelineClient);
+ }
+
+
+ @Override
+ protected void serviceStop() throws Exception {
+ stopped = true;
+ super.serviceStop();
+ }
+
+ public boolean isStopped() {
+ return stopped;
+ }
+
+ public ServiceTimelinePublisher(TimelineV2Client client) {
+ super(ServiceTimelinePublisher.class.getName());
+ timelineClient = client;
+ }
+
+ public void serviceAttemptRegistered(Service service,
+ org.apache.hadoop.conf.Configuration systemConf) {
+ long currentTimeMillis = service.getLaunchTime() == null
+ ? System.currentTimeMillis() : service.getLaunchTime().getTime();
+
+ TimelineEntity entity = createServiceAttemptEntity(service.getId());
+ entity.setCreatedTime(currentTimeMillis);
+
+ // create info keys
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(ServiceTimelineMetricsConstants.NAME, service.getName());
+ entityInfos.put(ServiceTimelineMetricsConstants.STATE,
+ service.getState().toString());
+ entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME,
+ currentTimeMillis);
+ entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS,
+ service.getQuicklinks());
+ entity.addInfo(entityInfos);
+
+ // add an event
+ TimelineEvent startEvent = new TimelineEvent();
+ startEvent.setId(ServiceTimelineEvent.SERVICE_ATTEMPT_REGISTERED.toString());
+ startEvent.setTimestamp(currentTimeMillis);
+ entity.addEvent(startEvent);
+
+ // publish before configurations published
+ putEntity(entity);
+
+ // publish system config - YarnConfiguration
+ populateTimelineEntity(systemConf.iterator(), service.getId(),
+ ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
+ // publish user conf
+ publishUserConf(service.getConfiguration(), service.getId(),
+ ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
+
+ // publish component as separate entity.
+ publishComponents(service.getComponents());
+ }
+
+ public void serviceAttemptUpdated(Service service) {
+ TimelineEntity entity = createServiceAttemptEntity(service.getId());
+ entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS,
+ service.getQuicklinks());
+ putEntity(entity);
+ }
+
+ public void serviceAttemptUnregistered(ServiceContext context,
+ String diagnostics) {
+ TimelineEntity entity = createServiceAttemptEntity(
+ context.attemptId.getApplicationId().toString());
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(ServiceTimelineMetricsConstants.STATE,
+ FinalApplicationStatus.ENDED);
+ entityInfos.put(DIAGNOSTICS_INFO, diagnostics);
+ entity.addInfo(entityInfos);
+
+ // add an event
+ TimelineEvent finishEvent = new TimelineEvent();
+ finishEvent
+ .setId(ServiceTimelineEvent.SERVICE_ATTEMPT_UNREGISTERED.toString());
+ finishEvent.setTimestamp(System.currentTimeMillis());
+ entity.addEvent(finishEvent);
+
+ putEntity(entity);
+ }
+
+ public void componentInstanceStarted(Container container,
+ ComponentInstance instance) {
+
+ TimelineEntity entity = createComponentInstanceEntity(container.getId());
+ entity.setCreatedTime(container.getLaunchTime().getTime());
+
+ // create info keys
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(ServiceTimelineMetricsConstants.BARE_HOST,
+ container.getBareHost());
+ entityInfos.put(ServiceTimelineMetricsConstants.STATE,
+ container.getState().toString());
+ entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME,
+ container.getLaunchTime().getTime());
+ entityInfos.put(ServiceTimelineMetricsConstants.COMPONENT_NAME,
+ instance.getCompName());
+ entityInfos.put(ServiceTimelineMetricsConstants.COMPONENT_INSTANCE_NAME,
+ instance.getCompInstanceName());
+ entity.addInfo(entityInfos);
+
+ // add an event
+ TimelineEvent startEvent = new TimelineEvent();
+ startEvent
+ .setId(ServiceTimelineEvent.COMPONENT_INSTANCE_REGISTERED.toString());
+ startEvent.setTimestamp(container.getLaunchTime().getTime());
+ entity.addEvent(startEvent);
+
+ putEntity(entity);
+ }
+
+ public void componentInstanceFinished(ComponentInstance instance,
+ int exitCode, ContainerState state, String diagnostics) {
+ TimelineEntity entity = createComponentInstanceEntity(
+ instance.getContainer().getId().toString());
+
+ // create info keys
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(ServiceTimelineMetricsConstants.EXIT_STATUS_CODE,
+ exitCode);
+ entityInfos.put(DIAGNOSTICS_INFO, diagnostics);
+ entityInfos.put(ServiceTimelineMetricsConstants.STATE, state);
+ entity.addInfo(entityInfos);
+
+ // add an event
+ TimelineEvent startEvent = new TimelineEvent();
+ startEvent
+ .setId(ServiceTimelineEvent.COMPONENT_INSTANCE_UNREGISTERED.toString());
+ startEvent.setTimestamp(System.currentTimeMillis());
+ entity.addEvent(startEvent);
+
+ putEntity(entity);
+ }
+
+ public void componentInstanceUpdated(Container container) {
+ TimelineEntity entity = createComponentInstanceEntity(container.getId());
+
+ // create info keys
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(ServiceTimelineMetricsConstants.IP, container.getIp());
+ entityInfos.put(ServiceTimelineMetricsConstants.HOSTNAME,
+ container.getHostname());
+ entityInfos.put(ServiceTimelineMetricsConstants.STATE,
+ container.getState().toString());
+ entity.addInfo(entityInfos);
+
+ TimelineEvent updateEvent = new TimelineEvent();
+ updateEvent
+ .setId(ServiceTimelineEvent.COMPONENT_INSTANCE_UPDATED.toString());
+ updateEvent.setTimestamp(System.currentTimeMillis());
+ entity.addEvent(updateEvent);
+
+ putEntity(entity);
+ }
+
+ private void publishComponents(List<Component> components) {
+ long currentTimeMillis = System.currentTimeMillis();
+ for (Component component : components) {
+ TimelineEntity entity = createComponentEntity(component.getName());
+ entity.setCreatedTime(currentTimeMillis);
+
+ // create info keys
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_ID,
+ component.getArtifact().getId());
+ entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_TYPE,
+ component.getArtifact().getType().toString());
+ if (component.getResource().getProfile() != null) {
+ entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_PROFILE,
+ component.getResource().getProfile());
+ }
+ entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_CPU,
+ component.getResource().getCpus());
+ entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_MEMORY,
+ component.getResource().getMemory());
+
+ if (component.getLaunchCommand() != null) {
+ entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_COMMAND,
+ component.getLaunchCommand());
+ }
+ entityInfos.put(ServiceTimelineMetricsConstants.RUN_PRIVILEGED_CONTAINER,
+ component.getRunPrivilegedContainer().toString());
+ if (component.getPlacementPolicy() != null) {
+ entityInfos.put(ServiceTimelineMetricsConstants.PLACEMENT_POLICY,
+ component.getPlacementPolicy().getLabel());
+ }
+ entity.addInfo(entityInfos);
+
+ putEntity(entity);
+
+ // publish component specific configurations
+ publishUserConf(component.getConfiguration(), component.getName(),
+ ServiceTimelineEntityType.COMPONENT.toString());
+ }
+ }
+
+ private void publishUserConf(Configuration configuration,
+ String entityId, String entityType) {
+ populateTimelineEntity(configuration.getProperties().entrySet().iterator(),
+ entityId, entityType);
+
+ populateTimelineEntity(configuration.getEnv().entrySet().iterator(),
+ entityId, entityType);
+
+ for (ConfigFile configFile : configuration.getFiles()) {
+ populateTimelineEntity(configFile.getProps().entrySet().iterator(),
+ entityId, entityType);
+ }
+ }
+
+ private void populateTimelineEntity(Iterator<Entry<String, String>> iterator,
+ String entityId, String entityType) {
+ int configSize = 0;
+ TimelineEntity entity = createTimelineEntity(entityId, entityType);
+ while (iterator.hasNext()) {
+ Entry<String, String> entry = iterator.next();
+ int size = entry.getKey().length() + entry.getValue().length();
+ configSize += size;
+ // Configs are split into multiple entities if they exceed 100kb in size.
+ if (configSize > ATS_CONFIG_PUBLISH_SIZE_BYTES) {
+ if (entity.getConfigs().size() > 0) {
+ putEntity(entity);
+ entity = createTimelineEntity(entityId, entityType);
+ }
+ configSize = size;
+ }
+ entity.addConfig(entry.getKey(), entry.getValue());
+ }
+ if (configSize > 0) {
+ putEntity(entity);
+ }
+ }
+
+ /**
+ * Called from ServiceMetricsSink at regular interval of time.
+ * @param metrics of service or components
+ * @param entityId Id of entity
+ * @param entityType Type of entity
+ * @param timestamp
+ */
+ public void publishMetrics(Iterable<AbstractMetric> metrics, String entityId,
+ String entityType, long timestamp) {
+ TimelineEntity entity = createTimelineEntity(entityId, entityType);
+ Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>();
+ for (AbstractMetric metric : metrics) {
+ TimelineMetric timelineMetric = new TimelineMetric();
+ timelineMetric.setId(metric.name());
+ timelineMetric.addValue(timestamp, metric.value());
+ entityMetrics.add(timelineMetric);
+ }
+ entity.setMetrics(entityMetrics);
+ putEntity(entity);
+ }
+
+ private TimelineEntity createServiceAttemptEntity(String serviceId) {
+ TimelineEntity entity = createTimelineEntity(serviceId,
+ ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
+ return entity;
+ }
+
+ private TimelineEntity createComponentInstanceEntity(String instanceId) {
+ TimelineEntity entity = createTimelineEntity(instanceId,
+ ServiceTimelineEntityType.COMPONENT_INSTANCE.toString());
+ return entity;
+ }
+
+ private TimelineEntity createComponentEntity(String componentId) {
+ TimelineEntity entity = createTimelineEntity(componentId,
+ ServiceTimelineEntityType.COMPONENT.toString());
+ return entity;
+ }
+
+ private TimelineEntity createTimelineEntity(String entityId,
+ String entityType) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId(entityId);
+ entity.setType(entityType);
+ return entity;
+ }
+
+ private void putEntity(TimelineEntity entity) {
+ try {
+ if (log.isDebugEnabled()) {
+ log.debug("Publishing the entity " + entity + ", JSON-style content: "
+ + TimelineUtils.dumpTimelineRecordtoJSON(entity));
+ }
+ if (timelineClient != null) {
+ timelineClient.putEntitiesAsync(entity);
+ } else {
+ log.error("Seems like client has been removed before the entity "
+ + "could be published for " + entity);
+ }
+ } catch (Exception e) {
+ log.error("Error when publishing entity " + entity, e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java
new file mode 100644
index 0000000..72f7842
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * ATS implementation
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.service.timelineservice;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java
new file mode 100644
index 0000000..2607c08
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.JsonMappingException;
+
+import java.io.IOException;
+
+/**
+ * Persistence of {@link SerializedApplicationReport}
+ *
+ */
+public class ApplicationReportSerDeser
+ extends JsonSerDeser<SerializedApplicationReport> {
+ public ApplicationReportSerDeser() {
+ super(SerializedApplicationReport.class);
+ }
+
+
+ private static final ApplicationReportSerDeser
+ staticinstance = new ApplicationReportSerDeser();
+
+ /**
+ * Convert an instance to a JSON string -sync access to a shared ser/deser
+ * object instance
+ * @param instance object to convert
+ * @return a JSON string description
+ * @throws JsonParseException parse problems
+ * @throws JsonMappingException O/J mapping problems
+ */
+ public static String toString(SerializedApplicationReport instance)
+ throws IOException, JsonGenerationException, JsonMappingException {
+ synchronized (staticinstance) {
+ return staticinstance.toJson(instance);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
new file mode 100644
index 0000000..86896b2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.registry.client.impl.zk.RegistryInternalConstants;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.encodeForRegistry;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.convertUsername;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.getCurrentUsernameUnencoded;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
+
+/**
+ * Generic code to get the URLs for clients via the registry
+ */
+public class ClientRegistryBinder {
+ private static final Logger log =
+ LoggerFactory.getLogger(ClientRegistryBinder.class);
+
+ private final RegistryOperations operations;
+
+ public ClientRegistryBinder(RegistryOperations operations) {
+ this.operations = operations;
+ }
+
+ /**
+ * Buld the user path -switches to the system path if the user is "".
+ * It also cross-converts the username to ascii via punycode
+ * @param username username or ""
+ * @return the path to the user
+ */
+ public static String homePathForUser(String username) {
+ Preconditions.checkArgument(username != null, "null user");
+
+ // catch recursion
+ if (username.startsWith(RegistryConstants.PATH_USERS)) {
+ return username;
+ }
+
+ if (username.isEmpty()) {
+ return RegistryConstants.PATH_SYSTEM_SERVICES;
+ }
+
+ // convert username to registry name
+ String convertedName = convertUsername(username);
+
+ return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
+ encodeForRegistry(convertedName));
+ }
+
+ /**
+ * Get the current username, before any encoding has been applied.
+ * @return the current user from the kerberos identity, falling back
+ * to the user and/or env variables.
+ */
+ public static String currentUsernameUnencoded() {
+ String env_hadoop_username = System.getenv(
+ RegistryInternalConstants.HADOOP_USER_NAME);
+ return getCurrentUsernameUnencoded(env_hadoop_username);
+ }
+
+ /**
+ * Qualify a user.
+ * <ol>
+ * <li> <code>"~"</code> maps to user home path home</li>
+ * <li> <code>"~user"</code> maps to <code>/users/$user</code></li>
+ * <li> <code>"/"</code> maps to <code>/services/</code></li>
+ * </ol>
+ * @param user the username
+ * @return the base path
+ */
+ public static String qualifyUser(String user) {
+ // qualify the user
+ String t = user.trim();
+ if (t.startsWith("/")) {
+ // already resolved
+ return t;
+ } else if (t.equals("~")) {
+ // self
+ return currentUsernameUnencoded();
+ } else if (t.startsWith("~")) {
+ // another user
+ // convert username to registry name
+ String convertedName = convertUsername(t.substring(1));
+
+ return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
+ encodeForRegistry(convertedName));
+ } else {
+ return "/" + t;
+ }
+ }
+
+ /**
+ * Look up an external REST API
+ * @param user user which will be qualified as per {@link #qualifyUser(String)}
+ * @param serviceClass service class
+ * @param instance instance name
+ * @param api API
+ * @return the API, or an exception is raised.
+ * @throws IOException
+ */
+ public String lookupExternalRestAPI(String user,
+ String serviceClass,
+ String instance,
+ String api)
+ throws IOException {
+ String qualified = qualifyUser(user);
+ String path = servicePath(qualified, serviceClass, instance);
+ String restAPI = resolveExternalRestAPI(api, path);
+ if (restAPI == null) {
+ throw new PathNotFoundException(path + " API " + api);
+ }
+ return restAPI;
+ }
+
+ /**
+ * Resolve a service record then return an external REST API exported it.
+ *
+ * @param api API to resolve
+ * @param path path of the service record
+ * @return null if the record exists but the API is absent or it has no
+ * REST endpoints.
+ * @throws IOException resolution problems, as covered in
+ * {@link RegistryOperations#resolve(String)}
+ */
+ protected String resolveExternalRestAPI(String api, String path) throws
+ IOException {
+ ServiceRecord record = operations.resolve(path);
+ return lookupRestAPI(record, api, true);
+ }
+
+ /**
+ * Look up an external REST API endpoint
+ * @param record service record
+ * @param api URI of api
+ * @param external flag to indicate this is an external record
+ * @return the first endpoint of the implementation, or null if there
+ * is no entry for the API, implementation or it's the wrong type.
+ */
+ public static String lookupRestAPI(ServiceRecord record,
+ String api, boolean external) throws InvalidRecordException {
+ try {
+ String url = null;
+ Endpoint endpoint = getEndpoint(record, api, external);
+ List<String> addresses =
+ RegistryTypeUtils.retrieveAddressesUriType(endpoint);
+ if (addresses != null && !addresses.isEmpty()) {
+ url = addresses.get(0);
+ }
+ return url;
+ } catch (InvalidRecordException e) {
+ log.debug("looking for API {}", api, e);
+ return null;
+ }
+ }
+
+ /**
+ * Get an endpont by API
+ * @param record service record
+ * @param api API
+ * @param external flag to indicate this is an external record
+ * @return the endpoint or null
+ */
+ public static Endpoint getEndpoint(ServiceRecord record,
+ String api,
+ boolean external) {
+ return external ? record.getExternalEndpoint(api)
+ : record.getInternalEndpoint(api);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java
new file mode 100644
index 0000000..9f0e5d4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+/**
+ * Some general comparators
+ */
+public class Comparators {
+
+ public static class LongComparator implements Comparator<Long>, Serializable {
+ @Override
+ public int compare(Long o1, Long o2) {
+ return o1.compareTo(o2);
+ }
+ }
+
+ public static class InvertedLongComparator
+ implements Comparator<Long>, Serializable {
+ @Override
+ public int compare(Long o1, Long o2) {
+ return o2.compareTo(o1);
+ }
+ }
+
+ /**
+ * Little template class to reverse any comparitor
+ * @param <CompareType> the type that is being compared
+ */
+ public static class ComparatorReverser<CompareType> implements Comparator<CompareType>,
+ Serializable {
+
+ final Comparator<CompareType> instance;
+
+ public ComparatorReverser(Comparator<CompareType> instance) {
+ this.instance = instance;
+ }
+
+ @Override
+ public int compare(CompareType first, CompareType second) {
+ return instance.compare(second, first);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
new file mode 100644
index 0000000..fe8cce8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.net.URL;
+import java.util.Map;
+
+/**
+ * Methods to aid in config, both in the Configuration class and
+ * with other parts of setting up Slider-initated processes.
+ *
+ * Some of the methods take an argument of a map iterable for their sources; this allows
+ * the same method
+ */
+public class ConfigHelper {
+ private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);
+
+ /**
+ * Set an entire map full of values
+ *
+ * @param config config to patch
+ * @param map map of data
+ * @param origin origin data
+ */
+ public static void addConfigMap(Configuration config,
+ Map<String, String> map,
+ String origin) throws BadConfigException {
+ addConfigMap(config, map.entrySet(), origin);
+ }
+
+ /**
+ * Set an entire map full of values
+ *
+ * @param config config to patch
+ * @param map map of data
+ * @param origin origin data
+ */
+ public static void addConfigMap(Configuration config,
+ Iterable<Map.Entry<String, String>> map,
+ String origin) throws BadConfigException {
+ for (Map.Entry<String, String> mapEntry : map) {
+ String key = mapEntry.getKey();
+ String value = mapEntry.getValue();
+ if (value == null) {
+ throw new BadConfigException("Null value for property " + key);
+ }
+ config.set(key, value, origin);
+ }
+ }
+
+ /**
+ * Convert to an XML string
+ * @param conf configuration
+ * @return conf
+ * @throws IOException
+ */
+ public static String toXml(Configuration conf) throws IOException {
+ StringWriter writer = new StringWriter();
+ conf.writeXml(writer);
+ return writer.toString();
+ }
+
+
+ /**
+ * Register a resource as a default resource.
+ * Do not attempt to use this unless you understand that the
+ * order in which default resources are loaded affects the outcome,
+ * and that subclasses of Configuration often register new default
+ * resources
+ * @param resource the resource name
+ * @return the URL or null
+ */
+ public static URL registerDefaultResource(String resource) {
+ URL resURL = getResourceUrl(resource);
+ if (resURL != null) {
+ Configuration.addDefaultResource(resource);
+ }
+ return resURL;
+ }
+
+ /**
+ * Load a configuration from a resource on this classpath.
+ * If the resource is not found, an empty configuration is returned
+ * @param resource the resource name
+ * @return the loaded configuration.
+ */
+ public static Configuration loadFromResource(String resource) {
+ Configuration conf = new Configuration(false);
+ URL resURL = getResourceUrl(resource);
+ if (resURL != null) {
+ log.debug("loaded resources from {}", resURL);
+ conf.addResource(resource);
+ } else{
+ log.debug("failed to find {} on the classpath", resource);
+ }
+ return conf;
+
+ }
+
+ /**
+ * Get the URL to a resource, null if not on the CP
+ * @param resource resource to look for
+ * @return the URL or null
+ */
+ public static URL getResourceUrl(String resource) {
+ return ConfigHelper.class.getClassLoader()
+ .getResource(resource);
+ }
+
+ /**
+ * This goes through the keyset of one configuration and retrieves each value
+ * from a value source -a different or the same configuration. This triggers
+ * the property resolution process of the value, resolving any variables against
+ * in-config or inherited configurations
+ * @param keysource source of keys
+ * @param valuesource the source of values
+ * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code>
+ */
+ public static Configuration resolveConfiguration(
+ Iterable<Map.Entry<String, String>> keysource,
+ Configuration valuesource) {
+ Configuration result = new Configuration(false);
+ for (Map.Entry<String, String> entry : keysource) {
+ String key = entry.getKey();
+ String value = valuesource.get(key);
+ Preconditions.checkState(value != null,
+ "no reference for \"%s\" in values", key);
+ result.set(key, value);
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java
new file mode 100644
index 0000000..a969be9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class ConfigUtils {
+ public static final String TEMPLATE_FILE = "template.file";
+
+ public static String replaceProps(Map<String, String> config, String content) {
+ Map<String, String> tokens = new HashMap<>();
+ for (Entry<String, String> entry : config.entrySet()) {
+ tokens.put("${" + entry.getKey() + "}", entry.getValue());
+ tokens.put("{{" + entry.getKey() + "}}", entry.getValue());
+ }
+ String value = content;
+ for (Map.Entry<String,String> token : tokens.entrySet()) {
+ value = value.replaceAll(Pattern.quote(token.getKey()),
+ Matcher.quoteReplacement(token.getValue()));
+ }
+ return value;
+ }
+
+ public static Map<String, String> replacePropsInConfig(
+ Map<String, String> config, Map<String, String> env) {
+ Map<String, String> tokens = new HashMap<>();
+ for (Entry<String, String> entry : env.entrySet()) {
+ tokens.put("${" + entry.getKey() + "}", entry.getValue());
+ }
+ Map<String, String> newConfig = new HashMap<>();
+ for (Entry<String, String> entry : config.entrySet()) {
+ String value = entry.getValue();
+ for (Map.Entry<String,String> token : tokens.entrySet()) {
+ value = value.replaceAll(Pattern.quote(token.getKey()),
+ Matcher.quoteReplacement(token.getValue()));
+ }
+ newConfig.put(entry.getKey(), entry.getValue());
+ }
+ return newConfig;
+ }
+
+ public static void prepConfigForTemplateOutputter(ConfigFormat configFormat,
+ Map<String, String> config, SliderFileSystem fileSystem,
+ String clusterName, String fileName) throws IOException {
+ if (!configFormat.equals(ConfigFormat.TEMPLATE)) {
+ return;
+ }
+ Path templateFile = null;
+ if (config.containsKey(TEMPLATE_FILE)) {
+ templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE));
+ if (!fileSystem.isFile(templateFile)) {
+ templateFile = fileSystem.buildResourcePath(clusterName,
+ config.get(TEMPLATE_FILE));
+ }
+ if (!fileSystem.isFile(templateFile)) {
+ throw new IOException("config specified template file " + config
+ .get(TEMPLATE_FILE) + " but " + templateFile + " doesn't exist");
+ }
+ }
+ if (templateFile == null && fileName != null) {
+ templateFile = fileSystem.buildResourcePath(fileName);
+ if (!fileSystem.isFile(templateFile)) {
+ templateFile = fileSystem.buildResourcePath(clusterName,
+ fileName);
+ }
+ }
+ if (fileSystem.isFile(templateFile)) {
+ config.put("content", fileSystem.cat(templateFile));
+ } else {
+ config.put("content", "");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
new file mode 100644
index 0000000..281e1dfe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
@@ -0,0 +1,521 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.util.Records;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class CoreFileSystem {
+ private static final Logger
+ log = LoggerFactory.getLogger(CoreFileSystem.class);
+
+ private static final String UTF_8 = "UTF-8";
+
+ protected final FileSystem fileSystem;
+ protected final Configuration configuration;
+
+ public CoreFileSystem(FileSystem fileSystem, Configuration configuration) {
+ Preconditions.checkNotNull(fileSystem,
+ "Cannot create a CoreFileSystem with a null FileSystem");
+ Preconditions.checkNotNull(configuration,
+ "Cannot create a CoreFileSystem with a null Configuration");
+ this.fileSystem = fileSystem;
+ this.configuration = configuration;
+ }
+
+ public CoreFileSystem(Configuration configuration) throws IOException {
+ Preconditions.checkNotNull(configuration,
+ "Cannot create a CoreFileSystem with a null Configuration");
+ this.fileSystem = FileSystem.get(configuration);
+ this.configuration = configuration;
+ }
+
+ /**
+ * Get the temp path for this cluster
+ * @param clustername name of the cluster
+ * @return path for temp files (is not purged)
+ */
+ public Path getTempPathForCluster(String clustername) {
+ Path clusterDir = buildClusterDirPath(clustername);
+ return new Path(clusterDir, YarnServiceConstants.TMP_DIR_PREFIX);
+ }
+
+ /**
+ * Returns the underlying FileSystem for this object.
+ *
+ * @return filesystem
+ */
+ public FileSystem getFileSystem() {
+ return fileSystem;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("CoreFileSystem{");
+ sb.append("fileSystem=").append(fileSystem.getUri());
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Build up the path string for a cluster instance -no attempt to
+ * create the directory is made
+ *
+ * @param clustername name of the cluster
+ * @return the path for persistent data
+ */
+ public Path buildClusterDirPath(String clustername) {
+ Preconditions.checkNotNull(clustername);
+ Path path = getBaseApplicationPath();
+ return new Path(path, YarnServiceConstants.SERVICES_DIRECTORY + "/" + clustername);
+ }
+
+
+ /**
+ * Build up the path string for keytab install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for keytab
+ */
+ public Path buildKeytabInstallationDirPath(String keytabFolder) {
+ Preconditions.checkNotNull(keytabFolder);
+ Path path = getBaseApplicationPath();
+ return new Path(path, YarnServiceConstants.KEYTAB_DIR + "/" + keytabFolder);
+ }
+
+ /**
+ * Build up the path string for keytab install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for keytab installation location
+ */
+ public Path buildKeytabPath(String keytabDir, String keytabName, String clusterName) {
+ Path homePath = getHomeDirectory();
+ Path baseKeytabDir;
+ if (keytabDir != null) {
+ baseKeytabDir = new Path(homePath, keytabDir);
+ } else {
+ baseKeytabDir = new Path(buildClusterDirPath(clusterName),
+ YarnServiceConstants.KEYTAB_DIR);
+ }
+ return keytabName == null ? baseKeytabDir :
+ new Path(baseKeytabDir, keytabName);
+ }
+
+ /**
+ * Build up the path string for resource install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for resource
+ */
+ public Path buildResourcePath(String resourceFolder) {
+ Preconditions.checkNotNull(resourceFolder);
+ Path path = getBaseApplicationPath();
+ return new Path(path, YarnServiceConstants.RESOURCE_DIR + "/" + resourceFolder);
+ }
+
+ /**
+ * Build up the path string for resource install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for resource
+ */
+ public Path buildResourcePath(String dirName, String fileName) {
+ Preconditions.checkNotNull(dirName);
+ Preconditions.checkNotNull(fileName);
+ Path path = getBaseApplicationPath();
+ return new Path(path, YarnServiceConstants.RESOURCE_DIR + "/" + dirName + "/" + fileName);
+ }
+
+ /**
+ * Create a directory with the given permissions.
+ *
+ * @param dir directory
+ * @param clusterPerms cluster permissions
+ * @throws IOException IO problem
+ * @throws BadClusterStateException any cluster state problem
+ */
+ @SuppressWarnings("deprecation")
+ public void createWithPermissions(Path dir, FsPermission clusterPerms) throws
+ IOException,
+ BadClusterStateException {
+ if (fileSystem.isFile(dir)) {
+ // HADOOP-9361 shows some filesystems don't correctly fail here
+ throw new BadClusterStateException(
+ "Cannot create a directory over a file %s", dir);
+ }
+ log.debug("mkdir {} with perms {}", dir, clusterPerms);
+ //no mask whatoever
+ fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");
+ fileSystem.mkdirs(dir, clusterPerms);
+ //and force set it anyway just to make sure
+ fileSystem.setPermission(dir, clusterPerms);
+ }
+
+ /**
+ * Verify that the cluster directory is not present
+ *
+ * @param clustername name of the cluster
+ * @param clusterDirectory actual directory to look for
+ * @throws IOException trouble with FS
+ * @throws SliderException If the directory exists
+ */
+ public void verifyClusterDirectoryNonexistent(String clustername,
+ Path clusterDirectory)
+ throws IOException, SliderException {
+ if (fileSystem.exists(clusterDirectory)) {
+ throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
+ ErrorStrings.PRINTF_E_INSTANCE_ALREADY_EXISTS, clustername,
+ clusterDirectory);
+ }
+ }
+ /**
+ * Verify that the given directory is not present
+ *
+ * @param clusterDirectory actual directory to look for
+ * @throws IOException trouble with FS
+ * @throws SliderException If the directory exists
+ */
+ public void verifyDirectoryNonexistent(Path clusterDirectory) throws
+ IOException,
+ SliderException {
+ if (fileSystem.exists(clusterDirectory)) {
+
+ log.error("Dir {} exists: {}",
+ clusterDirectory,
+ listFSDir(clusterDirectory));
+ throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
+ ErrorStrings.PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS,
+ clusterDirectory);
+ }
+ }
+
+ /**
+ * Verify that a user has write access to a directory.
+ * It does this by creating then deleting a temp file
+ *
+ * @param dirPath actual directory to look for
+ * @throws FileNotFoundException file not found
+ * @throws IOException trouble with FS
+ * @throws BadClusterStateException if the directory is not writeable
+ */
+ public void verifyDirectoryWriteAccess(Path dirPath) throws IOException,
+ SliderException {
+ verifyPathExists(dirPath);
+ Path tempFile = new Path(dirPath, "tmp-file-for-checks");
+ try {
+ FSDataOutputStream out ;
+ out = fileSystem.create(tempFile, true);
+ IOUtils.closeStream(out);
+ fileSystem.delete(tempFile, false);
+ } catch (IOException e) {
+ log.warn("Failed to create file {}: {}", tempFile, e);
+ throw new BadClusterStateException(e,
+ "Unable to write to directory %s : %s", dirPath, e.toString());
+ }
+ }
+
+ /**
+ * Verify that a path exists
+ * @param path path to check
+ * @throws FileNotFoundException file not found
+ * @throws IOException trouble with FS
+ */
+ public void verifyPathExists(Path path) throws IOException {
+ if (!fileSystem.exists(path)) {
+ throw new FileNotFoundException(path.toString());
+ }
+ }
+
+ /**
+ * Verify that a path exists
+ * @param path path to check
+ * @throws FileNotFoundException file not found or is not a file
+ * @throws IOException trouble with FS
+ */
+ public void verifyFileExists(Path path) throws IOException {
+ FileStatus status = fileSystem.getFileStatus(path);
+
+ if (!status.isFile()) {
+ throw new FileNotFoundException("Not a file: " + path.toString());
+ }
+ }
+
+ /**
+ * Given a path, check if it exists and is a file
+ *
+ * @param path
+ * absolute path to the file to check
+ * @return true if and only if path exists and is a file, false for all other
+ * reasons including if file check throws IOException
+ */
+ public boolean isFile(Path path) {
+ boolean isFile = false;
+ try {
+ FileStatus status = fileSystem.getFileStatus(path);
+ if (status.isFile()) {
+ isFile = true;
+ }
+ } catch (IOException e) {
+ // ignore, isFile is already set to false
+ }
+ return isFile;
+ }
+
+ /**
+ * Get the base path
+ *
+ * @return the base path optionally configured by
+ * {@link YarnServiceConf#YARN_SERVICE_BASE_PATH}
+ */
+ public Path getBaseApplicationPath() {
+ String configuredBasePath = configuration
+ .get(YarnServiceConf.YARN_SERVICE_BASE_PATH,
+ getHomeDirectory() + "/" + YarnServiceConstants.SERVICE_BASE_DIRECTORY);
+ return new Path(configuredBasePath);
+ }
+
+ /**
+ * Get slider dependency parent dir in HDFS
+ *
+ * @return the parent dir path of slider.tar.gz in HDFS
+ */
+ public Path getDependencyPath() {
+ String parentDir = YarnServiceConstants.DEPENDENCY_DIR;
+ return new Path(String.format(parentDir, VersionInfo.getVersion()));
+ }
+
+ /**
+ * Get slider.tar.gz absolute filepath in HDFS
+ *
+ * @return the absolute path to slider.tar.gz in HDFS
+ */
+ public Path getDependencyTarGzip() {
+ Path dependencyLibAmPath = getDependencyPath();
+ Path dependencyLibTarGzip = new Path(
+ dependencyLibAmPath.toUri().toString(),
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
+ + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+ return dependencyLibTarGzip;
+ }
+
+ public Path getHomeDirectory() {
+ return fileSystem.getHomeDirectory();
+ }
+
+ /**
+ * Create an AM resource from the
+ *
+ * @param destPath dest path in filesystem
+ * @param resourceType resource type
+ * @return the local resource for AM
+ */
+ public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException {
+ FileStatus destStatus = fileSystem.getFileStatus(destPath);
+ LocalResource amResource = Records.newRecord(LocalResource.class);
+ amResource.setType(resourceType);
+ // Set visibility of the resource
+ // Setting to most private option
+ amResource.setVisibility(LocalResourceVisibility.APPLICATION);
+ // Set the resource to be copied over
+ amResource.setResource(
+ URL.fromPath(fileSystem.resolvePath(destStatus.getPath())));
+ // Set timestamp and length of file so that the framework
+ // can do basic sanity checks for the local resource
+ // after it has been copied over to ensure it is the same
+ // resource the client intended to use with the service
+ amResource.setTimestamp(destStatus.getModificationTime());
+ amResource.setSize(destStatus.getLen());
+ return amResource;
+ }
+
+ /**
+ * Register all files under a fs path as a directory to push out
+ *
+ * @param srcDir src dir
+ * @param destRelativeDir dest dir (no trailing /)
+ * @return the map of entries
+ */
+ public Map<String, LocalResource> submitDirectory(Path srcDir, String destRelativeDir) throws IOException {
+ //now register each of the files in the directory to be
+ //copied to the destination
+ FileStatus[] fileset = fileSystem.listStatus(srcDir);
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>(fileset.length);
+ for (FileStatus entry : fileset) {
+
+ LocalResource resource = createAmResource(entry.getPath(),
+ LocalResourceType.FILE);
+ String relativePath = destRelativeDir + "/" + entry.getPath().getName();
+ localResources.put(relativePath, resource);
+ }
+ return localResources;
+ }
+
+ /**
+ * Submit a JAR containing a specific class, returning
+ * the resource to be mapped in
+ *
+ * @param clazz class to look for
+ * @param subdir subdirectory (expected to end in a "/")
+ * @param jarName <i>At the destination</i>
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName)
+ throws IOException, SliderException {
+ File localFile = SliderUtils.findContainingJarOrFail(clazz);
+ return submitFile(localFile, tempPath, subdir, jarName);
+ }
+
+ /**
+ * Submit a local file to the filesystem references by the instance's cluster
+ * filesystem
+ *
+ * @param localFile filename
+ * @param subdir subdirectory (expected to end in a "/")
+ * @param destFileName destination filename
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public LocalResource submitFile(File localFile, Path tempPath, String subdir, String destFileName)
+ throws IOException {
+ Path src = new Path(localFile.toString());
+ Path subdirPath = new Path(tempPath, subdir);
+ fileSystem.mkdirs(subdirPath);
+ Path destPath = new Path(subdirPath, destFileName);
+ log.debug("Copying {} (size={} bytes) to {}", localFile, localFile.length(), destPath);
+
+ fileSystem.copyFromLocalFile(false, true, src, destPath);
+
+ // Set the type of resource - file or archive
+ // archives are untarred at destination
+ // we don't need the jar file to be untarred for now
+ return createAmResource(destPath, LocalResourceType.FILE);
+ }
+
+ /**
+ * Submit the AM tar.gz resource referenced by the instance's cluster
+ * filesystem. Also, update the providerResources object with the new
+ * resource.
+ *
+ * @param providerResources
+ * the provider resource map to be updated
+ * @throws IOException
+ * trouble copying to HDFS
+ */
+ public void submitTarGzipAndUpdate(
+ Map<String, LocalResource> providerResources) throws IOException,
+ BadClusterStateException {
+ Path dependencyLibTarGzip = getDependencyTarGzip();
+ LocalResource lc = createAmResource(dependencyLibTarGzip,
+ LocalResourceType.ARCHIVE);
+ providerResources.put(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK, lc);
+ }
+
+ public void copyLocalFileToHdfs(File localPath,
+ Path destPath, FsPermission fp)
+ throws IOException {
+ if (localPath == null || destPath == null) {
+ throw new IOException("Either localPath or destPath is null");
+ }
+ fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
+ "000");
+ fileSystem.mkdirs(destPath.getParent(), fp);
+ log.info("Copying file {} to {}", localPath.toURI(),
+ fileSystem.getScheme() + ":/" + destPath.toUri());
+
+ fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()),
+ destPath);
+ // set file permissions of the destPath
+ fileSystem.setPermission(destPath, fp);
+ }
+
+ public void copyHdfsFileToLocal(Path hdfsPath, File destFile)
+ throws IOException {
+ if (hdfsPath == null || destFile == null) {
+ throw new IOException("Either hdfsPath or destPath is null");
+ }
+ log.info("Copying file {} to {}", hdfsPath.toUri(), destFile.toURI());
+
+ Path destPath = new Path(destFile.getPath());
+ fileSystem.copyToLocalFile(hdfsPath, destPath);
+ }
+
+ /**
+ * list entries in a filesystem directory
+ *
+ * @param path directory
+ * @return a listing, one to a line
+ * @throws IOException
+ */
+ public String listFSDir(Path path) throws IOException {
+ FileStatus[] stats = fileSystem.listStatus(path);
+ StringBuilder builder = new StringBuilder();
+ for (FileStatus stat : stats) {
+ builder.append(stat.getPath().toString())
+ .append("\t")
+ .append(stat.getLen())
+ .append("\n");
+ }
+ return builder.toString();
+ }
+
+ public String cat(Path path) throws IOException {
+ FileStatus status = fileSystem.getFileStatus(path);
+ byte[] b = new byte[(int) status.getLen()];
+ FSDataInputStream in = null;
+ try {
+ in = fileSystem.open(path);
+ int count = in.read(b);
+ return new String(b, 0, count, UTF_8);
+ } finally {
+ IOUtils.closeStream(in);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java
new file mode 100644
index 0000000..6fadfd3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import java.io.Closeable;
+
+/**
+ * A duration in milliseconds. This class can be used
+ * to count time, and to be polled to see if a time limit has
+ * passed.
+ */
+public class Duration implements Closeable {
+ public long start, finish;
+ public final long limit;
+
+ /**
+ * Create a duration instance with a limit of 0
+ */
+ public Duration() {
+ this(0);
+ }
+
+ /**
+ * Create a duration with a limit specified in millis
+ * @param limit duration in milliseconds
+ */
+ public Duration(long limit) {
+ this.limit = limit;
+ }
+
+ /**
+ * Start
+ * @return self
+ */
+ public Duration start() {
+ start = now();
+ return this;
+ }
+
+ /**
+ * The close operation relays to {@link #finish()}.
+ * Implementing it allows Duration instances to be automatically
+ * finish()'d in Java7 try blocks for when used in measuring durations.
+ */
+ @Override
+ public final void close() {
+ finish();
+ }
+
+ public void finish() {
+ finish = now();
+ }
+
+ protected long now() {
+ return System.nanoTime()/1000000;
+ }
+
+ public long getInterval() {
+ return finish - start;
+ }
+
+ /**
+ * return true if the limit has been exceeded
+ * @return true if a limit was set and the current time
+ * exceeds it.
+ */
+ public boolean getLimitExceeded() {
+ return limit >= 0 && ((now() - start) > limit);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("Duration");
+ if (finish >= start) {
+ builder.append(" finished at ").append(getInterval()).append(" millis;");
+ } else {
+ if (start > 0) {
+ builder.append(" started but not yet finished;");
+ } else {
+ builder.append(" unstarted;");
+ }
+ }
+ if (limit > 0) {
+ builder.append(" limit: ").append(limit).append(" millis");
+ if (getLimitExceeded()) {
+ builder.append(" - exceeded");
+ }
+ }
+ return builder.toString();
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
deleted file mode 100644
index 51bde7b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_LIST},
- commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
-
-public class ActionListArgs extends AbstractActionArgs {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_LIST;
- }
-
- @Parameter(names = {ARG_LIVE},
- description = "List only live application instances")
- public boolean live;
-
- @Parameter(names = {ARG_STATE},
- description = "list only applications in the specific YARN state")
- public String state = "";
-
- @Parameter(names = {ARG_VERBOSE},
- description = "print out information in details")
- public boolean verbose = false;
-
- @Parameter(names = {ARG_CONTAINERS},
- description = "List containers of an application instance")
- public boolean containers;
-
- @Parameter(names = {ARG_VERSION},
- description = "Filter containers by app version (used with " +
- ARG_CONTAINERS + ")")
- public String version;
-
- @Parameter(names = {ARG_COMPONENTS}, variableArity = true,
- description = "Filter containers by component names (used with " +
- ARG_CONTAINERS + ")")
- public Set<String> components = new HashSet<>(0);
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
deleted file mode 100644
index 0888812..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.UsageException;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_LOOKUP},
- commandDescription = SliderActions.DESCRIBE_ACTION_LOOKUP)
-
-public class ActionLookupArgs extends AbstractActionArgs {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_LOOKUP;
- }
-
- public int getMinParams() {
- return 0;
- }
- public int getMaxParams() {
- return 0;
- }
-
- @Parameter(names = {ARG_ID},
- description = "ID of the application")
- public String id;
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for any application report")
- public File outputFile;
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
- if (StringUtils.isEmpty(id)) {
- throw new BadCommandArgumentsException("Missing mandatory argument "
- + ARG_ID);
- }
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder(SliderActions.ACTION_LOOKUP);
- if (id!=null) {
- sb.append(" ");
- sb.append(ARG_ID).append(" ").append(id);
- }
- if (outputFile != null) {
- sb.append(" ");
- sb.append(ARG_OUTPUT).append(" ").append(outputFile.getAbsolutePath());
- }
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java
deleted file mode 100644
index 5a0b019..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_NODES},
- commandDescription = SliderActions.DESCRIBE_ACTION_NODES)
-public class ActionNodesArgs extends AbstractActionArgs {
-
- /**
- * Instance for API use; on CLI the name is derived from {@link #getClusterName()}.
- */
- public String instance;
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_NODES;
- }
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "Output file for the information")
- public File outputFile;
-
- @Parameter(names = {ARG_LABEL})
- public String label = "";
-
- @Parameter(names = {ARG_HEALTHY} )
- public boolean healthy;
-
- @Override
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder(
- "ActionNodesArgs{");
- sb.append("instance='").append(instance).append('\'');
- sb.append(", outputFile=").append(outputFile);
- sb.append(", label='").append(label).append('\'');
- sb.append(", healthy=").append(healthy);
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
deleted file mode 100644
index fb76451..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.UsageException;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
-import java.io.File;
-
-/**
- * Registry actions
- *
- * --instance {app name}, if a / is in it, refers underneath?
- * --dest {destfile}
- * --list : list instances of slider service
- * --listfiles
- */
-@Parameters(commandNames = {ACTION_REGISTRY},
- commandDescription = DESCRIBE_ACTION_REGISTRY)
-
-public class ActionRegistryArgs extends AbstractActionArgs {
-
- public static final String USAGE =
- "Usage: " + SliderActions.ACTION_REGISTRY
- + " ("
- + Arguments.ARG_LIST + "|"
- + Arguments.ARG_LISTCONF + "|"
- + Arguments.ARG_LISTEXP + "|"
- + Arguments.ARG_LISTFILES + "|"
- + Arguments.ARG_GETCONF + "|"
- + Arguments.ARG_GETEXP + "> "
- + Arguments.ARG_NAME + " <name> "
- + " )"
- + "[" + Arguments.ARG_VERBOSE + "] "
- + "[" + Arguments.ARG_USER + "] "
- + "[" + Arguments.ARG_OUTPUT + " <filename> ] "
- + "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] "
- + "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] "
- + System.getProperty("line.separator")
- + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
- ;
- public ActionRegistryArgs() {
- }
-
- public ActionRegistryArgs(String name) {
- this.name = name;
- }
-
- @Override
- public String getActionName() {
- return ACTION_REGISTRY;
- }
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-
- @Parameter(names = {ARG_LIST},
- description = "list services")
- public boolean list;
-
- @Parameter(names = {ARG_LISTCONF},
- description = "list configurations")
- public boolean listConf;
-
- @Parameter(names = {ARG_GETCONF},
- description = "get configuration")
- public String getConf;
-
- @Parameter(names = {ARG_LISTEXP},
- description = "list exports")
- public boolean listExports;
-
- @Parameter(names = {ARG_GETEXP},
- description = "get export")
- public String getExport;
-
- @Parameter(names = {ARG_LISTFILES},
- description = "list files")
- public String listFiles;
-
- @Parameter(names = {ARG_GETFILES},
- description = "get files")
- public String getFiles;
-
- //--format
- @Parameter(names = ARG_FORMAT,
- description = "Format for a response: <xml|json|properties>")
- public String format = ConfigFormat.XML.toString() ;
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
- description = "Output destination")
- public File out;
-
- @Parameter(names = {ARG_NAME},
- description = "name of an instance")
- public String name;
-
- @Parameter(names = {ARG_SERVICETYPE},
- description = "optional service type")
- public String serviceType = SliderKeys.APP_TYPE;
-
- @Parameter(names = {ARG_VERBOSE},
- description = "verbose output")
- public boolean verbose;
-
- @Parameter(names = {ARG_INTERNAL},
- description = "fetch internal registry entries")
- public boolean internal;
-
- @Parameter(names = {ARG_USER},
- description = "the name of the user whose application is being resolved")
- public String user;
-
- /**
- * validate health of all the different operations
- * @throws BadCommandArgumentsException
- */
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
-
- //verify that at most one of the operations is set
- int gets = s(getConf) + s(getFiles) + s(getExport);
- int lists = s(list) + s(listConf) + s(listFiles) + s(listExports);
- int set = lists + gets;
- if (set > 1) {
- throw new UsageException(USAGE);
- }
-
- if (out != null && ( set == 0)) {
- throw new UsageException("output path"
- + " is only supported on 'get' operations: ");
- }
- if (!list && !is(name)) {
- throw new UsageException("Argument " + ARG_NAME
- +" missing: ");
-
- }
- }
-
- private int s(String arg) {
- return is(arg) ? 1 : 0;
- }
-
- private boolean is(String arg) {
- return arg != null;
- }
-
- private int s(boolean arg) {
- return arg ? 1 : 0;
- }
-
- private String ifdef(String arg, boolean val) {
- return val ? (arg + " "): "";
- }
-
- private String ifdef(String arg, String val) {
- if (is(val)) {
- return arg + " " + val + " ";
- } else {
- return "";
- }
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder(ACTION_REGISTRY);
- sb.append(' ');
- sb.append(ifdef(ARG_LIST, list));
- sb.append(ifdef(ARG_LISTCONF, listConf));
- sb.append(ifdef(ARG_LISTFILES, listFiles));
- sb.append(ifdef(ARG_GETCONF, getConf));
- sb.append(ifdef(ARG_GETFILES, getFiles));
-
- sb.append(ifdef(ARG_NAME, name));
- sb.append(ifdef(ARG_SERVICETYPE, serviceType));
-
-
- sb.append(ifdef(ARG_VERBOSE, verbose));
- sb.append(ifdef(ARG_INTERNAL, internal));
-
- if (out != null) {
- sb.append(ifdef(ARG_OUTPUT, out.toString()));
- }
- sb.append(ifdef(ARG_FORMAT, format));
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
deleted file mode 100644
index a953bc7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.UsageException;
-
-import java.io.File;
-
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_RESOLVE;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
-
-/**
- * Resolve registry entries
- *
- * --path {path}
- * --out {destfile}
- * --verbose
- * --list
- */
-@Parameters(commandNames = {ACTION_RESOLVE},
- commandDescription = DESCRIBE_ACTION_REGISTRY)
-public class ActionResolveArgs extends AbstractActionArgs {
-
- public static final String USAGE =
- "Usage: " + SliderActions.ACTION_RESOLVE
- + " "
- + ARG_PATH + " <path> "
- + "[" + ARG_LIST + "] "
- + "[" + ARG_OUTPUT + " <filename> ] "
- + "[" + ARG_DESTDIR + " <directory> ] "
- ;
- public ActionResolveArgs() {
- }
-
- @Override
- public String getActionName() {
- return ACTION_RESOLVE;
- }
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-
- @Parameter(names = {ARG_LIST},
- description = "list services")
- public boolean list;
-
- @Parameter(names = {ARG_PATH},
- description = "resolve a path")
- public String path;
-
- @Parameter(names = {ARG_DESTDIR},
- description = "destination directory for operations")
- public File destdir;
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "dest file")
- public File out;
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder(ACTION_RESOLVE).append(" ");
- sb.append(ARG_PATH).append(" ").append(path).append(" ");
- if (list) {
- sb.append(ARG_LIST).append(" ");
- }
- if (destdir != null) {
- sb.append(ARG_DESTDIR).append(" ").append(destdir).append(" ");
- }
- if (out != null) {
- sb.append(ARG_OUTPUT).append(" ").append(out).append(" ");
- }
- return sb.toString();
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
- if (StringUtils.isEmpty(path)) {
- throw new BadCommandArgumentsException("Missing mandatory argument "
- + ARG_PATH);
- }
- if (list && out != null) {
- throw new BadCommandArgumentsException("Argument "
- + ARG_OUTPUT +
- " not supported for " + ARG_LIST);
- }
- if (out != null && destdir != null) {
- throw new BadCommandArgumentsException(
- ARG_OUTPUT + " and " + ARG_DESTDIR + " cannot be used together"
- );
- }
- }
-
- public String getPath() {
- return path;
- }
-
- public void setPath(String path) {
- this.path = path;
- }
-
- public boolean isList() {
- return list;
- }
-
- public void setList(boolean list) {
- this.list = list;
- }
-
- public File getDestdir() {
- return destdir;
- }
-
- public void setDestdir(File destdir) {
- this.destdir = destdir;
- }
-
- public File getOut() {
- return out;
- }
-
- public void setOut(File out) {
- this.out = out;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java
deleted file mode 100644
index 6d60ca7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_RESOURCE},
- commandDescription = SliderActions.DESCRIBE_ACTION_RESOURCE)
-
-public class ActionResourceArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_RESOURCE;
- }
-
- @Parameter(names = {ARG_INSTALL},
- description = "Install the resource(s)")
- public boolean install;
-
- @Parameter(names = {ARG_DELETE},
- description = "Delete the file")
- public boolean delete;
-
- @Parameter(names = {ARG_LIST},
- description = "List of installed files")
- public boolean list;
-
- @Parameter(names = {ARG_RESOURCE},
- description = "Name of the file or directory")
- public String resource;
-
- @Parameter(names = {ARG_DESTDIR},
- description = "The name of the folder in which to store the resources")
- public String folder;
-
- @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing resource(s)")
- public boolean overwrite = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 3;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
deleted file mode 100644
index 5285f7b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_STATUS},
- commandDescription = SliderActions.DESCRIBE_ACTION_STATUS)
-
-public class ActionStatusArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_STATUS;
- }
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "Output file for the status information")
- public String output;
-
- @Parameter(names = {ARG_LIFETIME},
- description = "Lifetime of the application from the time of request")
- public boolean lifetime;
-
- public String getOutput() {
- return output;
- }
-
- public void setOutput(String output) {
- this.output = output;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
deleted file mode 100644
index e8bdcad..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_START },
- commandDescription = SliderActions.DESCRIBE_ACTION_THAW)
-public class ActionThawArgs extends AbstractActionArgs implements
- WaitTimeAccessor,
- LaunchArgsAccessor {
-
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_START;
- }
-
- @Override
- public int getWaittime() {
- return launchArgs.getWaittime();
- }
-
- @ParametersDelegate
- LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
-
- @Parameter(names = {ARG_LIFETIME},
- description = "Life time of the application since application started at"
- + " running state")
- public long lifetime;
-
- @Override
- public String getRmAddress() {
- return launchArgs.getRmAddress();
- }
-
- @Override
- public void setWaittime(int waittime) {
- launchArgs.setWaittime(waittime);
- }
-
-
- @Override
- public File getOutputFile() {
- return launchArgs.getOutputFile();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java
deleted file mode 100644
index f1f0125..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.UsageException;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_TOKENS},
- commandDescription = "save tokens to a file or list tokens in a file")
-public class ActionTokensArgs extends AbstractActionArgs {
-
- public static final String DUPLICATE_ARGS = "Only one of " +
- ARG_SOURCE + " and " + ARG_OUTPUT + " allowed";
-
- public static final String MISSING_KT_PROVIDER =
- "Both " + ARG_KEYTAB + " and " + ARG_PRINCIPAL
- + " must be provided";
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_TOKENS;
- }
-
- @Parameter(names = {ARG_OUTPUT},
- description = "File to write")
- public File output;
-
- @Parameter(names = {ARG_SOURCE},
- description = "source file")
- public File source;
-
- @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
- public File keytab;
-
- @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
- public String principal="";
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
- if (output != null && source != null) {
- throw new BadCommandArgumentsException(DUPLICATE_ARGS);
- }
-
- // this is actually a !xor
- if (keytab != null ^ !principal.isEmpty()) {
- throw new BadCommandArgumentsException(MISSING_KT_PROVIDER);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpdateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpdateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpdateArgs.java
deleted file mode 100644
index 830e4ee..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpdateArgs.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_UPDATE},
- commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE)
-
-public class ActionUpdateArgs extends AbstractClusterBuildingActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_UPDATE;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java
deleted file mode 100644
index b909cdd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_UPGRADE },
- commandDescription = SliderActions.DESCRIBE_ACTION_UPGRADE)
-public class ActionUpgradeArgs extends AbstractClusterBuildingActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_UPGRADE;
- }
-
-// TODO upgrade container
-// @Parameter(names={ARG_CONTAINERS}, variableArity = true,
-// description = "stop specific containers")
-// public List<String> containers = new ArrayList<>(0);
-//
-// @Parameter(names={ARG_COMPONENTS}, variableArity = true,
-// description = "stop all containers of specific components")
-// public List<String> components = new ArrayList<>(0);
-//
-// @Parameter(names = {ARG_FORCE},
-// description = "force spec upgrade operation")
-// public boolean force;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java
deleted file mode 100644
index b0f17d0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-/**
- * The version command
- */
-@Parameters(commandNames = { SliderActions.ACTION_VERSION},
- commandDescription = SliderActions.DESCRIBE_ACTION_VERSION)
-public class ActionVersionArgs extends AbstractActionArgs {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_VERSION;
- }
-
- public int getMinParams() {
- return 0;
- }
-
- /**
- * This action does not need hadoop services
- * @return false
- */
- @Override
- public boolean getHadoopServicesRequired() {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java
deleted file mode 100644
index 3ef8e19..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-public class AddonArgsDelegate extends AbstractArgsDelegate {
-
- /**
- * This is a listing of addon packages
- */
- @Parameter(names = {ARG_ADDON},
- arity = 2,
- description = "--addon <name> <folder or package>",
- splitter = DontSplitArguments.class)
- public List<String> addonTuples = new ArrayList<>(0);
-
-
- /**
- * Get the list of addons (may be empty, but never null)
- *
- * @return map of named addons
- *
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, String> getAddonMap() throws BadCommandArgumentsException {
- return convertTupleListToMap("addon", addonTuples);
- }
-
- public List<String> getAddonTuples() {
- return addonTuples;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
deleted file mode 100644
index 0344305..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.converters.IParameterSplitter;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class DontSplitArguments implements IParameterSplitter {
-
- @Override
- public List<String> split(String value) {
- List<String> list = new ArrayList<>(1);
- list.add(value);
- return list;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
deleted file mode 100644
index 7524053..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import java.io.File;
-
-/**
- * Launch args for create and start and anything else that can start something
- */
-public interface LaunchArgsAccessor extends WaitTimeAccessor {
- String getRmAddress();
-
- File getOutputFile();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
deleted file mode 100644
index bc7e94c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-
-import java.io.File;
-
-/**
- * Any launch-time args
- */
-public class LaunchArgsDelegate extends WaitArgsDelegate implements
- LaunchArgsAccessor {
-
-
- //TODO: do we need this?
- @Parameter(names = ARG_RESOURCE_MANAGER,
- description = "Resource manager hostname:port ",
- required = false)
- private String rmAddress;
-
- @Override
- public String getRmAddress() {
- return rmAddress;
- }
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for any application report")
- public File outputFile;
-
- @Override
- public File getOutputFile() {
- return outputFile;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/OptionArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/OptionArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/OptionArgsDelegate.java
deleted file mode 100644
index e63bd12..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/OptionArgsDelegate.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Delegate for application and resource options.
- */
-public class OptionArgsDelegate extends AbstractArgsDelegate {
-
- /**
- * Options key value.
- */
- @Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
- description = ARG_OPTION + "<name> <value>",
- splitter = DontSplitArguments.class)
- public List<String> optionTuples = new ArrayList<>(0);
-
-
- /**
- * All the app component option triples.
- */
- @Parameter(names = {ARG_COMP_OPT, ARG_COMP_OPT_SHORT}, arity = 3,
- description = "Component option " + ARG_COMP_OPT +
- " <component> <name> <option>",
- splitter = DontSplitArguments.class)
- public List<String> compOptTriples = new ArrayList<>(0);
-
- public Map<String, String> getOptionsMap() throws
- BadCommandArgumentsException {
- return convertTupleListToMap(ARG_OPTION, optionTuples);
- }
-
- /**
- * Get the role heap mapping (may be empty, but never null).
- * @return role heap mapping
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, Map<String, String>> getCompOptionMap()
- throws BadCommandArgumentsException {
- return convertTripleListToMaps(ARG_COMP_OPT, compOptTriples);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java
deleted file mode 100644
index ccb526c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.converters.BaseConverter;
-import org.apache.hadoop.fs.Path;
-
-public class PathArgumentConverter extends BaseConverter<Path> {
-
- public PathArgumentConverter(String optionName) {
- super(optionName);
- }
-
- @Override
- public Path convert(String value) {
- return new Path(value);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java
deleted file mode 100644
index b0d1ebf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.converters.BaseConverter;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-
-public class URIArgumentConverter extends BaseConverter<URI> {
-
- public URIArgumentConverter(String optionName) {
- super(optionName);
- }
-
- @Override
- public URI convert(String value) {
- try {
- return new URI(value);
- } catch (URISyntaxException e) {
- throw new RuntimeException("Cannot make a URI from " + value);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java
deleted file mode 100644
index 8894309..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.converters.BaseConverter;
-
-import java.net.MalformedURLException;
-import java.net.URL;
-
-public class URLArgumentConverter extends BaseConverter<URL> {
- public URLArgumentConverter(String optionName) {
- super(optionName);
- }
-
- @Override
- public URL convert(String value) {
- try {
- return new URL(value);
- } catch (MalformedURLException e) {
- throw new RuntimeException("Cannot make a URL from " + value);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java
deleted file mode 100644
index 1c27c01..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-
-public class WaitArgsDelegate extends AbstractArgsDelegate implements
- WaitTimeAccessor {
-
-
- //--wait [timeout]
- @Parameter(names = {ARG_WAIT},
- description = "time to wait for an action to complete")
- public int waittime = 0;
-
-
- @Override
- public int getWaittime() {
- return waittime;
- }
-
- @Override
- public void setWaittime(int waittime) {
- this.waittime = waittime;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java
deleted file mode 100644
index 13d4d5a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-public interface WaitTimeAccessor {
- int getWaittime();
- void setWaittime(int waittime);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
deleted file mode 100644
index a83901b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import java.io.Serializable;
-import java.util.Comparator;
-
-/**
- * Some general comparators
- */
-public class Comparators {
-
- public static class LongComparator implements Comparator<Long>, Serializable {
- @Override
- public int compare(Long o1, Long o2) {
- return o1.compareTo(o2);
- }
- }
-
- public static class InvertedLongComparator
- implements Comparator<Long>, Serializable {
- @Override
- public int compare(Long o1, Long o2) {
- return o2.compareTo(o1);
- }
- }
-
- /**
- * Little template class to reverse any comparitor
- * @param <CompareType> the type that is being compared
- */
- public static class ComparatorReverser<CompareType> implements Comparator<CompareType>,
- Serializable {
-
- final Comparator<CompareType> instance;
-
- public ComparatorReverser(Comparator<CompareType> instance) {
- this.instance = instance;
- }
-
- @Override
- public int compare(CompareType first, CompareType second) {
- return instance.compare(second, first);
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java
deleted file mode 100644
index 9c653f3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.service.ServiceStateChangeListener;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * An extended composite service which stops itself if any child service
- * fails, or when all its children have successfully stopped without failure.
- *
- * Lifecycle
- * <ol>
- * <li>If any child exits with a failure: this service stops, propagating
- * the exception.</li>
- * <li>When all child services has stopped, this service stops itself</li>
- * </ol>
- *
- */
-public class WorkflowCompositeService extends CompositeService
- implements ServiceParent, ServiceStateChangeListener {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(WorkflowCompositeService.class);
-
- /**
- * Deadlock-avoiding overridden config for slider services; see SLIDER-1052
- */
- private volatile Configuration configuration;
-
- /**
- * Construct an instance
- * @param name name of this service instance
- */
- public WorkflowCompositeService(String name) {
- super(name);
- }
-
- @Override
- public Configuration getConfig() {
- return configuration;
- }
-
- @Override
- protected void setConfig(Configuration conf) {
- super.setConfig(conf);
- configuration = conf;
- }
-
- /**
- * Construct an instance with the default name.
- */
- public WorkflowCompositeService() {
- this("WorkflowCompositeService");
- }
-
- /**
- * Varargs constructor
- * @param name name of this service instance
- * @param children children
- */
- public WorkflowCompositeService(String name, Service... children) {
- this(name);
- for (Service child : children) {
- addService(child);
- }
- }
-
- /**
- * Construct with a list of children
- * @param name name of this service instance
- * @param children children to add
- */
- public WorkflowCompositeService(String name, List<Service> children) {
- this(name);
- for (Service child : children) {
- addService(child);
- }
- }
-
- /**
- * Add a service, and register it
- * @param service the {@link Service} to be added.
- * Important: do not add a service to a parent during your own serviceInit/start,
- * in Hadoop 2.2; you will trigger a ConcurrentModificationException.
- */
- @Override
- public synchronized void addService(Service service) {
- Preconditions.checkArgument(service != null, "null service argument");
- service.registerServiceListener(this);
- super.addService(service);
- }
-
- /**
- * When this service is started, any service stopping with a failure
- * exception is converted immediately into a failure of this service,
- * storing the failure and stopping ourselves.
- * @param child the service that has changed.
- */
- @Override
- public void stateChanged(Service child) {
- //if that child stopped while we are running:
- if (isInState(STATE.STARTED) && child.isInState(STATE.STOPPED)) {
- // a child service has stopped
- //did the child fail? if so: propagate
- Throwable failureCause = child.getFailureCause();
- if (failureCause != null) {
- LOG.info("Child service " + child + " failed", failureCause);
- //failure. Convert to an exception
- Exception e = (failureCause instanceof Exception) ?
- (Exception) failureCause : new Exception(failureCause);
- //flip ourselves into the failed state
- noteFailure(e);
- stop();
- } else {
- LOG.info("Child service completed {}", child);
- if (areAllChildrenStopped()) {
- LOG.info("All children are halted: stopping");
- stop();
- }
- }
- }
- }
-
- /**
- * Probe to query if all children are stopped -simply
- * by taking a snapshot of the child service list and enumerating
- * their state.
- * The state of the children may change during this operation -that will
- * not get picked up.
- * @return true if all the children are stopped.
- */
- private boolean areAllChildrenStopped() {
- List<Service> children = getServices();
- boolean stopped = true;
- for (Service child : children) {
- if (!child.isInState(STATE.STOPPED)) {
- stopped = false;
- break;
- }
- }
- return stopped;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowExecutorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowExecutorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowExecutorService.java
deleted file mode 100644
index 7409d32..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowExecutorService.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.service.AbstractService;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-
-/**
- * A service that hosts an executor -when the service is stopped,
- * {@link ExecutorService#shutdownNow()} is invoked.
- */
-public class WorkflowExecutorService<E extends ExecutorService> extends AbstractService {
-
- private E executor;
-
- /**
- * Construct an instance with the given name -but
- * no executor
- * @param name service name
- */
- public WorkflowExecutorService(String name) {
- this(name, null);
- }
-
- /**
- * Construct an instance with the given name and executor
- * @param name service name
- * @param executor exectuor
- */
- public WorkflowExecutorService(String name,
- E executor) {
- super(name);
- this.executor = executor;
- }
-
- /**
- * Get the executor
- * @return the executor
- */
- public synchronized E getExecutor() {
- return executor;
- }
-
- /**
- * Set the executor. Only valid if the current one is null
- * @param executor executor
- */
- public synchronized void setExecutor(E executor) {
- Preconditions.checkState(this.executor == null,
- "Executor already set");
- this.executor = executor;
- }
-
- /**
- * Execute the runnable with the executor (which
- * must have been created already)
- * @param runnable runnable to execute
- */
- public void execute(Runnable runnable) {
- getExecutor().execute(runnable);
- }
-
- /**
- * Submit a callable
- * @param callable callable
- * @param <V> type of the final get
- * @return a future to wait on
- */
- public <V> Future<V> submit(Callable<V> callable) {
- return getExecutor().submit(callable);
- }
-
- /**
- * Stop the service: halt the executor.
- * @throws Exception exception.
- */
- @Override
- protected void serviceStop() throws Exception {
- stopExecutor();
- super.serviceStop();
- }
-
- /**
- * Stop the executor if it is not null.
- * This uses {@link ExecutorService#shutdownNow()}
- * and so does not block until they have completed.
- */
- protected synchronized void stopExecutor() {
- if (executor != null) {
- executor.shutdownNow();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowRpcService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowRpcService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowRpcService.java
deleted file mode 100644
index b71530f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowRpcService.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.service.AbstractService;
-
-import java.net.InetSocketAddress;
-
-/**
- * A YARN service that maps the start/stop lifecycle of an RPC server
- * to the YARN service lifecycle.
- */
-public class WorkflowRpcService extends AbstractService {
-
- /** RPC server*/
- private final Server server;
-
- /**
- * Construct an instance
- * @param name service name
- * @param server service to stop
- */
- public WorkflowRpcService(String name, Server server) {
- super(name);
- Preconditions.checkArgument(server != null, "Null server");
- this.server = server;
- }
-
- /**
- * Get the server
- * @return the server
- */
- public Server getServer() {
- return server;
- }
-
- /**
- * Get the socket address of this server
- * @return the address this server is listening on
- */
- public InetSocketAddress getConnectAddress() {
- return NetUtils.getConnectAddress(server);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
- server.start();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (server != null) {
- server.stop();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowScheduledExecutorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowScheduledExecutorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowScheduledExecutorService.java
deleted file mode 100644
index e9f53ed..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowScheduledExecutorService.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import java.util.concurrent.ScheduledExecutorService;
-
-/**
- * Scheduled executor or subclass thereof
- * @param <E> scheduled executor service type
- */
-public class WorkflowScheduledExecutorService<E extends ScheduledExecutorService>
- extends WorkflowExecutorService<E> {
-
- public WorkflowScheduledExecutorService(String name) {
- super(name);
- }
-
- public WorkflowScheduledExecutorService(String name,
- E executor) {
- super(name, executor);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowSequenceService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowSequenceService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowSequenceService.java
deleted file mode 100644
index 97f97e8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowSequenceService.java
+++ /dev/null
@@ -1,306 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.service.ServiceStateChangeListener;
-import org.apache.hadoop.service.ServiceStateException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * This resembles the YARN CompositeService, except that it
- * starts one service after another
- *
- * Workflow
- * <ol>
- * <li>When the <code>WorkflowSequenceService</code> instance is
- * initialized, it only initializes itself.</li>
- *
- * <li>When the <code>WorkflowSequenceService</code> instance is
- * started, it initializes then starts the first of its children.
- * If there are no children, it immediately stops.</li>
- *
- * <li>When the active child stops, it did not fail, and the parent has not
- * stopped -then the next service is initialized and started. If there is no
- * remaining child the parent service stops.</li>
- *
- * <li>If the active child did fail, the parent service notes the exception
- * and stops -effectively propagating up the failure.
- * </li>
- * </ol>
- *
- * New service instances MAY be added to a running instance -but no guarantees
- * can be made as to whether or not they will be run.
- */
-
-public class WorkflowSequenceService extends AbstractService implements
- ServiceParent, ServiceStateChangeListener {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(WorkflowSequenceService.class);
-
- /**
- * list of services
- */
- private final List<Service> serviceList = new ArrayList<>();
-
- /**
- * The currently active service.
- * Volatile -may change & so should be read into a
- * local variable before working with
- */
- private volatile Service activeService;
-
- /**
- the previous service -the last one that finished.
- null if one did not finish yet
- */
- private volatile Service previousService;
-
- private boolean stopIfNoChildServicesAtStartup = true;
-
- /**
- * Construct an instance
- * @param name service name
- */
- public WorkflowSequenceService(String name) {
- super(name);
- }
-
- /**
- * Construct an instance with the default name
- */
- public WorkflowSequenceService() {
- this("WorkflowSequenceService");
- }
-
- /**
- * Create a service sequence with the given list of services
- * @param name service name
- * @param children initial sequence
- */
- public WorkflowSequenceService(String name, Service... children) {
- super(name);
- for (Service service : children) {
- addService(service);
- }
- } /**
- * Create a service sequence with the given list of services
- * @param name service name
- * @param children initial sequence
- */
- public WorkflowSequenceService(String name, List<Service> children) {
- super(name);
- for (Service service : children) {
- addService(service);
- }
- }
-
- /**
- * Get the current service -which may be null
- * @return service running
- */
- public Service getActiveService() {
- return activeService;
- }
-
- /**
- * Get the previously active service
- * @return the service last run, or null if there is none.
- */
- public Service getPreviousService() {
- return previousService;
- }
-
- protected void setStopIfNoChildServicesAtStartup(boolean stopIfNoChildServicesAtStartup) {
- this.stopIfNoChildServicesAtStartup = stopIfNoChildServicesAtStartup;
- }
-
- /**
- * When started
- * @throws Exception
- */
- @Override
- protected void serviceStart() throws Exception {
- if (!startNextService() && stopIfNoChildServicesAtStartup) {
- //nothing to start -so stop
- stop();
- }
- }
-
- @Override
- protected void serviceStop() throws Exception {
- //stop current service.
- //this triggers a callback that is caught and ignored
- Service current = activeService;
- previousService = current;
- activeService = null;
- if (current != null) {
- current.stop();
- }
- }
-
- /**
- * Start the next service in the list.
- * Return false if there are no more services to run, or this
- * service has stopped
- * @return true if a service was started
- * @throws RuntimeException from any init or start failure
- * @throws ServiceStateException if this call is made before
- * the service is started
- */
- public synchronized boolean startNextService() {
- if (isInState(STATE.STOPPED)) {
- //downgrade to a failed
- LOG.debug("Not starting next service -{} is stopped", this);
- return false;
- }
- if (!isInState(STATE.STARTED)) {
- //reject attempts to start a service too early
- throw new ServiceStateException(
- "Cannot start a child service when not started");
- }
- if (serviceList.isEmpty()) {
- //nothing left to run
- return false;
- }
- if (activeService != null && activeService.getFailureCause() != null) {
- //did the last service fail? Is this caused by some premature callback?
- LOG.debug("Not starting next service due to a failure of {}",
- activeService);
- return false;
- }
- //bear in mind that init & start can fail, which
- //can trigger re-entrant calls into the state change listener.
- //by setting the current service to null
- //the start-next-service logic is skipped.
- //now, what does that mean w.r.t exit states?
-
- activeService = null;
- Service head = serviceList.remove(0);
-
- try {
- head.init(getConfig());
- head.registerServiceListener(this);
- head.start();
- } catch (RuntimeException e) {
- noteFailure(e);
- throw e;
- }
- //at this point the service must have explicitly started & not failed,
- //else an exception would have been raised
- activeService = head;
- return true;
- }
-
- /**
- * State change event relays service stop events to
- * {@link #onServiceCompleted(Service)}. Subclasses can
- * extend that with extra logic
- * @param service the service that has changed.
- */
- @Override
- public void stateChanged(Service service) {
- // only react to the state change when it is the current service
- // and it has entered the STOPPED state
- if (service == activeService && service.isInState(STATE.STOPPED)) {
- onServiceCompleted(service);
- }
- }
-
- /**
- * handler for service completion: base class starts the next service
- * @param service service that has completed
- */
- protected synchronized void onServiceCompleted(Service service) {
- LOG.info("Running service stopped: {}", service);
- previousService = activeService;
-
-
- //start the next service if we are not stopped ourselves
- if (isInState(STATE.STARTED)) {
-
- //did the service fail? if so: propagate
- Throwable failureCause = service.getFailureCause();
- if (failureCause != null) {
- Exception e = (failureCause instanceof Exception) ?
- (Exception) failureCause : new Exception(failureCause);
- noteFailure(e);
- stop();
- }
-
- //start the next service
- boolean started;
- try {
- started = startNextService();
- } catch (Exception e) {
- //something went wrong here
- noteFailure(e);
- started = false;
- }
- if (!started) {
- //no start because list is empty
- //stop and expect the notification to go upstream
- stop();
- }
- } else {
- //not started, so just note that the current service
- //has gone away
- activeService = null;
- }
- }
-
- /**
- * Add the passed {@link Service} to the list of services managed by this
- * {@link WorkflowSequenceService}
- * @param service the {@link Service} to be added
- */
- @Override
- public synchronized void addService(Service service) {
- Preconditions.checkArgument(service != null, "null service argument");
- LOG.debug("Adding service {} ", service.getName());
- synchronized (serviceList) {
- serviceList.add(service);
- }
- }
-
- /**
- * Get an unmodifiable list of services
- * @return a list of child services at the time of invocation -
- * added services will not be picked up.
- */
- @Override //Parent
- public synchronized List<Service> getServices() {
- return Collections.unmodifiableList(serviceList);
- }
-
- @Override // Object
- public synchronized String toString() {
- return super.toString() + "; current service " + activeService
- + "; queued service count=" + serviceList.size();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/package-info.java
deleted file mode 100644
index 36d059a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/package-info.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-/**
-
-<p>
- This package contains classes which can be aggregated to build up
- complex workflows of services: sequences of operations, callbacks
- and composite services with a shared lifespan.
- </p>
-
-<h2>
- Core concepts:
-</h2>
-
-
-<p>
-The Workflow Services are set of Hadoop YARN services, all implementing
-the {@link org.apache.hadoop.service.Service} API.
-They are designed to be aggregated, to be composed to produce larger
-composite services which than perform ordered operations, notify other services
-when work has completed, and to propagate failure up the service hierarchy.
-</p>
-<p>
-Service instances may a limited lifespan, and may self-terminate when
-they consider it appropriate.</p>
-<p>
-Workflow Services that have children implement the
-{@link org.apache.slider.server.services.workflow.ServiceParent}
-class, which provides (thread-safe) access to the children -allowing new children
-to be added, and existing children to be ennumerated. The implement policies
-on how to react to the termination of children -so can sequence operations
-which terminate themselves when complete.
-</p>
-
-<p>
-Workflow Services may be subclassed to extend their behavior, or to use them
-in specific applications. Just as the standard
-{@link org.apache.hadoop.service.CompositeService}
-is often subclassed to aggregate child services, the
-{@link org.apache.slider.server.services.workflow.WorkflowCompositeService}
-can be used instead -adding the feature that failing services trigger automatic
-parent shutdown. If that is the desired operational mode of a class,
-swapping the composite service implementation may be sufficient to adopt it.
-</p>
-
-
-<h2> How do the workflow services differ from the standard YARN services? </h2>
-
- <p>
-
- There is exactly one standard YARN service for managing children, the
- {@link org.apache.hadoop.service.CompositeService}.
- </p><p>
- The {@link org.apache.slider.server.services.workflow.WorkflowCompositeService}
- shares the same model of "child services, all inited and started together".
- Where it differs is that if any child service stops -either due to a failure
- or to an action which invokes that service's
- {@link org.apache.hadoop.service.Service#stop()} method.
- </p>
- <p>
-
-In contrast, the original <code>CompositeService</code> class starts its children
-in its{@link org.apache.hadoop.service.Service#start()} method, but does not
-listen or react to any child service halting. As a result, changes in child
-state are not automatically detected or propagated, other than failures in
-the actual init() and start() methods.
-</p>
-
-<p>
-If a child service runs until completed -that is it will not be stopped until
-instructed to do so, and if it is only the parent service that attempts to
-stop the child, then this difference is unimportant.
-</p>
-<p>
-
-However, if any service that depends upon all it child services running -
-and if those child services are written so as to stop when they fail, using
-the <code>WorkflowCompositeService</code> as a base class will enable the
-parent service to be automatically notified of a child stopping.
-
-</p>
-<p>
-The {@link org.apache.slider.server.services.workflow.WorkflowSequenceService}
-resembles the composite service in API, but its workflow is different. It
-initializes and starts its children one-by-one, only starting the second after
-the first one succeeds, the third after the second, etc. If any service in
-the sequence fails, the parent <code>WorkflowSequenceService</code> stops,
-reporting the same exception.
-</p>
-
-<p>
-The {@link org.apache.slider.server.services.workflow.ForkedProcessService}:
-Executes a process when started, and binds to the life of that process. When the
-process terminates, so does the service -and vice versa. This service enables
-external processes to be executed as part of a sequence of operations -or,
-using the {@link org.apache.slider.server.services.workflow.WorkflowCompositeService}
-in parallel with other services, terminating the process when the other services
-stop -and vice versa.
-</p>
-
-<p>
-The {@link org.apache.slider.server.services.workflow.WorkflowCallbackService}
-executes a {@link java.util.concurrent.Callable} callback a specified delay
-after the service is started, then potentially terminates itself.
-This is useful for callbacks when a workflow reaches a specific point
--or simply for executing arbitrary code in the workflow.
-
- </p>
-
-
-<h2>
-Other Workflow Services
-</h2>
-
-There are some minor services that have proven useful within aggregate workflows,
-and simply in applications which are built from composite YARN services.
-
- <ul>
- <li>{@link org.apache.slider.server.services.workflow.WorkflowRpcService }:
- Maintains a reference to an RPC {@link org.apache.hadoop.ipc.Server} instance.
- When the service is started, so is the RPC server. Similarly, when the service
- is stopped, so is the RPC server instance.
- </li>
-
- <li>{@link org.apache.slider.server.services.workflow.ClosingService}: Closes
- an instance of {@link java.io.Closeable} when the service is stopped. This
- is purely a housekeeping class.
- </li>
-
- </ul>
-
- Lower-level classes
- <ul>
- <li>{@link org.apache.slider.server.services.workflow.ServiceTerminatingRunnable }:
- A {@link java.lang.Runnable} which runs the runnable supplied in its constructor
- then signals its owning service to stop once that runnable is completed.
- Any exception raised in the run is stored.
- </li>
- <li>{@link org.apache.slider.server.services.workflow.WorkflowExecutorService}:
- A base class for services that wish to have a {@link java.util.concurrent.ExecutorService}
- with a lifespan mapped to that of a service. When the service is stopped, the
- {@link java.util.concurrent.ExecutorService#shutdownNow()} method is called to
- attempt to shut down all running tasks.
- </li>
- <li>{@link org.apache.slider.server.services.workflow.ServiceThreadFactory}:
- This is a simple {@link java.util.concurrent.ThreadFactory} which generates
- meaningful thread names. It can be used as a parameter to constructors of
- {@link java.util.concurrent.ExecutorService} instances, to ensure that
- log information can tie back text to the related services</li>
- </ul>
-
-
-
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
deleted file mode 100644
index 76ce7a5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.yarnregistry;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.registry.client.api.BindFlags;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceId;
-import org.apache.slider.common.tools.SliderUtils;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join;
-
-/**
- * Registry view for providers. This tracks where the service
- * is registered, offers access to the record and other things.
- */
-public class YarnRegistryViewForProviders {
- private static final Log LOG =
- LogFactory.getLog(YarnRegistryViewForProviders.class);
-
- private final RegistryOperations registryOperations;
- private final String user;
- private final String sliderServiceClass;
- private final String instanceName;
- /**
- * Record used where the service registered itself.
- * Null until the service is registered
- */
- private ServiceRecord selfRegistration;
-
- /**
- * Path where record was registered
- * Null until the service is registered
- */
- private String selfRegistrationPath;
-
- public YarnRegistryViewForProviders(RegistryOperations registryOperations,
- String user,
- String sliderServiceClass,
- String instanceName,
- ApplicationAttemptId applicationAttemptId) {
- Preconditions.checkArgument(registryOperations != null,
- "null registry operations");
- Preconditions.checkArgument(user != null, "null user");
- Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass),
- "unset service class");
- Preconditions.checkArgument(SliderUtils.isSet(instanceName),
- "instanceName");
- Preconditions.checkArgument(applicationAttemptId != null,
- "null applicationAttemptId");
- this.registryOperations = registryOperations;
- this.user = user;
- this.sliderServiceClass = sliderServiceClass;
- this.instanceName = instanceName;
- }
-
- public String getUser() {
- return user;
- }
-
-
- private void setSelfRegistration(ServiceRecord selfRegistration) {
- this.selfRegistration = selfRegistration;
- }
-
- /**
- * Get the path to where the service has registered itself.
- * Null until the service is registered
- * @return the service registration path.
- */
- public String getSelfRegistrationPath() {
- return selfRegistrationPath;
- }
-
- /**
- * Get the absolute path to where the service has registered itself.
- * This includes the base registry path
- * Null until the service is registered
- * @return the service registration path.
- */
- public String getAbsoluteSelfRegistrationPath() {
- if (selfRegistrationPath == null) {
- return null;
- }
- String root = registryOperations.getConfig().getTrimmed(
- RegistryConstants.KEY_REGISTRY_ZK_ROOT,
- RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
- return RegistryPathUtils.join(root, selfRegistrationPath);
- }
-
- /**
- * Add a component under the slider name/entry
- * @param componentName component name
- * @param record record to put
- * @throws IOException
- */
- public void putComponent(String componentName,
- ServiceRecord record) throws
- IOException {
- putComponent(sliderServiceClass, instanceName,
- componentName,
- record);
- }
-
- /**
- * Add a component
- * @param serviceClass service class to use under ~user
- * @param componentName component name
- * @param record record to put
- * @throws IOException
- */
- public void putComponent(String serviceClass,
- String serviceName,
- String componentName,
- ServiceRecord record) throws IOException {
- String path = RegistryUtils.componentPath(
- user, serviceClass, serviceName, componentName);
- registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
- registryOperations.bind(path, record, BindFlags.OVERWRITE);
- }
-
- /**
- * Add a service under a path, optionally purging any history
- * @param username user
- * @param serviceClass service class to use under ~user
- * @param serviceName name of the service
- * @param record service record
- * @param deleteTreeFirst perform recursive delete of the path first.
- * @return the path the service was created at
- * @throws IOException
- */
- public String putService(String username,
- String serviceClass,
- String serviceName,
- ServiceRecord record,
- boolean deleteTreeFirst) throws IOException {
- String path = RegistryUtils.servicePath(
- username, serviceClass, serviceName);
- if (deleteTreeFirst) {
- registryOperations.delete(path, true);
- }
- registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
- registryOperations.bind(path, record, BindFlags.OVERWRITE);
- return path;
- }
-
- /**
- * Add a service under a path for the current user
- * @param record service record
- * @param deleteTreeFirst perform recursive delete of the path first
- * @return the path the service was created at
- * @throws IOException
- */
- public String registerSelf(
- ServiceRecord record,
- boolean deleteTreeFirst) throws IOException {
- selfRegistrationPath =
- putService(user, sliderServiceClass, instanceName, record, deleteTreeFirst);
- setSelfRegistration(record);
- return selfRegistrationPath;
- }
-
- /**
- * Delete a component
- * @param containerId component name
- * @throws IOException
- */
- public void deleteComponent(ComponentInstanceId instanceId,
- String containerId) throws IOException {
- String path = RegistryUtils.componentPath(
- user, sliderServiceClass, instanceName,
- containerId);
- LOG.info(instanceId + ": Deleting registry path " + path);
- registryOperations.delete(path, false);
- }
-
- /**
- * Delete the children of a path -but not the path itself.
- * It is not an error if the path does not exist
- * @param path path to delete
- * @param recursive flag to request recursive deletes
- * @throws IOException IO problems
- */
- public void deleteChildren(String path, boolean recursive) throws IOException {
- List<String> childNames = null;
- try {
- childNames = registryOperations.list(path);
- } catch (PathNotFoundException e) {
- return;
- }
- for (String childName : childNames) {
- String child = join(path, childName);
- registryOperations.delete(child, recursive);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java
deleted file mode 100644
index daaf0e9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.util;
-
-public interface RestApiConstants {
- String CONTEXT_ROOT = "/services/v1";
- String APPLICATIONS_API_RESOURCE_PATH = "/applications";
- String CONTAINERS_API_RESOURCE_PATH = "/containers";
- String SLIDER_APPMASTER_COMPONENT_NAME = "slider-appmaster";
- String SLIDER_CONFIG_SCHEMA = "http://example.org/specification/v2.0.0";
- String METAINFO_SCHEMA_VERSION = "2.1";
- String COMPONENT_TYPE_YARN_DOCKER = "yarn_docker";
-
- String DEFAULT_START_CMD = "/bootstrap/privileged-centos6-sshd";
- String DEFAULT_COMPONENT_NAME = "default";
- String DEFAULT_IMAGE = "centos:centos6";
- String DEFAULT_NETWORK = "bridge";
- String DEFAULT_COMMAND_PATH = "/usr/bin/docker";
- String DEFAULT_USE_NETWORK_SCRIPT = "yes";
-
- String PLACEHOLDER_APP_NAME = "${APP_NAME}";
- String PLACEHOLDER_APP_COMPONENT_NAME = "${APP_COMPONENT_NAME}";
- String PLACEHOLDER_COMPONENT_ID = "${COMPONENT_ID}";
-
- String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST";
- String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT";
- String PROPERTY_APP_LIFETIME = "docker.lifetime";
- String PROPERTY_APP_RUNAS_USER = "APP_RUNAS_USER";
- Long DEFAULT_UNLIMITED_LIFETIME = -1l;
-
- Integer HTTP_STATUS_CODE_ACCEPTED = 202;
- String ARTIFACT_TYPE_SLIDER_ZIP = "slider-zip";
-
- Integer GET_APPLICATIONS_THREAD_POOL_SIZE = 200;
-
- String PROPERTY_PYTHON_PATH = "python.path";
- String PROPERTY_DNS_DEPENDENCY = "site.global.dns.dependency";
-
- String COMMAND_ORDER_SUFFIX_START = "-START";
- String COMMAND_ORDER_SUFFIX_STARTED = "-RUNNING_BUT_UNREADY";
- String EXPORT_GROUP_NAME = "QuickLinks";
-
- Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001;
- Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002;
- Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003;
- Integer ERROR_CODE_APP_NAME_INVALID = 404004;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
deleted file mode 100644
index 74f7e06..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.util;
-
-public interface RestApiErrorMessages {
- String ERROR_APPLICATION_NAME_INVALID =
- "Application name is either empty or not provided";
- String ERROR_APPLICATION_NAME_INVALID_FORMAT =
- "Application name %s is not valid - only lower case letters, digits, " +
- "underscore and hyphen are allowed, and the name must be no more " +
- "than 63 characters";
- String ERROR_COMPONENT_NAME_INVALID =
- "Component name must be no more than %s characters: %s";
- String ERROR_USER_NAME_INVALID =
- "User name must be no more than 63 characters";
-
- String ERROR_APPLICATION_NOT_RUNNING = "Application not running";
- String ERROR_APPLICATION_DOES_NOT_EXIST = "Application not found";
- String ERROR_APPLICATION_IN_USE = "Application already exists in started"
- + " state";
- String ERROR_APPLICATION_INSTANCE_EXISTS = "Application already exists in"
- + " stopped/failed state (either restart with PUT or destroy with DELETE"
- + " before creating a new one)";
-
- String ERROR_SUFFIX_FOR_COMPONENT =
- " for component %s (nor at the global level)";
- String ERROR_ARTIFACT_INVALID = "Artifact is not provided";
- String ERROR_ARTIFACT_FOR_COMP_INVALID =
- ERROR_ARTIFACT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
- String ERROR_ARTIFACT_ID_INVALID =
- "Artifact id (like docker image name) is either empty or not provided";
- String ERROR_ARTIFACT_ID_FOR_COMP_INVALID =
- ERROR_ARTIFACT_ID_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
-
- String ERROR_RESOURCE_INVALID = "Resource is not provided";
- String ERROR_RESOURCE_FOR_COMP_INVALID =
- ERROR_RESOURCE_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
- String ERROR_RESOURCE_MEMORY_INVALID =
- "Application resource or memory not provided";
- String ERROR_RESOURCE_CPUS_INVALID =
- "Application resource or cpus not provided";
- String ERROR_RESOURCE_CPUS_INVALID_RANGE =
- "Unacceptable no of cpus specified, either zero or negative";
- String ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID =
- ERROR_RESOURCE_MEMORY_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
- String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID =
- ERROR_RESOURCE_CPUS_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
- String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE =
- ERROR_RESOURCE_CPUS_INVALID_RANGE
- + " for component %s (or at the global level)";
- String ERROR_CONTAINERS_COUNT_INVALID =
- "Invalid no of containers specified";
- String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID =
- ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
- String ERROR_DEPENDENCY_INVALID = "Dependency %s for component %s is " +
- "invalid, does not exist as a component";
- String ERROR_DEPENDENCY_CYCLE = "Invalid dependencies, a cycle may " +
- "exist: %s";
-
- String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED =
- "Cannot specify" + " cpus/memory along with profile";
- String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED =
- ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED
- + " for component %s";
- String ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET =
- "Resource profile is not " + "supported yet. Please specify cpus/memory.";
-
- String ERROR_NULL_ARTIFACT_ID =
- "Artifact Id can not be null if artifact type is none";
- String ERROR_ABSENT_NUM_OF_INSTANCE =
- "Num of instances should appear either globally or per component";
- String ERROR_ABSENT_LAUNCH_COMMAND =
- "Launch_command is required when type is not DOCKER";
-
- String ERROR_QUICKLINKS_FOR_COMP_INVALID = "Quicklinks specified at"
- + " component level, needs corresponding values set at application level";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
deleted file mode 100644
index 691f861..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
+++ /dev/null
@@ -1,392 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.slider.api.proto";
-option java_outer_classname = "Messages";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package org.apache.slider.api;
-
-//import "Security.proto";
-
-/*
- Look at SliderClusterProtocol.proto to see how to build this
-*/
-
-message RoleInstanceState {
- required string name = 1;
- optional string role = 2;
- required uint32 state = 4;
- required uint32 exitCode = 5;
- optional string command = 6;
- optional string diagnostics = 7;
- repeated string output = 8;
- repeated string environment = 9;
- required uint32 roleId = 10;
- required bool released = 11;
- required int64 createTime = 12;
- required int64 startTime = 13;
- required string host = 14;
- required string hostURL = 15;
- optional string appVersion = 16;
-}
-
-/**
- * stop the cluster
- */
-message StopClusterRequestProto {
- /**
- message to include
- */
- required string message = 1;
-}
-
-/**
- * stop the cluster
- */
-message StopClusterResponseProto {
-}
-
-/**
- * upgrade the containers
- */
-message UpgradeContainersRequestProto {
- /**
- message to include
- */
- required string message = 1;
- repeated string container = 2;
- repeated string component = 3;
-}
-
-/**
- * upgrade the containers
- */
-message UpgradeContainersResponseProto {
-}
-
-message FlexComponentsRequestProto {
- repeated ComponentCountProto components = 1;
-}
-
-message ComponentCountProto {
- optional string name = 1;
- optional int64 numberOfContainers = 2;
-}
-
-message FlexComponentsResponseProto {
-}
-
-/**
- * void request
- */
-message GetJSONClusterStatusRequestProto {
-}
-
-/**
- * response
- */
-message GetJSONClusterStatusResponseProto {
- required string clusterSpec = 1;
-}
-
-/**
- * list the nodes in a role
- */
-message ListNodeUUIDsByRoleRequestProto {
- required string role = 1;
-}
-
-/**
- * list the nodes in a role
- */
-message ListNodeUUIDsByRoleResponseProto {
- repeated string uuid = 1 ;
-}
-
-/**
- * get a node
- */
-message GetNodeRequestProto {
- required string uuid = 1;
-}
-
-
-/**
- * response on a node
- */
-message GetNodeResponseProto {
- required RoleInstanceState clusterNode = 1 ;
-}
-
-/**
- * list the nodes for the UUDs
- */
-message GetClusterNodesRequestProto {
- repeated string uuid = 1 ;
-}
-
-/**
- * list the nodes in a role
- */
-message GetClusterNodesResponseProto {
- repeated RoleInstanceState clusterNode = 1 ;
-}
-
-/**
- * Echo
- */
-message EchoRequestProto {
- required string text = 1;
-}
-
-/**
- * Echo reply
- */
-message EchoResponseProto {
- required string text = 1;
-}
-
-
-/**
- * Kill a container
- */
-message KillContainerRequestProto {
- required string id = 1;
-}
-
-/**
- * Kill reply
- */
-message KillContainerResponseProto {
- required bool success = 1;
-}
-
-/**
- * AM suicide
- */
-message AMSuicideRequestProto {
- required string text = 1;
- required int32 signal = 2;
- required int32 delay = 3;
-}
-
-/**
- * AM suicide reply. For this to be returned implies
- * a failure of the AM to kill itself
- */
-message AMSuicideResponseProto {
-
-}
-
-
-/**
- * Ask for the instance definition details
- */
-message GetInstanceDefinitionRequestProto {
-
-}
-
-/**
- * Get the definition back as three separate JSON strings
- */
-message GetInstanceDefinitionResponseProto {
- required string internal = 1;
- required string resources = 2;
- required string application = 3;
-}
-
-
- /* ************************************************************************
-
- REST model and operations.
- Below here the operations and payloads designed to mimic
- the REST API. That API is now the source of those
- specificatations; this is simply a derivative.
-
- **************************************************************************/
-
-/**
- * See org.apache.slider.api.types.ApplicationLivenessInformation
- */
-message ApplicationLivenessInformationProto {
- optional bool allRequestsSatisfied = 1;
- optional int32 requestsOutstanding = 2;
-}
-
-/*
- * see org.apache.slider.api.types.ComponentInformation
- */
-message ComponentInformationProto {
- optional string name = 1;
- optional int32 priority = 2;
- optional int32 desired = 3;
- optional int32 actual = 4;
- optional int32 releasing = 5;
- optional int32 requested = 6;
- optional int32 failed = 7;
- optional int32 started = 8;
- optional int32 startFailed = 9;
- optional int32 completed = 10;
- optional int32 totalRequested = 11;
- optional string failureMessage =12;
- optional int32 placementPolicy =13;
- repeated string containers = 14;
- optional int32 failedRecently = 15;
- optional int32 nodeFailed = 16;
- optional int32 preempted = 17;
- optional int32 pendingAntiAffineRequestCount = 18;
- optional bool isAARequestOutstanding = 19;
-}
-
-/*
- * see org.apache.slider.api.types.ContainerInformation
- */
-message ContainerInformationProto {
- optional string containerId = 1;
- optional string component = 2;
- optional bool released = 3;
- optional int32 state = 4;
- optional int32 exitCode = 5;
- optional string diagnostics = 6;
- optional int64 createTime = 7;
- optional int64 startTime = 8;
- repeated string output = 9;
- optional string host = 10;
- optional string hostURL = 11;
- optional string placement = 12;
- optional string appVersion = 13;
-}
-
-
-/*
- * see org.apache.slider.api.types.PingInformation
- */
-message PingInformationProto {
- optional string text = 1;
- optional string verb = 2;
- optional string body = 3;
- optional int64 time = 4;
-}
-
-message NodeEntryInformationProto {
- required int32 priority = 1;
- required int32 requested = 2;
- required int32 starting = 3;
- required int32 startFailed = 4;
- required int32 failed = 5;
- required int32 failedRecently= 6;
- required int32 preempted = 7;
- required int32 live = 8;
- required int32 releasing = 9;
- required int64 lastUsed = 10;
- required string name = 11;
-}
-
-message NodeInformationProto {
- required string hostname = 1;
- required string state = 2;
- required string httpAddress = 3;
- required string rackName = 4;
- required string labels = 5;
- required string healthReport= 6;
- required int64 lastUpdated = 7;
- repeated NodeEntryInformationProto entries = 8;
-}
-
-message GetModelRequestProto {
-}
-
-message GetModelDesiredRequestProto {
-}
-
-message GetModelDesiredAppconfRequestProto {
-}
-
-message GetModelDesiredResourcesRequestProto {
-}
-
-message GetModelResolvedAppconfRequestProto {
-}
-
-message GetModelResolvedResourcesRequestProto {
-}
-
-message GetModelLiveResourcesRequestProto {
-}
-
-message GetLiveContainersRequestProto {
-}
-
-message GetLiveContainersResponseProto {
- repeated string names = 1;
- repeated ContainerInformationProto containers = 2;
-}
-
-message GetLiveContainerRequestProto {
- required string containerId = 1;
-}
-
-
-message GetLiveComponentsRequestProto {
-}
-
-message GetLiveComponentsResponseProto {
-
- repeated string names = 1;
- repeated ComponentInformationProto components = 2;
-}
-
-message GetLiveComponentRequestProto {
- required string name = 1;
-}
-
-message GetApplicationLivenessRequestProto {
-}
-
-message EmptyPayloadProto {
-}
-
-/**
- Generic JSON, often containing data structures serialized as a string
-*/
-message WrappedJsonProto {
- required string json = 1;
-}
-
-message GetCertificateStoreRequestProto {
- optional string hostname = 1;
- required string requesterId = 2;
- required string password = 3;
- required string type = 4;
-}
-
-message GetCertificateStoreResponseProto {
- required bytes store = 1;
-}
-
-message GetLiveNodesRequestProto {
-}
-
-message GetLiveNodesResponseProto {
- repeated NodeInformationProto nodes = 1;
-}
-
-message GetLiveNodeRequestProto {
- required string name = 1;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
deleted file mode 100644
index 776ce28..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.slider.api.proto";
-option java_outer_classname = "SliderClusterAPI";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package org.apache.slider.api;
-
-/*
-
-Compiling
-
-Maven: How to do it as part of the build
- mvn install -DskipTests -Pcompile-protobuf
-
-How to do it so as to get error messages
-
-protoc --java_out=src/main/java \
- -Isrc/main/proto src/main/proto/SliderClusterMessages.proto \
- src/main/proto/SliderClusterProtocol.proto
-
-Once happy: commit the changes
-
-*/
-
-//import "Security.proto";
-import "SliderClusterMessages.proto";
-
-
-/**
- * Protocol used from between Slider Client and AM
- */
-service SliderClusterProtocolPB {
-
- /**
- * Stop the cluster
- */
-
- rpc stopCluster(StopClusterRequestProto)
- returns(StopClusterResponseProto);
-
- /**
- * Upgrade containers
- */
- rpc upgradeContainers(UpgradeContainersRequestProto)
- returns(UpgradeContainersResponseProto);
-
- rpc flexComponents(FlexComponentsRequestProto) returns (FlexComponentsResponseProto);
-
- /**
- * Get the current cluster status
- */
- rpc getJSONClusterStatus(GetJSONClusterStatusRequestProto)
- returns(GetJSONClusterStatusResponseProto);
-
- /**
- * List all running nodes in a role
- */
- rpc listNodeUUIDsByRole(ListNodeUUIDsByRoleRequestProto)
- returns(ListNodeUUIDsByRoleResponseProto);
-
- /**
- * Get the details on a node
- */
- rpc getNode(GetNodeRequestProto)
- returns(GetNodeResponseProto);
-
- /**
- * Get the
- * details on a list of nodes.
- * Unknown nodes are not returned
- * <i>Important: the order of the results are undefined</i>
- */
- rpc getClusterNodes(GetClusterNodesRequestProto)
- returns(GetClusterNodesResponseProto);
-
- /**
- * echo some text
- */
- rpc echo(EchoRequestProto)
- returns(EchoResponseProto);
-
- /**
- * kill a container
- */
- rpc killContainer(KillContainerRequestProto)
- returns(KillContainerResponseProto);
-
- /**
- * kill the AM
- */
- rpc amSuicide(AMSuicideRequestProto)
- returns(AMSuicideResponseProto);
-
- /* ************************************************************************
-
- REST model and operations.
- Below here the operations and payloads designed to mimic
- the REST API. That API is now the source of those
- specificatations; this is simply a derivative.
-
- **************************************************************************/
-
- rpc getLivenessInformation(GetApplicationLivenessRequestProto)
- returns(ApplicationLivenessInformationProto);
-
- rpc getLiveContainers(GetLiveContainersRequestProto)
- returns(GetLiveContainersResponseProto);
-
- rpc getLiveContainer(GetLiveContainerRequestProto)
- returns(ContainerInformationProto);
-
- rpc getLiveComponents(GetLiveComponentsRequestProto)
- returns(GetLiveComponentsResponseProto);
-
- rpc getLiveComponent(GetLiveComponentRequestProto)
- returns(ComponentInformationProto);
-
- rpc getLiveNodes(GetLiveNodesRequestProto)
- returns(GetLiveNodesResponseProto);
-
- rpc getLiveNode(GetLiveNodeRequestProto)
- returns(NodeInformationProto);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
deleted file mode 100644
index 9e67c15..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
+++ /dev/null
@@ -1,15 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-org.apache.slider.server.appmaster.rpc.SliderRPCSecurityInfo
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/log4j.properties
deleted file mode 100644
index 65a7ad0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/log4j.properties
+++ /dev/null
@@ -1,52 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This is a log4j config for slider
-
-log4j.rootLogger=INFO,stdout
-log4j.threshhold=ALL
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{3} (%F:%M(%L)) - %m%n
-
-log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
-log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
-log4j.appender.subprocess.layout.ConversionPattern=%c{1}: %m%n
-
-
-#at debug this provides details on what is going on
-log4j.logger.org.apache.slider=DEBUG
-#log4j.logger.org.apache.slider.exec.RunLongLivedApp=ERROR
-
-log4j.logger.org.apache.hadoop.security=DEBUG
-log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
-log4j.logger.org.apache.hadoop.yarn.service=DEBUG
-log4j.logger.org.apache.hadoop.yarn.client=DEBUG
-#crank back on some noise
-log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
-log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
-log4j.logger.org.apache.hadoop.yarn.client.RMProxy=WARN
-
-# for test runs we don't care about native code
-log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-# HDFS is noise on tets
-log4j.logger.org.apache.hadoop.hdfs.server.datanode=WARN
-log4j.logger.org.apache.hadoop.hdfs.server.namenode=WARN
-log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
-log4j.logger.org.apache.hadoop.hdfs=WARN
-
-log4j.logger.org.apache.zookeeper=WARN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/agent.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/agent.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/agent.txt
deleted file mode 100644
index 79c1972..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/agent.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-This is the conf directory for the python agent
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command.json
deleted file mode 100644
index 197a046..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command.json
+++ /dev/null
@@ -1,168 +0,0 @@
-{
- "roleCommand": "START",
- "clusterName": "c1",
- "hostname": "c6402.ambari.apache.org",
- "hostLevelParams": {
- "java_home": "/usr/jdk64/jdk1.7.0_45"
- },
- "commandType": "EXECUTION_COMMAND",
- "roleParams": {},
- "serviceName": "HBASE",
- "role": "HBASE_MASTER",
- "commandParams": {},
- "taskId": 24,
- "public_hostname": "c6402.ambari.apache.org",
- "configurations": {
- "hbase-log4j": {
- "log4j.threshold": "ALL",
- "log4j.rootLogger": "${hbase.root.logger}",
- "log4j.logger.org.apache.zookeeper": "INFO",
- "log4j.logger.org.apache.hadoop.hbase": "DEBUG",
- "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher": "INFO",
- "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil": "INFO",
- "log4j.category.SecurityLogger": "${hbase.security.logger}",
- "log4j.appender.console": "org.apache.log4j.ConsoleAppender",
- "log4j.appender.console.target": "System.err",
- "log4j.appender.console.layout": "org.apache.log4j.PatternLayout",
- "log4j.appender.console.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
- "log4j.appender.RFAS": "org.apache.log4j.RollingFileAppender",
- "log4j.appender.RFAS.layout": "org.apache.log4j.PatternLayout",
- "log4j.appender.RFAS.layout.ConversionPattern": "%d{ISO8601} %p %c: %m%n",
- "log4j.appender.RFAS.MaxFileSize": "${hbase.security.log.maxfilesize}",
- "log4j.appender.RFAS.MaxBackupIndex": "${hbase.security.log.maxbackupindex}",
- "log4j.appender.RFAS.File": "${hbase.log.dir}/${hbase.security.log.file}",
- "log4j.appender.RFA": "org.apache.log4j.RollingFileAppender",
- "log4j.appender.RFA.layout": "org.apache.log4j.PatternLayout",
- "log4j.appender.RFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
- "log4j.appender.RFA.MaxFileSize": "${hbase.log.maxfilesize}",
- "log4j.appender.RFA.MaxBackupIndex": "${hbase.log.maxbackupindex}",
- "log4j.appender.RFA.File": "${hbase.log.dir}/${hbase.log.file}",
- "log4j.appender.NullAppender": "org.apache.log4j.varia.NullAppender",
- "log4j.appender.DRFA": "org.apache.log4j.DailyRollingFileAppender",
- "log4j.appender.DRFA.layout": "org.apache.log4j.PatternLayout",
- "log4j.appender.DRFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
- "log4j.appender.DRFA.File": "${hbase.log.dir}/${hbase.log.file}",
- "log4j.appender.DRFA.DatePattern": ".yyyy-MM-dd",
- "log4j.additivity.SecurityLogger": "false",
- "hbase.security.logger": "INFO,console",
- "hbase.security.log.maxfilesize": "256MB",
- "hbase.security.log.maxbackupindex": "20",
- "hbase.security.log.file": "SecurityAuth.audit",
- "hbase.root.logger": "INFO,console",
- "hbase.log.maxfilesize": "256MB",
- "hbase.log.maxbackupindex": "20",
- "hbase.log.file": "hbase.log",
- "hbase.log.dir": "."
- },
- "global": {
- "hbase_root": "/share/hbase/hbase-0.96.1-hadoop2",
- "hbase_pid_dir": "/var/run/hbase",
- "proxyuser_group": "users",
- "syncLimit": "5",
- "hbase_regionserver_heapsize": "1024m",
- "rca_enabled": "false",
- "tickTime": "2000",
- "hbase_master_heapsize": "1024m",
- "initLimit": "10",
- "user_group": "hadoop",
- "hbase_user": "hbase",
- "hbase_log_dir": "/var/log/hbase"
- },
- "hdfs-site": {
- "dfs.namenode.checkpoint.period": "21600",
- "dfs.namenode.avoid.write.stale.datanode": "true",
- "dfs.namenode.checkpoint.txns": "1000000",
- "dfs.block.access.token.enable": "true",
- "dfs.support.append": "true",
- "dfs.datanode.address": "0.0.0.0:${ambari.dfs.datanode.port}",
- "dfs.cluster.administrators": " hdfs",
- "dfs.replication": "3",
- "ambari.dfs.datanode.http.port": "50075",
- "dfs.datanode.balance.bandwidthPerSec": "6250000",
- "dfs.namenode.safemode.threshold-pct": "1.0f",
- "dfs.namenode.checkpoint.edits.dir": "${dfs.namenode.checkpoint.dir}",
- "dfs.permissions.enabled": "true",
- "dfs.client.read.shortcircuit": "true",
- "dfs.namenode.https-address": "c6402.ambari.apache.org:50470",
- "dfs.journalnode.edits.dir": "/grid/0/hdfs/journal",
- "dfs.blocksize": "134217728",
- "dfs.datanode.max.transfer.threads": "1024",
- "dfs.datanode.du.reserved": "1073741824",
- "dfs.webhdfs.enabled": "true",
- "dfs.namenode.handler.count": "100",
- "dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary",
- "fs.permissions.umask-mode": "022",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
- "dfs.datanode.ipc.address": "0.0.0.0:8010",
- "dfs.datanode.data.dir": "/hadoop/hdfs/data",
- "dfs.namenode.http-address": "c6402.ambari.apache.org:50070",
- "dfs.blockreport.initialDelay": "120",
- "dfs.datanode.failed.volumes.tolerated": "0",
- "dfs.namenode.accesstime.precision": "0",
- "ambari.dfs.datanode.port": "50010",
- "dfs.namenode.avoid.read.stale.datanode": "true",
- "dfs.namenode.secondary.http-address": "c6402.ambari.apache.org:50090",
- "dfs.namenode.stale.datanode.interval": "30000",
- "dfs.heartbeat.interval": "3",
- "dfs.client.read.shortcircuit.streams.cache.size": "4096",
- "dfs.permissions.superusergroup": "hdfs",
- "dfs.https.port": "50470",
- "dfs.journalnode.http-address": "0.0.0.0:8480",
- "dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket",
- "dfs.namenode.write.stale.datanode.ratio": "1.0f",
- "dfs.hosts.exclude": "/etc/hadoop/conf/dfs.exclude",
- "dfs.datanode.data.dir.perm": "750",
- "dfs.namenode.name.dir.restore": "true",
- "dfs.replication.max": "50",
- "dfs.namenode.name.dir": "/hadoop/hdfs/namenode"
- },
- "hbase-site": {
- "hbase.hstore.flush.retries.number": "120",
- "hbase.client.keyvalue.maxsize": "10485760",
- "hbase.hstore.compactionThreshold": "3",
- "hbase.rootdir": "hdfs://c6402.ambari.apache.org:8020/apps/hbase/data",
- "hbase.stagingdir": "hdfs://c6402.ambari.apache.org:8020/apps/hbase/staging",
- "hbase.regionserver.handler.count": "60",
- "hbase.regionserver.global.memstore.lowerLimit": "0.38",
- "hbase.hregion.memstore.block.multiplier": "2",
- "hbase.hregion.memstore.flush.size": "134217728",
- "hbase.superuser": "hbase",
- "hbase.zookeeper.property.clientPort": "2181",
- "hbase.regionserver.global.memstore.upperLimit": "0.4",
- "zookeeper.session.timeout": "30000",
- "hbase.tmp.dir": "/hadoop/hbase",
- "hbase.hregion.max.filesize": "10737418240",
- "hfile.block.cache.size": "0.40",
- "hbase.security.authentication": "simple",
- "hbase.defaults.for.version.skip": "true",
- "hbase.zookeeper.quorum": "c6402.ambari.apache.org",
- "zookeeper.znode.parent": "/hbase-unsecure",
- "hbase.hstore.blockingStoreFiles": "10",
- "hbase.hregion.majorcompaction": "86400000",
- "hbase.security.authorization": "false",
- "hbase.cluster.distributed": "true",
- "hbase.hregion.memstore.mslab.enabled": "true",
- "hbase.client.scanner.caching": "100",
- "hbase.zookeeper.useMulti": "true",
- "hbase.regionserver.info.port": "0",
- "hbase.master.info.port": "60010"
- },
- "core-site": {
- "io.serializations": "org.apache.hadoop.io.serializer.WritableSerialization",
- "gluster.daemon.user": "null",
- "fs.trash.interval": "360",
- "hadoop.security.authentication": "simple",
- "io.compression.codecs": "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec",
- "mapreduce.jobtracker.webinterface.trusted": "false",
- "fs.AbstractFileSystem.glusterfs.impl": "null",
- "fs.defaultFS": "hdfs://c6402.ambari.apache.org:8020",
- "ipc.client.connect.max.retries": "50",
- "ipc.client.idlethreshold": "8000",
- "io.file.buffer.size": "131072",
- "hadoop.security.authorization": "false",
- "hadoop.security.auth_to_local": "\n RULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/\n RULE:[2:$1@$0](jhs@.*)s/.*/mapred/\n RULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/\n RULE:[2:$1@$0](hm@.*)s/.*/hbase/\n RULE:[2:$1@$0](rs@.*)s/.*/hbase/\n DEFAULT",
- "ipc.client.connection.maxidletime": "30000"
- }
- },
- "commandId": "2-2"
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
deleted file mode 100644
index 057f6c5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.common.tools;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/** Test slider util methods. */
-public class TestSliderUtils {
- protected static final Logger log =
- LoggerFactory.getLogger(TestSliderUtils.class);
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- //@Test
- public void testTruncate() {
- Assert.assertEquals(SliderUtils.truncate(null, 5), null);
- Assert.assertEquals(SliderUtils.truncate("323", -1), "323");
- Assert.assertEquals(SliderUtils.truncate("3232", 5), "3232");
- Assert.assertEquals(SliderUtils.truncate("1234567890", 0), "1234567890");
- Assert.assertEquals(SliderUtils.truncate("123456789012345", 15), "123456789012345");
- Assert.assertEquals(SliderUtils.truncate("123456789012345", 14), "12345678901...");
- Assert.assertEquals(SliderUtils.truncate("1234567890", 1), "1");
- Assert.assertEquals(SliderUtils.truncate("1234567890", 10), "1234567890");
- Assert.assertEquals(SliderUtils.truncate("", 10), "");
- }
-
- //@Test
- public void testApplicationReportComparison() {
- List<ApplicationReport> instances = getApplicationReports();
-
- SliderUtils.sortApplicationsByMostRecent(instances);
-
- Assert.assertEquals(1000, instances.get(0).getStartTime());
- Assert.assertEquals(1000, instances.get(1).getStartTime());
- Assert.assertEquals(1000, instances.get(2).getStartTime());
- Assert.assertEquals(1000, instances.get(3).getStartTime());
-
- instances = getApplicationReports();
-
- SliderUtils.sortApplicationReport(instances);
- Assert.assertEquals(1000, instances.get(0).getStartTime());
- Assert.assertEquals(1000, instances.get(1).getStartTime());
- Assert.assertEquals(1000, instances.get(2).getStartTime());
- Assert.assertEquals(1000, instances.get(3).getStartTime());
-
- Assert.assertTrue(instances.get(0).getYarnApplicationState() == YarnApplicationState.ACCEPTED ||
- instances.get(0).getYarnApplicationState() == YarnApplicationState.RUNNING);
- Assert.assertTrue(instances.get(1).getYarnApplicationState() == YarnApplicationState.ACCEPTED ||
- instances.get(1).getYarnApplicationState() == YarnApplicationState.RUNNING);
- Assert.assertTrue(instances.get(2).getYarnApplicationState() == YarnApplicationState.ACCEPTED ||
- instances.get(2).getYarnApplicationState() == YarnApplicationState.RUNNING);
- Assert.assertTrue(instances.get(3).getYarnApplicationState() == YarnApplicationState.KILLED);
- }
-
- private List<ApplicationReport> getApplicationReports() {
- List<ApplicationReport> instances = new ArrayList<ApplicationReport>();
- instances.add(getApplicationReport(1000, 0, "app1", YarnApplicationState.ACCEPTED));
- instances.add(getApplicationReport(900, 998, "app1", YarnApplicationState.KILLED));
- instances.add(getApplicationReport(900, 998, "app2", YarnApplicationState.FAILED));
- instances.add(getApplicationReport(1000, 0, "app2", YarnApplicationState.RUNNING));
- instances.add(getApplicationReport(800, 837, "app3", YarnApplicationState.FINISHED));
- instances.add(getApplicationReport(1000, 0, "app3", YarnApplicationState.RUNNING));
- instances.add(getApplicationReport(900, 998, "app3", YarnApplicationState.KILLED));
- instances.add(getApplicationReport(800, 837, "app4", YarnApplicationState.FINISHED));
- instances.add(getApplicationReport(1000, 1050, "app4", YarnApplicationState.KILLED));
- instances.add(getApplicationReport(900, 998, "app4", YarnApplicationState.FINISHED));
-
- Assert.assertEquals("app1", instances.get(0).getApplicationType());
- Assert.assertEquals("app1", instances.get(1).getApplicationType());
- Assert.assertEquals("app2", instances.get(2).getApplicationType());
- Assert.assertEquals("app2", instances.get(3).getApplicationType());
- return instances;
- }
-
- private ApplicationReportPBImpl getApplicationReport(long startTime,
- long finishTime,
- String name,
- YarnApplicationState state) {
- ApplicationReportPBImpl ar = new ApplicationReportPBImpl();
- ar.setFinishTime(finishTime);
- ar.setStartTime(startTime);
- ar.setApplicationType(name);
- ar.setYarnApplicationState(state);
- return ar;
- }
-
-
- //@Test
- public void testGetHdpVersion() {
- String hdpVersion = "2.3.2.0-2766";
- Assert.assertEquals("Version should be empty", null,
- SliderUtils.getHdpVersion());
- }
-
- //@Test
- public void testIsHdp() {
- Assert.assertFalse("Should be false", SliderUtils.isHdp());
- }
-
- //@Test
- public void testWrite() throws IOException {
- File testWriteFile = folder.newFile("testWrite");
- SliderUtils.write(testWriteFile, "test".getBytes("UTF-8"));
- Assert.assertTrue(FileUtils.readFileToString(testWriteFile, "UTF-8").equals("test"));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java
deleted file mode 100644
index 073fd51..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.ChecksumFileSystem;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.service.ServiceStateException;
-import org.apache.hadoop.util.Shell;
-import org.apache.slider.utils.YarnMiniClusterTestBase;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.List;
-import java.util.regex.Pattern;
-
-/**
- * Test windows support.
- */
-public class TestWindowsSupport extends YarnMiniClusterTestBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestWindowsSupport.class);
-
- private static final Pattern HAS_DRIVE_LETTER_SPECIFIER =
- Pattern.compile("^/?[a-zA-Z]:");
- public static final String WINDOWS_FILE =
- "C:\\Users\\Administrator\\AppData\\Local\\Temp" +
- "\\junit3180177850133852404\\testpkg\\appdef_1.zip";
-
-
- private static boolean hasWindowsDrive(String path) {
- return HAS_DRIVE_LETTER_SPECIFIER.matcher(path).find();
- }
-
- private static int startPositionWithoutWindowsDrive(String path) {
- if (hasWindowsDrive(path)) {
- return path.charAt(0) == '/' ? 3 : 2;
- } else {
- return 0;
- }
- }
-
- //@Test
- public void testHasWindowsDrive() throws Throwable {
- assertTrue(hasWindowsDrive(WINDOWS_FILE));
- }
-
- //@Test
- public void testStartPosition() throws Throwable {
- assertEquals(2, startPositionWithoutWindowsDrive(WINDOWS_FILE));
- }
-
- //@Test
- public void testPathHandling() throws Throwable {
- assumeWindows();
-
- Path path = new Path(WINDOWS_FILE);
- URI uri = path.toUri();
- // assert "file" == uri.scheme
- assertNull(uri.getAuthority());
-
- Configuration conf = new Configuration();
-
- FileSystem localfs = FileSystem.get(uri, conf);
- assertTrue(localfs instanceof ChecksumFileSystem);
- try {
- FileStatus stat = localfs.getFileStatus(path);
- fail("expected an exception, got " + stat);
- } catch (FileNotFoundException fnfe) {
- // expected
- }
-
- try {
- FSDataInputStream appStream = localfs.open(path);
- } catch (FileNotFoundException fnfe) {
- // expected
- }
- }
-
- //@Test
- public void testExecNonexistentBinary() throws Throwable {
- assumeWindows();
- List<String> commands = Arrays.asList("undefined-application", "--version");
- try {
- exec(0, commands);
- fail("expected an exception");
- } catch (ServiceStateException e) {
- if (!(e.getCause() instanceof FileNotFoundException)) {
- throw e;
- }
- }
- }
- //@Test
- public void testExecNonexistentBinary2() throws Throwable {
- assumeWindows();
- assertFalse(doesAppExist(Arrays.asList("undefined-application",
- "--version")));
- }
-
- //@Test
- public void testEmitKillCommand() throws Throwable {
-
- int result = killJavaProcesses("regionserver", 9);
- // we know the exit code if there is no supported kill operation
- assertTrue(getKillSupported() || result == -1);
- }
-
- //@Test
- public void testHadoopHomeDefined() throws Throwable {
- assumeWindows();
- String hadoopHome = Shell.getHadoopHome();
- LOG.info("HADOOP_HOME={}", hadoopHome);
- }
-
- //@Test
- public void testHasWinutils() throws Throwable {
- assumeWindows();
- SliderUtils.maybeVerifyWinUtilsValid();
- }
-
- //@Test
- public void testExecWinutils() throws Throwable {
- assumeWindows();
- String winUtilsPath = Shell.getWinUtilsPath();
- assertTrue(SliderUtils.isSet(winUtilsPath));
- File winUtils = new File(winUtilsPath);
- LOG.debug("Winutils is at {}", winUtils);
-
- exec(0, Arrays.asList(winUtilsPath, "systeminfo"));
- }
-
- //@Test
- public void testPath() throws Throwable {
- String path = extractPath();
- LOG.info("Path value = {}", path);
- }
-
- //@Test
- public void testFindJavac() throws Throwable {
- String name = Shell.WINDOWS ? "javac.exe" : "javac";
- assertNotNull(locateExecutable(name));
- }
-
- //@Test
- public void testHadoopDLL() throws Throwable {
- assumeWindows();
- // split the path
- File exepath = locateExecutable("HADOOP.DLL");
- assertNotNull(exepath);
- LOG.info("Hadoop DLL at: {}", exepath);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java
deleted file mode 100644
index ed9337d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.registry.server.services.MicroZookeeperServiceKeys;
-import org.apache.slider.client.SliderClient;
-import org.apache.slider.core.zk.ZKIntegration;
-import org.apache.slider.utils.KeysForTests;
-import org.apache.slider.utils.YarnZKMiniClusterTestBase;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.ZooDefs;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Test ZK integration.
- */
-public class TestZKIntegration extends YarnZKMiniClusterTestBase implements
- KeysForTests {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestZKIntegration.class);
-
- public static final String USER = KeysForTests.USERNAME;
- public static final int CONNECT_TIMEOUT = 5000;
- private ZKIntegration zki;
-
- @Before
- public void createCluster() {
- Configuration conf = getConfiguration();
- String name = methodName.getMethodName();
- File zkdir = new File("target/zk/${name}");
- FileUtil.fullyDelete(zkdir);
- conf.set(MicroZookeeperServiceKeys.KEY_ZKSERVICE_DIR, zkdir
- .getAbsolutePath());
- createMicroZKCluster("-"+ name, conf);
- }
-
- @After
- public void closeZKI() throws IOException {
- if (zki != null) {
- zki.close();
- zki = null;
- }
- }
-
- public ZKIntegration initZKI() throws IOException, InterruptedException {
- zki = createZKIntegrationInstance(
- getZKBinding(), methodName.getMethodName(), true, false,
- CONNECT_TIMEOUT);
- return zki;
- }
-
- //@Test
- public void testListUserClustersWithoutAnyClusters() throws Throwable {
- assertHasZKCluster();
- initZKI();
- String userPath = ZKIntegration.mkSliderUserPath(USER);
- List<String> clusters = this.zki.getClusters();
- assertTrue(SliderUtils.isEmpty(clusters));
- }
-
- //@Test
- public void testListUserClustersWithOneCluster() throws Throwable {
- assertHasZKCluster();
-
- initZKI();
- String userPath = ZKIntegration.mkSliderUserPath(USER);
- String fullPath = zki.createPath(userPath, "/cluster-",
- ZooDefs.Ids.OPEN_ACL_UNSAFE,
- CreateMode.EPHEMERAL_SEQUENTIAL);
- LOG.info("Ephemeral path {}", fullPath);
- List<String> clusters = zki.getClusters();
- assertEquals(1, clusters.size());
- assertTrue(fullPath.endsWith(clusters.get(0)));
- }
-
- //@Test
- public void testListUserClustersWithTwoCluster() throws Throwable {
- initZKI();
- String userPath = ZKIntegration.mkSliderUserPath(USER);
- String c1 = createEphemeralChild(zki, userPath);
- LOG.info("Ephemeral path $c1");
- String c2 = createEphemeralChild(zki, userPath);
- LOG.info("Ephemeral path $c2");
- List<String> clusters = zki.getClusters();
- assertEquals(2, clusters.size());
- assertTrue((c1.endsWith(clusters.get(0)) && c2.endsWith(clusters.get(1))) ||
- (c1.endsWith(clusters.get(1)) && c2.endsWith(clusters.get(0))));
- }
-
- //@Test
- public void testCreateAndDeleteDefaultZKPath() throws Throwable {
- MockSliderClient client = new MockSliderClient();
-
- String path = client.createZookeeperNodeInner("cl1", true);
- zki = client.getLastZKIntegration();
-
- String zkPath = ZKIntegration.mkClusterPath(USER, "cl1");
- assertEquals("zkPath must be as expected", zkPath,
- "/services/slider/users/" + USER + "/cl1");
- assertEquals(path, zkPath);
- assertNull("ZKIntegration should be null.", zki);
- zki = createZKIntegrationInstance(getZKBinding(), "cl1", true, false,
- CONNECT_TIMEOUT);
- assertFalse(zki.exists(zkPath));
-
- path = client.createZookeeperNodeInner("cl1", false);
- zki = client.getLastZKIntegration();
- assertNotNull(zki);
- assertEquals("zkPath must be as expected", zkPath,
- "/services/slider/users/" + USER + "/cl1");
- assertEquals(path, zkPath);
- assertTrue(zki.exists(zkPath));
- zki.createPath(zkPath, "/cn", ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode
- .PERSISTENT);
- assertTrue(zki.exists(zkPath + "/cn"));
- client.deleteZookeeperNode("cl1");
- assertFalse(zki.exists(zkPath));
- }
-
- public static String createEphemeralChild(ZKIntegration zki, String userPath)
- throws KeeperException, InterruptedException {
- return zki.createPath(userPath, "/cluster-",
- ZooDefs.Ids.OPEN_ACL_UNSAFE,
- CreateMode.EPHEMERAL_SEQUENTIAL);
- }
-
- /**
- * Test slider client that overriddes ZK client.
- */
- public class MockSliderClient extends SliderClient {
- private ZKIntegration zki;
-
- @Override
- public String getUsername() {
- return USER;
- }
-
- @Override
- protected ZKIntegration getZkClient(String clusterName, String user) {
- try {
- zki = createZKIntegrationInstance(getZKBinding(), clusterName, true,
- false, CONNECT_TIMEOUT);
- } catch (Exception e) {
- fail("creating ZKIntergration threw an exception");
- }
- return zki;
- }
-
- @Override
- public Configuration getConfig() {
- return new Configuration();
- }
-
- public ZKIntegration getLastZKIntegration() {
- return zki;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
deleted file mode 100644
index 54c3576..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.core.registry.docstore;
-
-import com.google.common.base.Charsets;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.powermock.api.easymock.PowerMock;
-import org.yaml.snakeyaml.Yaml;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.expect;
-import static org.mockito.Matchers.anyString;
-import static org.powermock.api.easymock.PowerMock.createNiceMock;
-
-public class TestPublishedConfigurationOutputter {
- private static HashMap<String, String> config = new HashMap<>();
-
- @Rule
- public TemporaryFolder tmpDir = new TemporaryFolder();
-
- @Before
- public void setup() {
- config.put("key1", "val1");
- }
-
- //@Test
- public void testJson() throws IOException {
- PublishedConfigurationOutputter configurationOutputter =
- PublishedConfigurationOutputter.createOutputter(ConfigFormat.JSON,
- new PublishedConfiguration("description",
- config.entrySet()));
-
- String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
- "");
- assert "{\"key1\":\"val1\"}".equals(output);
-
- File file = tmpDir.newFile();
- configurationOutputter.save(file);
-
- ObjectMapper mapper = new ObjectMapper();
- @SuppressWarnings("unchecked")
- Map<String, String> read = mapper.readValue(file, Map.class);
- assert 1 == read.size();
- assert "val1".equals(read.get("key1"));
- }
-
- //@Test
- public void testXml() throws IOException {
- PublishedConfigurationOutputter configurationOutputter =
- PublishedConfigurationOutputter.createOutputter(ConfigFormat.XML,
- new PublishedConfiguration("description",
- config.entrySet()));
-
- String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
- "");
- assert output.contains("<name>key1</name><value>val1</value>");
-
- File file = tmpDir.newFile();
- configurationOutputter.save(file);
-
- assert FileUtils.readFileToString(file, Charsets.UTF_8)
- .replaceAll("( |\\r|\\n)", "")
- .contains("<name>key1</name><value>val1</value>");
- }
-
- //@Test
- public void testHadoopXml() throws IOException {
- PublishedConfigurationOutputter configurationOutputter =
- PublishedConfigurationOutputter.createOutputter(ConfigFormat.HADOOP_XML,
- new PublishedConfiguration("description",
- config.entrySet()));
-
- String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
- "");
- assert output.contains("<name>key1</name><value>val1</value>");
-
- File file = tmpDir.newFile();
- configurationOutputter.save(file);
-
- assert FileUtils.readFileToString(file, Charsets.UTF_8)
- .replaceAll("( |\\r|\\n)", "")
- .contains("<name>key1</name><value>val1</value>");
- }
-
- //@Test
- public void testProperties() throws IOException {
- PublishedConfigurationOutputter configurationOutputter =
- PublishedConfigurationOutputter.createOutputter(ConfigFormat.PROPERTIES,
- new PublishedConfiguration("description",
- config.entrySet()));
-
- String output = configurationOutputter.asString();
- assert output.contains("key1=val1");
-
- File file = tmpDir.newFile();
- configurationOutputter.save(file);
-
- Properties properties = new Properties();
- FileInputStream fis = null;
- try {
- fis = new FileInputStream(file);
- properties.load(fis);
- } finally {
- if (fis != null) {
- fis.close();
- }
- }
- assert 1 == properties.size();
- assert "val1".equals(properties.getProperty("key1"));
- }
-
- //@Test
- public void testYaml() throws IOException {
- PublishedConfigurationOutputter configurationOutputter =
- PublishedConfigurationOutputter.createOutputter(ConfigFormat.YAML,
- new PublishedConfiguration("description",
- config.entrySet()));
-
- String output = configurationOutputter.asString().replaceAll("(\\r|\\n)",
- "");
- assert "key1: val1".equals(output);
-
- File file = tmpDir.newFile();
- configurationOutputter.save(file);
-
- Yaml yaml = new Yaml();
- FileInputStream fis = null;
- Map<String, String> read;
- try {
- fis = new FileInputStream(file);
- read = (Map<String, String>) yaml.load(fis);
- } finally {
- if (fis != null) {
- fis.close();
- }
- }
- assert 1 == read.size();
- assert "val1".equals(read.get("key1"));
- }
-
- //@Test
- public void testEnv() throws IOException {
- HashMap<String, String> envConfig = new HashMap<>(config);
- envConfig.put("content", "content {{key1}} ");
-
- PublishedConfigurationOutputter configurationOutputter =
- PublishedConfigurationOutputter.createOutputter(ConfigFormat.ENV,
- new PublishedConfiguration("description",
- envConfig.entrySet()));
-
- String output = configurationOutputter.asString();
- assert "content val1 ".equals(output);
-
- File file = tmpDir.newFile();
- configurationOutputter.save(file);
-
- assert "content val1 ".equals(FileUtils.readFileToString(file,
- Charsets.UTF_8));
- }
-
- //@Test
- public void testTemplate1() throws IOException {
- HashMap<String, String> templateConfig = new HashMap<>(config);
- templateConfig.put(ConfigUtils.TEMPLATE_FILE, "templateFileName");
-
- SliderFileSystem fileSystem = createNiceMock(SliderFileSystem.class);
- expect(fileSystem.buildResourcePath(anyString())).andReturn(new Path("path")).anyTimes();
- expect(fileSystem.isFile(anyObject(Path.class))).andReturn(true).anyTimes();
- expect(fileSystem.cat(anyObject(Path.class))).andReturn("content {{key1}}\n more ${key1} content").anyTimes();
-
- PowerMock.replay(fileSystem);
-
- ConfigUtils.prepConfigForTemplateOutputter(ConfigFormat.TEMPLATE,
- templateConfig, fileSystem, "clusterName", null);
- PublishedConfigurationOutputter configurationOutputter =
- PublishedConfigurationOutputter.createOutputter(ConfigFormat.TEMPLATE,
- new PublishedConfiguration("description",
- templateConfig.entrySet()));
-
- String output = configurationOutputter.asString();
- assert "content val1\n more val1 content".equals(output);
-
- File file = tmpDir.newFile();
- configurationOutputter.save(file);
-
- PowerMock.verify(fileSystem);
-
- assert "content val1\n more val1 content".equals(
- FileUtils.readFileToString(file, Charsets.UTF_8));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java
deleted file mode 100644
index 1e5d769..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.other;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.util.DiskChecker;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
-import org.apache.slider.utils.YarnMiniClusterTestBase;
-import org.junit.After;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * This test class exists to look at permissions of the filesystem, especially
- * that created by Mini YARN clusters. On some windows jenkins machines,
- * YARN actions were failing as the directories had the wrong permissions
- * (i.e. too lax)
- */
-public class TestFilesystemPermissions extends YarnMiniClusterTestBase {
-
- private static final Logger LOG = LoggerFactory.getLogger(
- TestFilesystemPermissions.class);
-
- private List<File> filesToDelete = new ArrayList<>();
-
- @After
- public void deleteFiles() {
- for (File f : filesToDelete) {
- FileUtil.fullyDelete(f, true);
- }
- }
-
- //@Test
- public void testJavaFSOperations() throws Throwable {
- assertNativeLibrariesPresent();
- File subdir = testDir();
- subdir.mkdir();
- assertTrue(subdir.isDirectory());
- assertTrue(FileUtil.canRead(subdir));
- assertTrue(FileUtil.canWrite(subdir));
- assertTrue(FileUtil.canExecute(subdir));
- }
-
- //@Test
- public void testDiskCheckerOperations() throws Throwable {
- assertNativeLibrariesPresent();
- File subdir = testDir();
- subdir.mkdir();
- DiskChecker checker = new DiskChecker();
- checker.checkDir(subdir);
- }
-
- //@Test
- public void testDiskCheckerMkdir() throws Throwable {
- assertNativeLibrariesPresent();
- File subdir = testDir();
- subdir.mkdirs();
- DiskChecker checker = new DiskChecker();
- checker.checkDir(subdir);
- }
-
- /**
- * Get a test dir for this method; one that will be deleted on teardown.
- * @return a filename unique to this test method
- */
- File testDir() {
- File parent = new File("target/testfspermissions");
- parent.mkdir();
- File testdir = new File(parent, methodName.getMethodName());
- filesToDelete.add(testdir);
- return testdir;
- }
-
-
- //@Test
- public void testPermsMap() throws Throwable {
- File dir = testDir();
- String diruri = dir.toURI().toString();
- FileContext lfs = createLocalFS(dir, getConfiguration());
- getLocalDirsPathPermissionsMap(lfs, diruri);
- }
-
- //@Test
- public void testInitLocaldir() throws Throwable {
- File dir = testDir();
- String diruri = dir.toURI().toString();
- FileContext lfs = createLocalFS(dir, getConfiguration());
- initializeLocalDir(lfs, diruri);
- List<String> localDirs = getInitializedLocalDirs(lfs, Arrays.asList(
- diruri));
- assertEquals(1, localDirs.size());
- }
-
-
- //@Test
- public void testValidateMiniclusterPerms() throws Throwable {
- int numLocal = 1;
- String cluster = createMiniCluster("", getConfiguration(), 1, numLocal, 1,
- false);
- File workDir = getMiniCluster().getTestWorkDir();
- List<File> localdirs = new ArrayList<>();
- for (File file : workDir.listFiles()) {
- if (file.isDirectory() && file.getAbsolutePath().contains("-local")) {
- // local dir
- localdirs.add(file);
- }
- }
- assertEquals(numLocal, localdirs.size());
- FileContext lfs = createLocalFS(workDir, getConfiguration());
- for (File file : localdirs) {
- checkLocalDir(lfs, file.toURI().toString());
- }
- }
-
- FileContext createLocalFS(File dir, Configuration conf)
- throws UnsupportedFileSystemException {
- return FileContext.getFileContext(dir.toURI(), conf);
- }
-
- /**
- * Extracted from ResourceLocalizationService.
- * @param lfs
- * @param localDir
- * @return perms map
- * @see ResourceLocalizationService
- */
- private Map<Path, FsPermission> getLocalDirsPathPermissionsMap(
- FileContext lfs,
- String localDir) {
- Map<Path, FsPermission> localDirPathFsPermissionsMap = new HashMap<>();
-
- FsPermission defaultPermission =
- FsPermission.getDirDefault().applyUMask(lfs.getUMask());
- FsPermission nmPrivatePermission =
- ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask());
-
- Path userDir = new Path(localDir, ContainerLocalizer.USERCACHE);
- Path fileDir = new Path(localDir, ContainerLocalizer.FILECACHE);
- Path sysDir = new Path(
- localDir,
- ResourceLocalizationService.NM_PRIVATE_DIR);
-
- localDirPathFsPermissionsMap.put(userDir, defaultPermission);
- localDirPathFsPermissionsMap.put(fileDir, defaultPermission);
- localDirPathFsPermissionsMap.put(sysDir, nmPrivatePermission);
- return localDirPathFsPermissionsMap;
- }
-
- private boolean checkLocalDir(FileContext lfs, String localDir)
- throws IOException {
-
- Map<Path, FsPermission> pathPermissionMap =
- getLocalDirsPathPermissionsMap(lfs, localDir);
-
- for (Map.Entry<Path, FsPermission> entry : pathPermissionMap.entrySet()) {
- FileStatus status;
- status = lfs.getFileStatus(entry.getKey());
-
- if (!status.getPermission().equals(entry.getValue())) {
- String msg =
- "Permissions incorrectly set for dir " + entry.getKey() +
- ", should be " + entry.getValue() + ", actual value = " +
- status.getPermission();
- throw new YarnRuntimeException(msg);
- }
- }
- return true;
- }
-
-
- private void initializeLocalDir(FileContext lfs, String localDir)
- throws IOException {
-
- Map<Path, FsPermission> pathPermissionMap =
- getLocalDirsPathPermissionsMap(lfs, localDir);
- for (Map.Entry<Path, FsPermission> entry : pathPermissionMap.entrySet()) {
- FileStatus status;
- try {
- status = lfs.getFileStatus(entry.getKey());
- } catch (FileNotFoundException fs) {
- status = null;
- }
-
- if (status == null) {
- lfs.mkdir(entry.getKey(), entry.getValue(), true);
- status = lfs.getFileStatus(entry.getKey());
- }
- FsPermission perms = status.getPermission();
- if (!perms.equals(entry.getValue())) {
- lfs.setPermission(entry.getKey(), entry.getValue());
- }
- }
- }
-
- synchronized private List<String> getInitializedLocalDirs(FileContext lfs,
- List<String> dirs) throws IOException {
- List<String> checkFailedDirs = new ArrayList<String>();
- for (String dir : dirs) {
- try {
- checkLocalDir(lfs, dir);
- } catch (YarnRuntimeException e) {
- checkFailedDirs.add(dir);
- }
- }
- for (String dir : checkFailedDirs) {
- LOG.info("Attempting to initialize " + dir);
- initializeLocalDir(lfs, dir);
- checkLocalDir(lfs, dir);
- }
- return dirs;
- }
-
-
- private void createDir(FileContext localFs, Path dir, FsPermission perm)
- throws IOException {
- if (dir == null) {
- return;
- }
- try {
- localFs.getFileStatus(dir);
- } catch (FileNotFoundException e) {
- createDir(localFs, dir.getParent(), perm);
- localFs.mkdir(dir, perm, false);
- if (!perm.equals(perm.applyUMask(localFs.getUMask()))) {
- localFs.setPermission(dir, perm);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java
deleted file mode 100644
index a45b27d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.other;
-
-import org.apache.slider.utils.SliderTestUtils;
-import org.apache.slider.utils.TestUtility;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-/**
- * This test exists to diagnose local FS permissions.
- */
-public class TestLocalDirStatus extends SliderTestUtils {
-
-
- public static final int SIZE = 0x200000;
-
- //@Test
- public void testTempDir() throws Throwable {
- File tmpf = null;
- try {
- tmpf = File.createTempFile("testl", ".bin");
- createAndReadFile(tmpf, SIZE);
- tmpf.delete();
- assertFalse(tmpf.exists());
- } finally {
- if (tmpf != null) {
- tmpf.delete();
- }
- }
- }
-
- //@Test
- public void testTargetDir() throws Throwable {
- File target = target();
- File tmpf = null;
- try {
- tmpf = File.createTempFile("testl", ".bin", target);
- createAndReadFile(tmpf, SIZE);
- tmpf.delete();
- assertFalse(tmpf.exists());
- } finally {
- if (tmpf != null) {
- tmpf.delete();
- }
-
- }
- }
-
- public File target() {
- File target = new File("target").getAbsoluteFile();
- assertTrue(target.exists());
- return target;
- }
-
- //@Test
- public void testRenameInTargetDir() throws Throwable {
- File target = target();
- File tmpf = null;
- File dst= null;
- try {
- tmpf = File.createTempFile("testl", ".bin", target);
- dst = File.createTempFile("test-dest", ".bin", target);
- createRenameAndReadFile(tmpf, dst, SIZE);
- assertFalse(tmpf.exists());
- dst.delete();
- } finally {
- if (tmpf != null) {
- tmpf.delete();
- }
- if (dst != null) {
- dst.delete();
- }
- }
- }
-
- //@Test
- public void testRenameInTmpDir() throws Throwable {
- File tmpf = null;
- File dst= null;
- try {
- tmpf = File.createTempFile("testl", ".bin");
- dst = File.createTempFile("test-dest", ".bin");
- createRenameAndReadFile(tmpf, dst, SIZE);
- assertFalse(tmpf.exists());
- dst.delete();
- } finally {
- if (tmpf != null) {
- tmpf.delete();
- }
- if (dst != null) {
- dst.delete();
- }
- }
- }
-
- protected void createAndReadFile(File path, int len) throws IOException {
- byte[] dataset = TestUtility.dataset(len, 32, 128);
- writeFile(path, dataset);
- assertTrue(path.exists());
- assertEquals(len, path.length());
- byte[] persisted = readFile(path);
- TestUtility.compareByteArrays(dataset, persisted, len);
- }
-
- protected void createRenameAndReadFile(File src, File dst, int len)
- throws IOException {
- byte[] dataset = TestUtility.dataset(len, 32, 128);
- writeFile(src, dataset);
- assertTrue(src.exists());
- assertEquals(len, src.length());
- dst.delete();
- assertFalse(dst.exists());
- assertTrue(src.renameTo(dst));
- assertEquals(len, dst.length());
- byte[] persisted = readFile(dst);
- TestUtility.compareByteArrays(dataset, persisted, len);
- }
-
- protected void writeFile(File path, byte[] dataset)
- throws IOException {
- FileOutputStream out = new FileOutputStream(path);
- try {
- out.write(dataset);
- out.flush();
- } finally {
- out.close();
- }
- }
-
- protected byte[] readFile(File path) throws IOException {
- assertTrue(path.getAbsoluteFile().exists());
- assertTrue(path.getAbsoluteFile().isFile());
- int len = (int)path.length();
- byte[] dataset = new byte[len];
- FileInputStream ins = new FileInputStream(path);
- try {
- ins.read(dataset);
- } finally {
- ins.close();
- }
- return dataset;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java
deleted file mode 100644
index 7bb8707..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.api.resource.ConfigFile.TypeEnum;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-
-/**
- * Test the AbstractClientProvider shared methods.
- */
-public class TestAbstractClientProvider {
- private static final String EXCEPTION_PREFIX = "Should have thrown " +
- "exception: ";
- private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
- "exception: ";
-
- private static class ClientProvider extends AbstractClientProvider {
- @Override
- public void validateArtifact(Artifact artifact, FileSystem fileSystem)
- throws IOException {
- }
-
- @Override
- protected void validateConfigFile(ConfigFile configFile,
- FileSystem fileSystem) throws IOException {
- }
- }
-
- //@Test
- public void testConfigFiles() throws IOException {
- ClientProvider clientProvider = new ClientProvider();
- FileSystem mockFs = createNiceMock(FileSystem.class);
- expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes();
- replay(mockFs);
-
- ConfigFile configFile = new ConfigFile();
- List<ConfigFile> configFiles = new ArrayList<>();
- configFiles.add(configFile);
-
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- Assert.fail(EXCEPTION_PREFIX + "null file type");
- } catch (IllegalArgumentException e) {
- }
-
- configFile.setType(TypeEnum.TEMPLATE);
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- Assert.fail(EXCEPTION_PREFIX + "empty src_file for type template");
- } catch (IllegalArgumentException e) {
- }
-
- configFile.setSrcFile("srcfile");
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- Assert.fail(EXCEPTION_PREFIX + "empty dest file");
- } catch (IllegalArgumentException e) {
- }
-
- configFile.setDestFile("destfile");
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- configFile = new ConfigFile();
- configFile.setType(TypeEnum.JSON);
- configFile.setSrcFile(null);
- configFile.setDestFile("path/destfile2");
- configFiles.add(configFile);
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- Assert.fail(EXCEPTION_PREFIX + "dest file with multiple path elements");
- } catch (IllegalArgumentException e) {
- }
-
- configFile.setDestFile("/path/destfile2");
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- configFile.setDestFile("destfile");
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- Assert.fail(EXCEPTION_PREFIX + "duplicate dest file");
- } catch (IllegalArgumentException e) {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java
deleted file mode 100644
index c51eee2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultClientProvider;
-import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
-import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderService;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.Artifact.TypeEnum;
-import org.apache.hadoop.yarn.service.provider.docker.DockerClientProvider;
-import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
-import org.apache.hadoop.yarn.service.provider.docker.DockerProviderService;
-import org.apache.hadoop.yarn.service.provider.tarball.TarballClientProvider;
-import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
-import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderService;
-import org.junit.Test;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test provider factories.
- */
-public class TestProviderFactory {
- //@Test
- public void testDockerFactory() throws Throwable {
- ProviderFactory factory = ProviderFactory
- .createSliderProviderFactory(new Artifact().type(TypeEnum.DOCKER));
- assertTrue(factory instanceof DockerProviderFactory);
- assertTrue(factory.createClientProvider() instanceof DockerClientProvider);
- assertTrue(factory.createServerProvider() instanceof DockerProviderService);
- assertTrue(ProviderFactory.getProviderService(new Artifact()
- .type(TypeEnum.DOCKER)) instanceof DockerProviderService);
- }
-
- //@Test
- public void testTarballFactory() throws Throwable {
- ProviderFactory factory = ProviderFactory
- .createSliderProviderFactory(new Artifact().type(TypeEnum.TARBALL));
- assertTrue(factory instanceof TarballProviderFactory);
- assertTrue(factory.createClientProvider() instanceof TarballClientProvider);
- assertTrue(factory.createServerProvider() instanceof
- TarballProviderService);
- assertTrue(ProviderFactory.getProviderService(new Artifact()
- .type(TypeEnum.TARBALL)) instanceof TarballProviderService);
- }
-
- //@Test
- public void testDefaultFactory() throws Throwable {
- ProviderFactory factory = ProviderFactory
- .createSliderProviderFactory(null);
- assertTrue(factory instanceof DefaultProviderFactory);
- assertTrue(factory.createClientProvider() instanceof DefaultClientProvider);
- assertTrue(factory.createServerProvider() instanceof DefaultProviderService);
- assertTrue(ProviderFactory.getProviderService(null) instanceof
- DefaultProviderService);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java
deleted file mode 100644
index 5743119..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.registry;
-
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.Arrays;
-
-/**
- * Test config set name validation.
- */
-public class TestConfigSetNaming {
-
- void assertValid(String name) {
- PublishedConfigSet.validateName(name);
- }
-
- void assertInvalid(String name) {
- try {
- PublishedConfigSet.validateName(name);
- Assert.fail("Invalid name was unexpectedly parsed: " + name);
- } catch (IllegalArgumentException expected) {
- // expected
- }
- }
-
- //@Test
- public void testLowerCase() throws Throwable {
- assertValid("abcdefghijklmnopqrstuvwxyz");
- }
-
- //@Test
- public void testUpperCaseInvalid() throws Throwable {
- assertInvalid("ABCDEFGHIJKLMNOPQRSTUVWXYZ");
- }
-
- //@Test
- public void testNumbers() throws Throwable {
- assertValid("01234567890");
- }
-
- //@Test
- public void testChars() throws Throwable {
- assertValid("a-_+");
- }
-
- //@Test
- public void testInvalids() throws Throwable {
- for (String s : Arrays.asList(
- "",
- " ",
- "*",
- "a/b",
- "b\\a",
- "\"",
- "'",
- "\u0000",
- "\u0f00",
- "key.value",
- "-",
- "+",
- "_",
- "?")) {
- assertInvalid(s);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java
deleted file mode 100644
index 5e6b650..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.registry;
-
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.slider.core.registry.SliderRegistryUtils;
-import org.apache.slider.utils.SliderTestUtils;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test registry paths.
- */
-public class TestRegistryPaths {
-
- //@Test
- public void testHomedirKerberos() throws Throwable {
- String home = RegistryUtils.homePathForUser("hbase@HADOOP.APACHE.ORG");
- try {
- assertEquals("/users/hbase", home);
- } catch (AssertionError e) {
- SliderTestUtils.skip("homedir filtering not yet in hadoop registry " +
- "module");
- }
- }
-
- //@Test
- public void testHomedirKerberosHost() throws Throwable {
- String home = RegistryUtils.homePathForUser("hbase/localhost@HADOOP" +
- ".APACHE.ORG");
- try {
- assertEquals("/users/hbase", home);
- } catch (AssertionError e) {
- SliderTestUtils.skip("homedir filtering not yet in hadoop registry " +
- "module");
- }
- }
-
- //@Test
- public void testRegistryPathForInstance() throws Throwable {
- String path = SliderRegistryUtils.registryPathForInstance("instance");
- assertTrue(path.endsWith("/instance"));
- }
-
- //@Test
- public void testPathResolution() throws Throwable {
- String home = RegistryUtils.homePathForCurrentUser();
- assertEquals(home, SliderRegistryUtils.resolvePath("~"));
- assertEquals(home +"/", SliderRegistryUtils.resolvePath("~/"));
- assertEquals(home +"/something", SliderRegistryUtils.resolvePath(
- "~/something"));
- assertEquals("~unresolved", SliderRegistryUtils.resolvePath(
- "~unresolved"));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java
deleted file mode 100644
index a63f4b6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.ServiceOperations;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.services.workflow.ServiceThreadFactory;
-import org.apache.slider.server.services.workflow.WorkflowExecutorService;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test AM actions.
- */
-public class TestActions {
- protected static final Logger LOG =
- LoggerFactory.getLogger(TestActions.class);
-
- private QueueService queues;
- private WorkflowExecutorService<ExecutorService> executorService;
-
-
- @Before
- public void createService() {
- queues = new QueueService();
-
- Configuration conf = new Configuration();
- queues.init(conf);
-
- queues.start();
-
- executorService = new WorkflowExecutorService<>("AmExecutor",
- Executors.newCachedThreadPool(
- new ServiceThreadFactory("AmExecutor", true)));
-
- executorService.init(conf);
- executorService.start();
- }
-
- @After
- public void destroyService() {
- ServiceOperations.stop(executorService);
- ServiceOperations.stop(queues);
- }
-
- //@Test
- public void testBasicService() throws Throwable {
- queues.start();
- }
-
- //@Test
- public void testDelayLogic() throws Throwable {
- ActionNoteExecuted action = new ActionNoteExecuted("", 1000);
- long now = System.currentTimeMillis();
-
- long delay = action.getDelay(TimeUnit.MILLISECONDS);
- assertTrue(delay >= 800);
- assertTrue(delay <= 1800);
-
- ActionNoteExecuted a2 = new ActionNoteExecuted("a2", 10000);
- assertTrue(action.compareTo(a2) < 0);
- assertTrue(a2.compareTo(action) > 0);
- assertEquals(0, action.compareTo(action));
-
- }
-
- //@Test
- public void testActionDelayedExecutorTermination() throws Throwable {
- long start = System.currentTimeMillis();
-
- ActionStopQueue stopAction = new ActionStopQueue(1000);
- queues.scheduledActions.add(stopAction);
- queues.run();
- AsyncAction take = queues.actionQueue.take();
- assertEquals(take, stopAction);
- long stop = System.currentTimeMillis();
- assertTrue(stop - start > 500);
- assertTrue(stop - start < 1500);
- }
-
- //@Test
- public void testImmediateQueue() throws Throwable {
- ActionNoteExecuted noteExecuted = new ActionNoteExecuted("executed", 0);
- queues.put(noteExecuted);
- queues.put(new ActionStopQueue(0));
- QueueExecutor ex = new QueueExecutor(queues);
- ex.run();
- assertTrue(queues.actionQueue.isEmpty());
- assertTrue(noteExecuted.executed.get());
- }
-
- //@Test
- public void testActionOrdering() throws Throwable {
-
- ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500);
- ActionStopQueue stop = new ActionStopQueue(1500);
- ActionNoteExecuted note2 = new ActionNoteExecuted("note2", 800);
-
- List<AsyncAction> actions = Arrays.asList(note1, stop, note2);
- Collections.sort(actions);
- assertEquals(actions.get(0), note1);
- assertEquals(actions.get(1), note2);
- assertEquals(actions.get(2), stop);
- }
-
- //@Test
- public void testDelayedQueueWithReschedule() throws Throwable {
-
- ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500);
- ActionStopQueue stop = new ActionStopQueue(1500);
- ActionNoteExecuted note2 = new ActionNoteExecuted("note2", 800);
-
- assertTrue(note2.compareTo(stop) < 0);
- assertTrue(note1.getNanos() < note2.getNanos());
- assertTrue(note2.getNanos() < stop.getNanos());
- queues.schedule(note1);
- queues.schedule(note2);
- queues.schedule(stop);
- // async to sync expected to run in order
- runQueuesToCompletion();
- assertTrue(note1.executed.get());
- assertTrue(note2.executed.get());
- }
-
- public void runQueuesToCompletion() {
- queues.run();
- assertTrue(queues.scheduledActions.isEmpty());
- assertFalse(queues.actionQueue.isEmpty());
- QueueExecutor ex = new QueueExecutor(queues);
- ex.run();
- // flush all stop commands from the queue
- queues.flushActionQueue(ActionStopQueue.class);
-
- assertTrue(queues.actionQueue.isEmpty());
- }
-
- //@Test
- public void testRenewedActionFiresOnceAtLeast() throws Throwable {
- ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500);
- RenewingAction renewer = new RenewingAction(
- note1,
- 500,
- 100,
- TimeUnit.MILLISECONDS,
- 3);
- queues.schedule(renewer);
- ActionStopQueue stop = new ActionStopQueue(4, TimeUnit.SECONDS);
- queues.schedule(stop);
- // this runs all the delayed actions FIRST, so can't be used
- // to play tricks of renewing actions ahead of the stop action
- runQueuesToCompletion();
- assertEquals(1, renewer.executionCount.intValue());
- assertEquals(1, note1.executionCount.intValue());
- // assert the renewed item is back in
- assertTrue(queues.scheduledActions.contains(renewer));
- }
-
-
- //@Test
- public void testRenewingActionOperations() throws Throwable {
- ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500);
- RenewingAction renewer = new RenewingAction(
- note1,
- 100,
- 100,
- TimeUnit.MILLISECONDS,
- 3);
- queues.renewing("note", renewer);
- assertTrue(queues.removeRenewingAction("note"));
- queues.stop();
- assertTrue(queues.waitForServiceToStop(10000));
- }
-
- /**
- * Test action.
- */
- public class ActionNoteExecuted extends AsyncAction {
- private final AtomicBoolean executed = new AtomicBoolean(false);
- private final AtomicLong executionTimeNanos = new AtomicLong();
- private final AtomicLong executionCount = new AtomicLong();
-
- public ActionNoteExecuted(String text, int delay) {
- super(text, delay);
- }
-
- @Override
- public void execute(
- SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
- LOG.info("Executing {}", name);
- executed.set(true);
- executionTimeNanos.set(System.nanoTime());
- executionCount.incrementAndGet();
- LOG.info(this.toString());
-
- synchronized (this) {
- this.notify();
- }
- }
-
- @Override
- public String toString() {
- return super.toString() + " executed=" + executed.get() + "; count=" +
- executionCount.get() + ";";
- }
-
- public long getExecutionCount() {
- return executionCount.get();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java
deleted file mode 100644
index 6f4ca42..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.providers.PlacementPolicy;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-
-import static org.apache.slider.api.ResourceKeys.COMPONENT_PLACEMENT_POLICY;
-
-/**
- * Class for basis of Anti-affine placement tests; sets up role2
- * for anti-affinity.
- */
-public class BaseMockAppStateAATest extends BaseMockAppStateTest
- implements MockRoles {
-
- /** Role status for the base AA role. */
- private RoleStatus aaRole;
-
- /** Role status for the AA role requiring a node with the gpu label. */
- private RoleStatus gpuRole;
-
- @Override
- public Application buildApplication() {
- Application application = factory.newApplication(0, 0, 0)
- .name(getValidTestName());
- application.getComponent(ROLE1).getConfiguration().setProperty(
- COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy
- .ANTI_AFFINITY_REQUIRED));
- application.getComponent(ROLE1).getConfiguration().setProperty(
- ResourceKeys.YARN_LABEL_EXPRESSION, LABEL_GPU);
- application.getComponent(ROLE2).getConfiguration().setProperty(
- COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy
- .ANTI_AFFINITY_REQUIRED));
- return application;
- }
-
-
- @Override
- public void setup() throws Exception {
- super.setup();
- aaRole = lookupRole(ROLE2);
- gpuRole = lookupRole(ROLE1);
- }
-
- protected RoleStatus getAaRole() {
- return aaRole;
- }
-
- protected RoleStatus getGpuRole() {
- return gpuRole;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.java
deleted file mode 100644
index e0f7c1f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.core.main.LauncherExitCodes;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.NodeMap;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Test Anti-affine placement with a cluster of size 1.
- */
-public class TestMockAppStateAAOvercapacity extends BaseMockAppStateAATest
- implements MockRoles {
-
- private static final int NODES = 1;
-
- @Override
- public MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(NODES, 1);
- }
-
- void assertAllContainersAA() {
- assertAllContainersAA(getAaRole().getKey());
- }
-
- /**
- *
- * @throws Throwable
- */
- //@Test
- public void testOvercapacityRecovery() throws Throwable {
- RoleStatus aaRole = getAaRole();
-
- describe("Ask for 1 more than the no of available nodes;" +
- "verify the state. kill the allocated container and review");
- //more than expected
- int desired = 3;
- aaRole.setDesired(desired);
- assertTrue(appState.getRoleHistory().canPlaceAANodes());
-
- //first request
- List<AbstractRMOperation> operations =
- appState.reviewRequestAndReleaseNodes();
- assertTrue(aaRole.isAARequestOutstanding());
- assertEquals(1, aaRole.getRequested());
- assertEquals(desired - 1, aaRole.getAAPending());
- List<AbstractRMOperation> operationsOut = new ArrayList<>();
- // allocate and re-submit
- List<RoleInstance> instances = submitOperations(operations,
- EMPTY_ID_LIST, operationsOut);
- assertEquals(1, instances.size());
- assertAllContainersAA();
-
- // expect an outstanding AA request to be unsatisfied
- assertTrue(aaRole.getRunning() < aaRole.getDesired());
- assertEquals(0, aaRole.getRequested());
- assertFalse(aaRole.isAARequestOutstanding());
- assertEquals(desired - 1, aaRole.getAAPending());
- List<Container> allocatedContainers = engine.execute(operations,
- EMPTY_ID_LIST);
- assertEquals(0, allocatedContainers.size());
-
- // now lets trigger a failure
- NodeMap nodemap = cloneNodemap();
- assertEquals(1, nodemap.size());
-
- RoleInstance instance = instances.get(0);
- ContainerId cid = instance.getContainerId();
-
- AppState.NodeCompletionResult result = appState.onCompletedContainer(
- containerStatus(cid, LauncherExitCodes.EXIT_TASK_LAUNCH_FAILURE));
- assertTrue(result.containerFailed);
-
- assertEquals(1, aaRole.getFailed());
- assertEquals(0, aaRole.getRunning());
- List<NodeInstance> availablePlacements = appState.getRoleHistory()
- .findNodeForNewAAInstance(aaRole);
- assertEquals(1, availablePlacements.size());
- describe("expecting a successful review with available placements of " +
- availablePlacements);
- operations = appState.reviewRequestAndReleaseNodes();
- assertEquals(1, operations.size());
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[53/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
deleted file mode 100644
index 982a114..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
+++ /dev/null
@@ -1,493 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.compinstance;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.client.api.NMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.service.ServiceScheduler;
-import org.apache.hadoop.yarn.service.api.records.ContainerState;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
-import org.apache.hadoop.yarn.state.SingleArcTransition;
-import org.apache.hadoop.yarn.state.StateMachine;
-import org.apache.hadoop.yarn.state.StateMachineFactory;
-import org.apache.hadoop.yarn.util.BoundedAppender;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import org.apache.hadoop.yarn.service.servicemonitor.probe.ProbeStatus;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.text.MessageFormat;
-import java.util.Date;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
-
-import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.KILLED_BY_APPMASTER;
-import static org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.*;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.*;
-
-public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
- Comparable<ComponentInstance> {
- private static final Logger LOG =
- LoggerFactory.getLogger(ComponentInstance.class);
-
- private StateMachine<ComponentInstanceState, ComponentInstanceEventType,
- ComponentInstanceEvent> stateMachine;
- private Component component;
- private final ReadLock readLock;
- private final WriteLock writeLock;
-
- private ComponentInstanceId compInstanceId = null;
- private Path compInstanceDir;
- private Container container;
- private YarnRegistryViewForProviders yarnRegistryOperations;
- private FileSystem fs;
- private boolean timelineServiceEnabled = false;
- private ServiceTimelinePublisher serviceTimelinePublisher;
- private ServiceScheduler scheduler;
- private BoundedAppender diagnostics = new BoundedAppender(64 * 1024);
- private volatile ScheduledFuture containerStatusFuture;
- private volatile ContainerStatus status;
- private long containerStartedTime = 0;
- // This container object is used for rest API query
- private org.apache.hadoop.yarn.service.api.records.Container containerSpec;
-
- private static final StateMachineFactory<ComponentInstance,
- ComponentInstanceState, ComponentInstanceEventType, ComponentInstanceEvent>
- stateMachineFactory =
- new StateMachineFactory<ComponentInstance, ComponentInstanceState,
- ComponentInstanceEventType, ComponentInstanceEvent>(INIT)
- .addTransition(INIT, RUNNING_BUT_UNREADY, STARTED,
- new ContainerStartedTransition())
-
- //From Running
- .addTransition(RUNNING_BUT_UNREADY, INIT, STOP,
- new ContainerStoppedTransition())
- .addTransition(RUNNING_BUT_UNREADY, READY, BECOME_READY,
- new ContainerBecomeReadyTransition())
-
- // FROM READY
- .addTransition(READY, RUNNING_BUT_UNREADY, BECOME_NOT_READY,
- new ContainerBecomeNotReadyTransition())
- .addTransition(READY, INIT, STOP, new ContainerStoppedTransition())
- .installTopology();
-
-
-
- public ComponentInstance(Component component,
- ComponentInstanceId compInstanceId) {
- this.stateMachine = stateMachineFactory.make(this);
- this.component = component;
- this.compInstanceId = compInstanceId;
- this.scheduler = component.getScheduler();
- this.yarnRegistryOperations =
- component.getScheduler().getYarnRegistryOperations();
- this.serviceTimelinePublisher =
- component.getScheduler().getServiceTimelinePublisher();
- if (YarnConfiguration
- .timelineServiceV2Enabled(component.getScheduler().getConfig())) {
- this.timelineServiceEnabled = true;
- }
- ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
- this.readLock = lock.readLock();
- this.writeLock = lock.writeLock();
- this.fs = scheduler.getContext().fs.getFileSystem();
- }
-
- private static class ContainerStartedTransition extends BaseTransition {
- @Override public void transition(ComponentInstance compInstance,
- ComponentInstanceEvent event) {
- // Query container status for ip and host
- compInstance.containerStatusFuture =
- compInstance.scheduler.executorService.scheduleAtFixedRate(
- new ContainerStatusRetriever(compInstance.scheduler,
- compInstance.getContainerId(), compInstance), 0, 1,
- TimeUnit.SECONDS);
-
- org.apache.hadoop.yarn.service.api.records.Container container =
- new org.apache.hadoop.yarn.service.api.records.Container();
- container.setId(compInstance.getContainerId().toString());
- container.setLaunchTime(new Date());
- container.setState(ContainerState.RUNNING_BUT_UNREADY);
- container.setBareHost(compInstance.container.getNodeId().getHost());
- container.setComponentName(compInstance.getCompInstanceName());
- if (compInstance.containerSpec != null) {
- // remove the previous container.
- compInstance.getCompSpec().removeContainer(compInstance.containerSpec);
- }
- compInstance.containerSpec = container;
- compInstance.getCompSpec().addContainer(container);
- compInstance.containerStartedTime = System.currentTimeMillis();
-
- if (compInstance.timelineServiceEnabled) {
- compInstance.serviceTimelinePublisher
- .componentInstanceStarted(container, compInstance);
- }
- }
- }
-
- private static class ContainerBecomeReadyTransition extends BaseTransition {
- @Override
- public void transition(ComponentInstance compInstance,
- ComponentInstanceEvent event) {
- compInstance.component.incContainersReady();
- compInstance.containerSpec.setState(ContainerState.READY);
- }
- }
-
- private static class ContainerBecomeNotReadyTransition extends BaseTransition {
- @Override
- public void transition(ComponentInstance compInstance,
- ComponentInstanceEvent event) {
- compInstance.component.decContainersReady();
- compInstance.containerSpec.setState(ContainerState.RUNNING_BUT_UNREADY);
- }
- }
-
- private static class ContainerStoppedTransition extends BaseTransition {
- @Override
- public void transition(ComponentInstance compInstance,
- ComponentInstanceEvent event) {
- // re-ask the failed container.
- Component comp = compInstance.component;
- comp.requestContainers(1);
- LOG.info(compInstance.getCompInstanceId()
- + ": Container completed. Requested a new container." + System
- .lineSeparator() + " exitStatus={}, diagnostics={}.",
- event.getStatus().getExitStatus(),
- event.getStatus().getDiagnostics());
- String containerDiag =
- compInstance.getCompInstanceId() + ": " + event.getStatus()
- .getDiagnostics();
- compInstance.diagnostics.append(containerDiag + System.lineSeparator());
-
- boolean shouldExit = false;
- // check if it exceeds the failure threshold
- if (comp.currentContainerFailure > comp.maxContainerFailurePerComp) {
- String exitDiag = MessageFormat.format(
- "[COMPONENT {0}]: Failed {1} times, exceeded the limit - {2}. Shutting down now... "
- + System.lineSeparator(),
- comp.getName(), comp.currentContainerFailure, comp.maxContainerFailurePerComp);
- compInstance.diagnostics.append(exitDiag);
- // append to global diagnostics that will be reported to RM.
- comp.getScheduler().getDiagnostics().append(containerDiag);
- comp.getScheduler().getDiagnostics().append(exitDiag);
- LOG.warn(exitDiag);
- shouldExit = true;
- }
-
- // clean up registry
- // hdfs dir content will be overwritten when a new container gets started,
- // so no need remove.
- compInstance.scheduler.executorService
- .submit(compInstance::cleanupRegistry);
-
- // remove the failed ContainerId -> CompInstance mapping
- comp.getScheduler().removeLiveCompInstance(event.getContainerId());
-
- if (compInstance.timelineServiceEnabled) {
- // record in ATS
- compInstance.serviceTimelinePublisher
- .componentInstanceFinished(compInstance,
- event.getStatus().getExitStatus(), event.getStatus().getState(),
- containerDiag);
- }
-
- compInstance.containerSpec.setState(ContainerState.STOPPED);
- if (shouldExit) {
- // Sleep for 5 seconds in hope that the state can be recorded in ATS.
- // in case there's a client polling the comp state, it can be notified.
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- LOG.error("Interrupted on sleep while exiting.", e);
- }
- ExitUtil.terminate(-1);
- }
- }
- }
-
- public ComponentInstanceState getState() {
- this.readLock.lock();
-
- try {
- return this.stateMachine.getCurrentState();
- } finally {
- this.readLock.unlock();
- }
- }
-
- @Override
- public void handle(ComponentInstanceEvent event) {
- try {
- writeLock.lock();
- ComponentInstanceState oldState = getState();
- try {
- stateMachine.doTransition(event.getType(), event);
- } catch (InvalidStateTransitionException e) {
- LOG.error(getCompInstanceId() + ": Invalid event " + event.getType() +
- " at " + oldState, e);
- }
- if (oldState != getState()) {
- LOG.info(getCompInstanceId() + " Transitioned from " + oldState + " to "
- + getState() + " on " + event.getType() + " event");
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- public void setContainer(Container container) {
- this.container = container;
- this.compInstanceId.setContainerId(container.getId());
- }
-
- public String getCompInstanceName() {
- return compInstanceId.getCompInstanceName();
- }
-
- public ContainerStatus getContainerStatus() {
- return status;
- }
-
- public void updateContainerStatus(ContainerStatus status) {
- this.status = status;
- org.apache.hadoop.yarn.service.api.records.Container container =
- getCompSpec().getContainer(getContainerId().toString());
- if (container != null) {
- container.setIp(StringUtils.join(",", status.getIPs()));
- container.setHostname(status.getHost());
- if (timelineServiceEnabled) {
- serviceTimelinePublisher.componentInstanceUpdated(container);
- }
- }
- updateServiceRecord(yarnRegistryOperations, status);
- }
-
- public ContainerId getContainerId() {
- return container.getId();
- }
-
- public String getCompName() {
- return compInstanceId.getCompName();
- }
-
- public void setCompInstanceDir(Path dir) {
- this.compInstanceDir = dir;
- }
-
- public Component getComponent() {
- return component;
- }
-
- public Container getContainer() {
- return container;
- }
-
- public ComponentInstanceId getCompInstanceId() {
- return compInstanceId;
- }
-
- public NodeId getNodeId() {
- return this.container.getNodeId();
- }
-
- public org.apache.hadoop.yarn.service.api.records.Component getCompSpec() {
- return component.getComponentSpec();
- }
-
- private static class BaseTransition implements
- SingleArcTransition<ComponentInstance, ComponentInstanceEvent> {
-
- @Override public void transition(ComponentInstance compInstance,
- ComponentInstanceEvent event) {
- }
- }
-
- public ProbeStatus ping() {
- if (component.getProbe() == null) {
- ProbeStatus status = new ProbeStatus();
- status.setSuccess(true);
- return status;
- }
- return component.getProbe().ping(this);
- }
-
- // Write service record into registry
- private void updateServiceRecord(
- YarnRegistryViewForProviders yarnRegistry, ContainerStatus status) {
- ServiceRecord record = new ServiceRecord();
- String containerId = status.getContainerId().toString();
- record.set(YarnRegistryAttributes.YARN_ID, containerId);
- record.description = getCompInstanceName();
- record.set(YarnRegistryAttributes.YARN_PERSISTENCE,
- PersistencePolicies.CONTAINER);
- record.set("yarn:ip", status.getIPs());
- record.set("yarn:hostname", status.getHost());
- try {
- yarnRegistry
- .putComponent(RegistryPathUtils.encodeYarnID(containerId), record);
- } catch (IOException e) {
- LOG.error(
- "Failed to update service record in registry: " + containerId + "");
- }
- }
-
- // Release the container , cleanup registry, hdfs dir, and record in ATS
- public void destroy() {
- LOG.info(getCompInstanceId() + ": Flexed down by user, destroying.");
- diagnostics.append(getCompInstanceId() + ": Flexed down by user");
- if (container != null) {
- scheduler.removeLiveCompInstance(container.getId());
- component.getScheduler().getAmRMClient()
- .releaseAssignedContainer(container.getId());
- getCompSpec().removeContainer(containerSpec);
- }
- if (timelineServiceEnabled) {
- serviceTimelinePublisher
- .componentInstanceFinished(this, KILLED_BY_APPMASTER, COMPLETE,
- diagnostics.toString());
- }
- scheduler.executorService.submit(this::cleanupRegistryAndCompHdfsDir);
- }
-
- private void cleanupRegistry() {
- ContainerId containerId = getContainerId();
- String cid = RegistryPathUtils.encodeYarnID(containerId.toString());
- try {
- yarnRegistryOperations.deleteComponent(getCompInstanceId(), cid);
- } catch (IOException e) {
- LOG.error(getCompInstanceId() + ": Failed to delete registry", e);
- }
- }
-
- //TODO Maybe have a dedicated cleanup service.
- public void cleanupRegistryAndCompHdfsDir() {
- cleanupRegistry();
- try {
- if (compInstanceDir != null && fs.exists(compInstanceDir)) {
- boolean deleted = fs.delete(compInstanceDir, true);
- if (!deleted) {
- LOG.error(getCompInstanceId()
- + ": Failed to delete component instance dir: "
- + compInstanceDir);
- } else {
- LOG.info(getCompInstanceId() + ": Deleted component instance dir: "
- + compInstanceDir);
- }
- }
- } catch (IOException e) {
- LOG.warn(getCompInstanceId() + ": Failed to delete directory", e);
- }
- }
-
- // Query container status until ip and hostname are available and update
- // the service record into registry service
- private static class ContainerStatusRetriever implements Runnable {
- private ContainerId containerId;
- private NodeId nodeId;
- private NMClient nmClient;
- private ComponentInstance instance;
- ContainerStatusRetriever(ServiceScheduler scheduler,
- ContainerId containerId, ComponentInstance instance) {
- this.containerId = containerId;
- this.nodeId = instance.getNodeId();
- this.nmClient = scheduler.getNmClient().getClient();
- this.instance = instance;
- }
- @Override public void run() {
- ContainerStatus status = null;
- try {
- status = nmClient.getContainerStatus(containerId, nodeId);
- } catch (Exception e) {
- if (e instanceof YarnException) {
- throw new YarnRuntimeException(
- instance.compInstanceId + " Failed to get container status on "
- + nodeId + " , cancelling.", e);
- }
- LOG.error(instance.compInstanceId + " Failed to get container status on "
- + nodeId + ", will try again", e);
- return;
- }
- if (SliderUtils.isEmpty(status.getIPs()) || SliderUtils
- .isUnset(status.getHost())) {
- return;
- }
- instance.updateContainerStatus(status);
- LOG.info(
- instance.compInstanceId + " IP = " + status.getIPs() + ", host = "
- + status.getHost() + ", cancel container status retriever");
- instance.containerStatusFuture.cancel(false);
- }
- }
-
- @Override
- public int compareTo(ComponentInstance to) {
- long delta = containerStartedTime - to.containerStartedTime;
- if (delta == 0) {
- return getCompInstanceId().compareTo(to.getCompInstanceId());
- } else if (delta < 0) {
- return -1;
- } else {
- return 1;
- }
- }
-
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- ComponentInstance instance = (ComponentInstance) o;
-
- if (containerStartedTime != instance.containerStartedTime)
- return false;
- return compInstanceId.equals(instance.compInstanceId);
- }
-
- @Override public int hashCode() {
- int result = compInstanceId.hashCode();
- result = 31 * result + (int) (containerStartedTime ^ (containerStartedTime
- >>> 32));
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEvent.java
deleted file mode 100644
index 14a9e09..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEvent.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.compinstance;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.event.AbstractEvent;
-
-public class ComponentInstanceEvent
- extends AbstractEvent<ComponentInstanceEventType> {
-
- private ContainerId id;
- private ContainerStatus status;
- private boolean shouldDestroy = false;
-
- public ComponentInstanceEvent(ContainerId containerId,
- ComponentInstanceEventType componentInstanceEventType) {
- super(componentInstanceEventType);
- this.id = containerId;
- }
-
- public ContainerId getContainerId() {
- return id;
- }
-
- public ContainerStatus getStatus() {
- return this.status;
- }
-
- public ComponentInstanceEvent setStatus(ContainerStatus status) {
- this.status = status;
- return this;
- }
-
- public void setShouldDestroy() {
- shouldDestroy = true;
- }
-
- public boolean shouldDestroy() {
- return shouldDestroy;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEventType.java
deleted file mode 100644
index b3fe1e6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEventType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.compinstance;
-
-public enum ComponentInstanceEventType {
-
- STARTED,
- STOP,
- BECOME_READY,
- BECOME_NOT_READY
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceId.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceId.java
deleted file mode 100644
index c3c55d9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceId.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.compinstance;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-
-public class ComponentInstanceId implements Comparable<ComponentInstanceId> {
-
- private long Id;
- private String name;
- private ContainerId containerId;
-
- public ComponentInstanceId(long id, String name) {
- Id = id;
- this.name = name;
- }
-
- public long getId() {
- return Id;
- }
-
- public String getCompName() {
- return name;
- }
-
- public String getCompInstanceName() {
- return getCompName() + "-" + getId();
- }
-
- public void setContainerId(ContainerId containerId) {
- this.containerId = containerId;
- }
-
- @Override
- public String toString() {
- if (containerId == null) {
- return "[COMPINSTANCE " + getCompInstanceName() + "]";
- } else {
- return "[COMPINSTANCE " + getCompInstanceName() + " : " + containerId + "]";
- }
- }
-
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- ComponentInstanceId that = (ComponentInstanceId) o;
-
- if (getId() != that.getId())
- return false;
- return getCompName() != null ? getCompName().equals(that.getCompName()) :
- that.getCompName() == null;
-
- }
-
- @Override public int hashCode() {
- int result = (int) (getId() ^ (getId() >>> 32));
- result = 31 * result + (getCompName() != null ? getCompName().hashCode() : 0);
- return result;
- }
-
- @Override
- public int compareTo(ComponentInstanceId to) {
- int delta = this.getCompName().compareTo(to.getCompName());
- if (delta == 0) {
- return Long.compare(this.getId(), to.getId());
- } else if (delta < 0) {
- return -1;
- } else {
- return 1;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceState.java
deleted file mode 100644
index f2d8cea..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceState.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.compinstance;
-
-public enum ComponentInstanceState {
- INIT,
- RUNNING_BUT_UNREADY,
- READY,
- UPGRADING
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
deleted file mode 100644
index 331871a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ /dev/null
@@ -1,493 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.component;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceId;
-import org.apache.hadoop.yarn.service.ContainerFailureTracker;
-import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.hadoop.yarn.service.ServiceScheduler;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
-import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
-import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
-import org.apache.hadoop.yarn.state.MultipleArcTransition;
-import org.apache.hadoop.yarn.state.SingleArcTransition;
-import org.apache.hadoop.yarn.state.StateMachine;
-import org.apache.hadoop.yarn.state.StateMachineFactory;
-import org.apache.hadoop.yarn.util.Apps;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.servicemonitor.probe.MonitorUtils;
-import org.apache.hadoop.yarn.service.servicemonitor.probe.Probe;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.*;
-import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.STARTED;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.STOP;
-import static org.apache.hadoop.yarn.service.component.ComponentState.*;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_THRESHOLD;
-
-public class Component implements EventHandler<ComponentEvent> {
- private static final Logger LOG = LoggerFactory.getLogger(Component.class);
-
- private org.apache.hadoop.yarn.service.api.records.Component componentSpec;
- private long allocateId;
- private Priority priority;
- private ServiceMetrics componentMetrics;
- private ServiceScheduler scheduler;
- private ServiceContext context;
- private AMRMClientAsync<ContainerRequest> amrmClient;
- private AtomicLong instanceIdCounter = new AtomicLong();
- private Map<ComponentInstanceId, ComponentInstance> compInstances =
- new ConcurrentHashMap<>();
- // component instances to be assigned with a container
- private List<ComponentInstance> pendingInstances = new LinkedList<>();
- private ContainerFailureTracker failureTracker;
- private Probe probe;
- private final ReentrantReadWriteLock.ReadLock readLock;
- private final ReentrantReadWriteLock.WriteLock writeLock;
- public int maxContainerFailurePerComp;
- // The number of containers failed since last reset. This excludes preempted,
- // disk_failed containers etc. This will be reset to 0 periodically.
- public volatile int currentContainerFailure;
-
- private StateMachine<ComponentState, ComponentEventType, ComponentEvent>
- stateMachine;
- private AsyncDispatcher compInstanceDispatcher;
- private static final StateMachineFactory<Component, ComponentState, ComponentEventType, ComponentEvent>
- stateMachineFactory =
- new StateMachineFactory<Component, ComponentState, ComponentEventType, ComponentEvent>(
- INIT)
- // INIT will only got to FLEXING
- .addTransition(INIT, EnumSet.of(STABLE, FLEXING),
- FLEX, new FlexComponentTransition())
-
- // container allocated by RM
- .addTransition(FLEXING, FLEXING, CONTAINER_ALLOCATED,
- new ContainerAllocatedTransition())
- // container launched on NM
- .addTransition(FLEXING, EnumSet.of(STABLE, FLEXING),
- CONTAINER_STARTED, new ContainerStartedTransition())
- // container failed while flexing
- .addTransition(FLEXING, FLEXING, CONTAINER_COMPLETED,
- new ContainerCompletedTransition())
- // Flex while previous flex is still in progress
- .addTransition(FLEXING, EnumSet.of(FLEXING), FLEX,
- new FlexComponentTransition())
-
- // container failed while stable
- .addTransition(STABLE, FLEXING, CONTAINER_COMPLETED,
- new ContainerCompletedTransition())
- // Ignore surplus container
- .addTransition(STABLE, STABLE, CONTAINER_ALLOCATED,
- new ContainerAllocatedTransition())
- // Flex by user
- // For flex up, go to FLEXING state
- // For flex down, go to STABLE state
- .addTransition(STABLE, EnumSet.of(STABLE, FLEXING),
- FLEX, new FlexComponentTransition())
- .installTopology();
-
- public Component(
- org.apache.hadoop.yarn.service.api.records.Component component,
- long allocateId, ServiceContext context) {
- this.allocateId = allocateId;
- this.priority = Priority.newInstance((int) allocateId);
- this.componentSpec = component;
- componentMetrics = ServiceMetrics.register(component.getName(),
- "Metrics for component " + component.getName());
- componentMetrics
- .tag("type", "Metrics type [component or service]", "component");
- this.scheduler = context.scheduler;
- this.context = context;
- amrmClient = scheduler.getAmRMClient();
- ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
- this.readLock = lock.readLock();
- this.writeLock = lock.writeLock();
- this.stateMachine = stateMachineFactory.make(this);
- compInstanceDispatcher = scheduler.getCompInstanceDispatcher();
- failureTracker =
- new ContainerFailureTracker(context, this);
- probe = MonitorUtils.getProbe(componentSpec.getReadinessCheck());
- maxContainerFailurePerComp = componentSpec.getConfiguration()
- .getPropertyInt(CONTAINER_FAILURE_THRESHOLD, 10);
- createNumCompInstances(component.getNumberOfContainers());
- }
-
- private void createNumCompInstances(long count) {
- for (int i = 0; i < count; i++) {
- createOneCompInstance();
- }
- }
-
- private void createOneCompInstance() {
- ComponentInstanceId id =
- new ComponentInstanceId(instanceIdCounter.getAndIncrement(),
- componentSpec.getName());
- ComponentInstance instance = new ComponentInstance(this, id);
- compInstances.put(id, instance);
- pendingInstances.add(instance);
- }
-
- private static class FlexComponentTransition implements
- MultipleArcTransition<Component, ComponentEvent, ComponentState> {
- // For flex up, go to FLEXING state
- // For flex down, go to STABLE state
- @Override
- public ComponentState transition(Component component,
- ComponentEvent event) {
- component.setDesiredContainers((int)event.getDesired());
- if (!component.areDependenciesReady()) {
- LOG.info("[FLEX COMPONENT {}]: Flex deferred because dependencies not"
- + " satisfied.", component.getName());
- return component.getState();
- }
- if (component.getState() == INIT) {
- // This happens on init
- LOG.info("[INIT COMPONENT " + component.getName() + "]: " + event
- .getDesired() + " instances.");
- component.requestContainers(event.getDesired());
- return FLEXING;
- }
- long before = component.getComponentSpec().getNumberOfContainers();
- long delta = event.getDesired() - before;
- component.getComponentSpec().setNumberOfContainers(event.getDesired());
- if (delta > 0) {
- // Scale up
- LOG.info("[FLEX UP COMPONENT " + component.getName() + "]: scaling up from "
- + before + " to " + event.getDesired());
- component.requestContainers(delta);
- component.createNumCompInstances(delta);
- return FLEXING;
- } else if (delta < 0){
- delta = 0 - delta;
- // scale down
- LOG.info("[FLEX DOWN COMPONENT " + component.getName()
- + "]: scaling down from " + before + " to " + event.getDesired());
- List<ComponentInstance> list =
- new ArrayList<>(component.compInstances.values());
-
- // sort in Most recent -> oldest order, destroy most recent ones.
- Collections.sort(list, Collections.reverseOrder());
- for (int i = 0; i < delta; i++) {
- ComponentInstance instance = list.get(i);
- // remove the instance
- component.compInstances.remove(instance.getCompInstanceId());
- component.pendingInstances.remove(instance);
- component.componentMetrics.containersFailed.incr();
- component.componentMetrics.containersRunning.decr();
- // decrement id counter
- component.instanceIdCounter.decrementAndGet();
- instance.destroy();
- }
- return STABLE;
- } else {
- LOG.info("[FLEX COMPONENT " + component.getName() + "]: already has " +
- event.getDesired() + " instances, ignoring");
- return STABLE;
- }
- }
- }
-
- private static class ContainerAllocatedTransition extends BaseTransition {
- @Override
- public void transition(Component component, ComponentEvent event) {
- component.assignContainerToCompInstance(event.getContainer());
- }
- }
-
- private static class ContainerStartedTransition implements
- MultipleArcTransition<Component,ComponentEvent,ComponentState> {
-
- @Override public ComponentState transition(Component component,
- ComponentEvent event) {
- component.compInstanceDispatcher.getEventHandler().handle(
- new ComponentInstanceEvent(event.getInstance().getContainerId(),
- STARTED));
- component.incRunningContainers();
- return checkIfStable(component);
- }
- }
-
- private static ComponentState checkIfStable(Component component) {
- // if desired == running
- if (component.componentMetrics.containersRunning.value() == component
- .getComponentSpec().getNumberOfContainers()) {
- return STABLE;
- } else {
- return FLEXING;
- }
- }
-
- private static class ContainerCompletedTransition extends BaseTransition {
- @Override
- public void transition(Component component, ComponentEvent event) {
- component.updateMetrics(event.getStatus());
-
- // add back to pending list
- component.pendingInstances.add(event.getInstance());
- LOG.info(
- "[COMPONENT {}]: {} completed, num pending comp instances increased to {}.",
- component.getName(), event.getStatus().getContainerId(),
- component.pendingInstances.size());
- component.compInstanceDispatcher.getEventHandler().handle(
- new ComponentInstanceEvent(event.getStatus().getContainerId(),
- STOP).setStatus(event.getStatus()));
- }
- }
-
- public ServiceMetrics getCompMetrics () {
- return componentMetrics;
- }
-
- private void assignContainerToCompInstance(Container container) {
- if (pendingInstances.size() == 0) {
- LOG.info(
- "[COMPONENT {}]: No pending component instance left, release surplus container {}",
- getName(), container.getId());
- scheduler.getAmRMClient().releaseAssignedContainer(container.getId());
- componentMetrics.surplusContainers.incr();
- scheduler.getServiceMetrics().surplusContainers.incr();
- return;
- }
- ComponentInstance instance = pendingInstances.remove(0);
- LOG.info(
- "[COMPONENT {}]: {} allocated, num pending component instances reduced to {}",
- getName(), container.getId(), pendingInstances.size());
- instance.setContainer(container);
- scheduler.addLiveCompInstance(container.getId(), instance);
- LOG.info(
- "[COMPONENT {}]: Assigned {} to component instance {} and launch on host {} ",
- getName(), container.getId(), instance.getCompInstanceName(),
- container.getNodeId());
- scheduler.getContainerLaunchService()
- .launchCompInstance(scheduler.getApp(), instance, container);
- }
-
- @SuppressWarnings({ "unchecked" })
- public void requestContainers(long count) {
- Resource resource = Resource
- .newInstance(componentSpec.getResource().getMemoryMB(),
- componentSpec.getResource().getCpus());
-
- for (int i = 0; i < count; i++) {
- //TODO Once YARN-5468 is done, use that for anti-affinity
- ContainerRequest request =
- ContainerRequest.newBuilder().capability(resource).priority(priority)
- .allocationRequestId(allocateId).relaxLocality(true).build();
- amrmClient.addContainerRequest(request);
- }
- }
-
- private void setDesiredContainers(int n) {
- int delta = n - scheduler.getServiceMetrics().containersDesired.value();
- if (delta > 0) {
- scheduler.getServiceMetrics().containersDesired.incr(delta);
- } else {
- scheduler.getServiceMetrics().containersDesired.decr(delta);
- }
- componentMetrics.containersDesired.set(n);
- }
-
-
-
- private void updateMetrics(ContainerStatus status) {
- switch (status.getExitStatus()) {
- case SUCCESS:
- componentMetrics.containersSucceeded.incr();
- scheduler.getServiceMetrics().containersSucceeded.incr();
- return;
- case PREEMPTED:
- componentMetrics.containersPreempted.incr();
- scheduler.getServiceMetrics().containersPreempted.incr();
- break;
- case DISKS_FAILED:
- componentMetrics.containersDiskFailure.incr();
- scheduler.getServiceMetrics().containersDiskFailure.incr();
- break;
- default:
- break;
- }
-
- // containersFailed include preempted, disks_failed etc.
- componentMetrics.containersFailed.incr();
- scheduler.getServiceMetrics().containersFailed.incr();
-
- // dec running container
- decRunningContainers();
-
- if (Apps.shouldCountTowardsNodeBlacklisting(status.getExitStatus())) {
- String host = scheduler.getLiveInstances().get(status.getContainerId())
- .getNodeId().getHost();
- failureTracker.incNodeFailure(host);
- currentContainerFailure++ ;
- }
- }
-
- public boolean areDependenciesReady() {
- List<String> dependencies = componentSpec.getDependencies();
- if (SliderUtils.isEmpty(dependencies)) {
- return true;
- }
- for (String dependency : dependencies) {
- Component dependentComponent =
- scheduler.getAllComponents().get(dependency);
- if (dependentComponent == null) {
- LOG.error("Couldn't find dependency {} for {} (should never happen)",
- dependency, getName());
- continue;
- }
- if (dependentComponent.getNumReadyInstances() < dependentComponent
- .getNumDesiredInstances()) {
- LOG.info("[COMPONENT {}]: Dependency {} not satisfied, only {} of {}"
- + " instances are ready.", getName(), dependency,
- dependentComponent.getNumReadyInstances(),
- dependentComponent.getNumDesiredInstances());
- return false;
- }
- }
- return true;
- }
-
- private void incRunningContainers() {
- componentMetrics.containersRunning.incr();
- scheduler.getServiceMetrics().containersRunning.incr();
- }
-
- public void incContainersReady() {
- componentMetrics.containersReady.incr();
- }
-
- public void decContainersReady() {
- componentMetrics.containersReady.decr();
- }
-
- private void decRunningContainers() {
- componentMetrics.containersRunning.decr();
- scheduler.getServiceMetrics().containersRunning.decr();
- }
-
- public int getNumReadyInstances() {
- return componentMetrics.containersReady.value();
- }
-
- public int getNumRunningInstances() {
- return componentMetrics.containersRunning.value();
- }
-
- public int getNumDesiredInstances() {
- return componentMetrics.containersDesired.value();
- }
-
- public Map<ComponentInstanceId, ComponentInstance> getAllComponentInstances() {
- return compInstances;
- }
-
- public org.apache.hadoop.yarn.service.api.records.Component getComponentSpec() {
- return this.componentSpec;
- }
-
- public void resetCompFailureCount() {
- LOG.info("[COMPONENT {}]: Reset container failure count from {} to 0.",
- getName(), currentContainerFailure);
- currentContainerFailure = 0;
- failureTracker.resetContainerFailures();
- }
-
- public Probe getProbe() {
- return probe;
- }
-
- public Priority getPriority() {
- return priority;
- }
-
- public long getAllocateId() {
- return allocateId;
- }
-
- public String getName () {
- return componentSpec.getName();
- }
-
- public ComponentState getState() {
- this.readLock.lock();
-
- try {
- return this.stateMachine.getCurrentState();
- } finally {
- this.readLock.unlock();
- }
- }
- public ServiceScheduler getScheduler() {
- return scheduler;
- }
-
- @Override
- public void handle(ComponentEvent event) {
- try {
- writeLock.lock();
- ComponentState oldState = getState();
- try {
- stateMachine.doTransition(event.getType(), event);
- } catch (InvalidStateTransitionException e) {
- LOG.error(MessageFormat.format("[COMPONENT {0}]: Invalid event {1} at {2}",
- componentSpec.getName(), event.getType(), oldState), e);
- }
- if (oldState != getState()) {
- LOG.info("[COMPONENT {}] Transitioned from {} to {} on {} event.",
- componentSpec.getName(), oldState, getState(), event.getType());
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- private static class BaseTransition implements
- SingleArcTransition<Component, ComponentEvent> {
-
- @Override public void transition(Component component,
- ComponentEvent event) {
- }
- }
-
- public ServiceContext getContext() {
- return context;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
deleted file mode 100644
index ed892dd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.component;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-
-public class ComponentEvent extends AbstractEvent<ComponentEventType> {
- private long desired;
- private final String name;
- private final ComponentEventType type;
- private Container container;
- private ComponentInstance instance;
- private ContainerStatus status;
-
- public ComponentEvent(String name, ComponentEventType type) {
- super(type);
- this.name = name;
- this.type = type;
- }
-
- public String getName() {
- return name;
- }
-
- public ComponentEventType getType() {
- return type;
- }
-
- public long getDesired() {
- return desired;
- }
-
- public ComponentEvent setDesired(long desired) {
- this.desired = desired;
- return this;
- }
-
- public Container getContainer() {
- return container;
- }
-
- public ComponentEvent setContainer(Container container) {
- this.container = container;
- return this;
- }
-
- public ComponentInstance getInstance() {
- return instance;
- }
-
- public ComponentEvent setInstance(ComponentInstance instance) {
- this.instance = instance;
- return this;
- }
-
- public ContainerStatus getStatus() {
- return status;
- }
-
- public ComponentEvent setStatus(ContainerStatus status) {
- this.status = status;
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
deleted file mode 100644
index 6729699..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.component;
-
-public enum ComponentEventType {
- FLEX,
- CONTAINER_ALLOCATED,
- CONTAINER_STARTED,
- CONTAINER_COMPLETED
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
deleted file mode 100644
index a5f9ff4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.component;
-
-public enum ComponentState {
- INIT,
- FLEXING,
- STABLE
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
deleted file mode 100644
index cbbb206..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-public interface RestApiConstants {
-
- // Rest endpoints
- String CONTEXT_ROOT = "/services/v1";
- String VERSION = "/version";
- String APP_ROOT_PATH = "/applications";
- String APP_PATH = "/applications/{app_name}";
- String COMPONENT_PATH = "/applications/{app_name}/components/{component_name}";
-
- // Query param
- String APP_NAME = "app_name";
- String COMPONENT_NAME = "component_name";
-
- String DEFAULT_COMPONENT_NAME = "default";
-
- String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST";
- String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT";
- Long DEFAULT_UNLIMITED_LIFETIME = -1l;
-
- Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001;
- Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002;
- Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003;
- Integer ERROR_CODE_APP_NAME_INVALID = 404004;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
deleted file mode 100644
index bdef600..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
-
-public interface SliderExitCodes extends LauncherExitCodes {
-
- /**
- * starting point for exit codes; not an exception itself
- */
- int _EXIT_CODE_BASE = 64;
-
- /**
- * service entered the failed state: {@value}
- */
- int EXIT_YARN_SERVICE_FAILED = 65;
-
- /**
- * service was killed: {@value}
- */
- int EXIT_YARN_SERVICE_KILLED = 66;
-
- /**
- * timeout on monitoring client: {@value}
- */
- int EXIT_TIMED_OUT = 67;
-
- /**
- * service finished with an error: {@value}
- */
- int EXIT_YARN_SERVICE_FINISHED_WITH_ERROR = 68;
-
- /**
- * the application instance is unknown: {@value}
- */
- int EXIT_UNKNOWN_INSTANCE = 69;
-
- /**
- * the application instance is in the wrong state for that operation: {@value}
- */
- int EXIT_BAD_STATE = 70;
-
- /**
- * A spawned master process failed
- */
- int EXIT_PROCESS_FAILED = 71;
-
- /**
- * The instance failed -too many containers were
- * failing or some other threshold was reached
- */
- int EXIT_DEPLOYMENT_FAILED = 72;
-
- /**
- * The application is live -and the requested operation
- * does not work if the cluster is running
- */
- int EXIT_APPLICATION_IN_USE = 73;
-
- /**
- * There already is an application instance of that name
- * when an attempt is made to create a new instance
- */
- int EXIT_INSTANCE_EXISTS = 75;
-
- /**
- * Exit code when the configurations in valid/incomplete: {@value}
- */
- int EXIT_BAD_CONFIGURATION = 77;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
deleted file mode 100644
index 33fc671..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.yarn.service.api.records.Configuration;
-
-public class YarnServiceConf {
-
- // Retry settings for the ServiceClient to talk to Service AppMaster
- public static final String CLIENT_AM_RETRY_MAX_WAIT_MS = "yarn.service.client-am.retry.max-wait-ms";
- public static final String CLIENT_AM_RETRY_MAX_INTERVAL_MS = "yarn.service.client-am.retry-interval-ms";
-
- // Retry settings for container failures
- public static final String CONTAINER_RETRY_MAX = "yarn.service.container-failure.retry.max";
- public static final String CONTAINER_RETRY_INTERVAL = "yarn.service.container-failure.retry-interval";
-
- public static final String AM_RESTART_MAX = "yarn.service.am-restart.max-attempts";
- public static final String AM_RESOURCE_MEM = "yarn.service.am-resource.memory";
- public static final long DEFAULT_KEY_AM_RESOURCE_MEM = 1024;
-
- public static final String YARN_QUEUE = "yarn.service.queue";
-
- /**
- * The yarn service base path:
- * Defaults to HomeDir/.yarn/
- */
- public static final String YARN_SERVICE_BASE_PATH = "yarn.service.base.path";
-
- //TODO rename
- /** Declare that a keytab must be provided */
- public static final String KEY_AM_LOGIN_KEYTAB_REQUIRED = "slider.am.login.keytab.required";
- public static final String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
- public static final String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir";
- public static final String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
-
- /**
- * maximum number of failed containers (in a single component)
- * before the app exits
- */
- public static final String CONTAINER_FAILURE_THRESHOLD =
- "yarn.service.container-failure-per-component.threshold";
- /**
- * Maximum number of container failures on a node before the node is blacklisted
- */
- public static final String NODE_BLACKLIST_THRESHOLD =
- "yarn.service.node-blacklist.threshold";
-
- /**
- * The failure count for CONTAINER_FAILURE_THRESHOLD and NODE_BLACKLIST_THRESHOLD
- * gets reset periodically, the unit is seconds.
- */
- public static final String CONTAINER_FAILURE_WINDOW =
- "yarn.service.failure-count-reset.window";
-
- /**
- * interval between readiness checks.
- */
- public static final String READINESS_CHECK_INTERVAL = "yarn.service.readiness-check-interval.seconds";
- public static final int DEFAULT_READINESS_CHECK_INTERVAL = 30; // seconds
-
- /**
- * Get long value for the property. First get from the userConf, if not
- * present, get from systemConf.
- *
- * @param name name of the property
- * @param defaultValue default value of the property, if it is not defined in
- * userConf and systemConf.
- * @param userConf Configuration provided by client in the JSON definition
- * @param systemConf The YarnConfiguration in the system.
- * @return long value for the property
- */
- public static long getLong(String name, long defaultValue,
- Configuration userConf, org.apache.hadoop.conf.Configuration systemConf) {
- return userConf.getPropertyLong(name, systemConf.getLong(name, defaultValue));
- }
-
- public static int getInt(String name, int defaultValue,
- Configuration userConf, org.apache.hadoop.conf.Configuration systemConf) {
- return userConf.getPropertyInt(name, systemConf.getInt(name, defaultValue));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
deleted file mode 100644
index cbcba82..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-public interface YarnServiceConstants {
-
- /**
- * The path under which cluster and temp data are stored
- */
- String SERVICE_BASE_DIRECTORY = ".yarn";
-
- /**
- * The paths under which Service AM dependency libraries are stored
- */
- String DEPENDENCY_LOCALIZED_DIR_LINK = "service_dep";
- String DEPENDENCY_DIR = "/yarn-services/%s/";
- String DEPENDENCY_TAR_GZ_FILE_NAME = "service-dep";
- String DEPENDENCY_TAR_GZ_FILE_EXT = ".tar.gz";
- String DEPENDENCY_DIR_PERMISSIONS = "755";
-
- /**
- * Application type for YARN service
- */
- String APP_TYPE = "yarn-service";
-
- String KEYTAB_DIR = "keytabs";
- String RESOURCE_DIR = "resources";
-
-
- String SERVICES_DIRECTORY = "services";
-
- /**
- * JVM property to define the service lib directory;
- * this is set by the yarn.sh script
- */
- String PROPERTY_LIB_DIR = "service.libdir";
-
- /**
- * name of generated dir for this conf
- */
- String SUBMITTED_CONF_DIR = "conf";
-
- /**
- * Service AM log4j file name
- */
- String YARN_SERVICE_LOG4J_FILENAME = "yarnservice-log4j.properties";
-
- /**
- * Log4j sysprop to name the resource
- */
- String SYSPROP_LOG4J_CONFIGURATION = "log4j.configuration";
-
- /**
- * sysprop for Service AM log4j directory
- */
- String SYSPROP_LOG_DIR = "LOG_DIR";
-
- String TMP_DIR_PREFIX = "tmp";
-
-
- String SERVICE_CORE_JAR = "yarn-service-core.jar";
-
- String STDOUT_AM = "serviceam-out.txt";
- String STDERR_AM = "serviceam-err.txt";
-
- String HADOOP_USER_NAME = "HADOOP_USER_NAME";
-
- String APP_CONF_DIR = "conf";
-
- String APP_LIB_DIR = "lib";
-
- String OUT_FILE = "stdout.txt";
- String ERR_FILE = "stderr.txt";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
deleted file mode 100644
index e4eae20..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.containerlaunch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
-import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.utils.CoreFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import static org.apache.hadoop.yarn.service.provider.docker.DockerKeys.DEFAULT_DOCKER_NETWORK;
-
-/**
- * Launcher of applications: base class
- */
-public class AbstractLauncher {
- private static final Logger log =
- LoggerFactory.getLogger(AbstractLauncher.class);
- public static final String CLASSPATH = "CLASSPATH";
- /**
- * Filesystem to use for the launch
- */
- protected final CoreFileSystem coreFileSystem;
- /**
- * Env vars; set up at final launch stage
- */
- protected final Map<String, String> envVars = new HashMap<>();
- protected final ContainerLaunchContext containerLaunchContext =
- Records.newRecord(ContainerLaunchContext.class);
- protected final List<String> commands = new ArrayList<>(20);
- protected final Map<String, LocalResource> localResources = new HashMap<>();
- protected final Map<String, String> mountPaths = new HashMap<>();
- private final Map<String, ByteBuffer> serviceData = new HashMap<>();
- // security
- protected final Credentials credentials;
- protected boolean yarnDockerMode = false;
- protected String dockerImage;
- protected String dockerNetwork = DEFAULT_DOCKER_NETWORK;
- protected String dockerHostname;
- protected String runPrivilegedContainer;
-
-
- /**
- * Create instance.
- * @param coreFileSystem filesystem
- * @param credentials initial set of credentials -null is permitted
- */
- public AbstractLauncher(
- CoreFileSystem coreFileSystem,
- Credentials credentials) {
- this.coreFileSystem = coreFileSystem;
- this.credentials = credentials != null ? credentials: new Credentials();
- }
-
- public void setYarnDockerMode(boolean yarnDockerMode){
- this.yarnDockerMode = yarnDockerMode;
- }
-
- /**
- * Get the env vars to work on
- * @return env vars
- */
- public Map<String, String> getEnv() {
- return envVars;
- }
-
- /**
- * Get the launch commands.
- * @return the live list of commands
- */
- public List<String> getCommands() {
- return commands;
- }
-
- public void addLocalResource(String subPath, LocalResource resource) {
- localResources.put(subPath, resource);
- }
-
- public void addLocalResource(String subPath, LocalResource resource, String mountPath) {
- localResources.put(subPath, resource);
- mountPaths.put(subPath, mountPath);
- }
-
- /**
- * Accessor to the credentials
- * @return the credentials associated with this launcher
- */
- public Credentials getCredentials() {
- return credentials;
- }
-
-
- public void addCommand(String cmd) {
- commands.add(cmd);
- }
-
- /**
- * Complete the launch context (copy in env vars, etc).
- * @return the container to launch
- */
- public ContainerLaunchContext completeContainerLaunch() throws IOException {
-
- String cmdStr = SliderUtils.join(commands, " ", false);
- log.debug("Completed setting up container command {}", cmdStr);
- containerLaunchContext.setCommands(commands);
-
- //env variables
- if (log.isDebugEnabled()) {
- log.debug("Environment variables");
- for (Map.Entry<String, String> envPair : envVars.entrySet()) {
- log.debug(" \"{}\"=\"{}\"", envPair.getKey(), envPair.getValue());
- }
- }
- containerLaunchContext.setEnvironment(envVars);
-
- //service data
- if (log.isDebugEnabled()) {
- log.debug("Service Data size");
- for (Map.Entry<String, ByteBuffer> entry : serviceData.entrySet()) {
- log.debug("\"{}\"=> {} bytes of data", entry.getKey(),
- entry.getValue().array().length);
- }
- }
- containerLaunchContext.setServiceData(serviceData);
-
- // resources
- dumpLocalResources();
- containerLaunchContext.setLocalResources(localResources);
-
- //tokens
- log.debug("{} tokens", credentials.numberOfTokens());
- containerLaunchContext.setTokens(CredentialUtils.marshallCredentials(
- credentials));
-
- if(yarnDockerMode){
- Map<String, String> env = containerLaunchContext.getEnvironment();
- env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker");
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage);
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", dockerNetwork);
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME",
- dockerHostname);
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer);
- StringBuilder sb = new StringBuilder();
- for (Entry<String,String> mount : mountPaths.entrySet()) {
- if (sb.length() > 0) {
- sb.append(",");
- }
- sb.append(mount.getKey());
- sb.append(":");
- sb.append(mount.getValue());
- }
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS", sb.toString());
- log.info("yarn docker env var has been set {}", containerLaunchContext.getEnvironment().toString());
- }
-
- return containerLaunchContext;
- }
-
- public void setRetryContext(int maxRetries, int retryInterval) {
- ContainerRetryContext retryContext = ContainerRetryContext
- .newInstance(ContainerRetryPolicy.RETRY_ON_ALL_ERRORS, null, maxRetries,
- retryInterval);
- containerLaunchContext.setContainerRetryContext(retryContext);
- }
-
- /**
- * Dump local resources at debug level
- */
- private void dumpLocalResources() {
- if (log.isDebugEnabled()) {
- log.debug("{} resources: ", localResources.size());
- for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) {
-
- String key = entry.getKey();
- LocalResource val = entry.getValue();
- log.debug(key + "=" + SliderUtils.stringify(val.getResource()));
- }
- }
- }
-
- /**
- * This is critical for an insecure cluster -it passes
- * down the username to YARN, and so gives the code running
- * in containers the rights it needs to work with
- * data.
- * @throws IOException problems working with current user
- */
- protected void propagateUsernameInInsecureCluster() throws IOException {
- //insecure cluster: propagate user name via env variable
- String userName = UserGroupInformation.getCurrentUser().getUserName();
- envVars.put(YarnServiceConstants.HADOOP_USER_NAME, userName);
- }
-
- /**
- * Utility method to set up the classpath
- * @param classpath classpath to use
- */
- public void setClasspath(ClasspathConstructor classpath) {
- setEnv(CLASSPATH, classpath.buildClasspath());
- }
-
- /**
- * Set an environment variable in the launch context
- * @param var variable name
- * @param value value (must be non null)
- */
- public void setEnv(String var, String value) {
- Preconditions.checkArgument(var != null, "null variable name");
- Preconditions.checkArgument(value != null, "null value");
- envVars.put(var, value);
- }
-
-
- public void putEnv(Map<String, String> map) {
- envVars.putAll(map);
- }
-
-
- public void setDockerImage(String dockerImage) {
- this.dockerImage = dockerImage;
- }
-
- public void setDockerNetwork(String dockerNetwork) {
- this.dockerNetwork = dockerNetwork;
- }
-
- public void setDockerHostname(String dockerHostname) {
- this.dockerHostname = dockerHostname;
- }
-
- public void setRunPrivilegedContainer(boolean runPrivilegedContainer) {
- if (runPrivilegedContainer) {
- this.runPrivilegedContainer = Boolean.toString(true);
- } else {
- this.runPrivilegedContainer = Boolean.toString(false);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
deleted file mode 100644
index 22b3877..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.containerlaunch;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * build a classpath -allows for entries to be injected in front of
- * YARN classpath as well as behind, adds appropriate separators,
- * extraction of local classpath, etc.
- */
-public class ClasspathConstructor {
-
- public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR;
- private final List<String> pathElements = new ArrayList<>();
-
- public ClasspathConstructor() {
- }
-
-
- /**
- * Get the list of JARs from the YARN settings
- * @param config configuration
- */
- public List<String> yarnApplicationClasspath(Configuration config) {
- String[] cp = config.getTrimmedStrings(
- YarnConfiguration.YARN_APPLICATION_CLASSPATH,
- YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH);
- return cp != null ? Arrays.asList(cp) : new ArrayList<String>(0);
-
- }
-
-
- @Override
- public String toString() {
- return buildClasspath();
- }
-
- public String buildClasspath() {
- return SliderUtils.join(pathElements,
- CLASS_PATH_SEPARATOR,
- false);
- }
-
- /**
- * Get a copy of the path list
- * @return the JARs
- */
- public List<String> getPathElements() {
- return Collections.unmodifiableList(pathElements);
- }
-
- /**
- * Append an entry
- * @param path path
- */
- public void append(String path) {
- pathElements.add(path);
- }
-
- /**
- * Insert a path at the front of the list. This places it ahead of
- * the standard YARN artifacts
- * @param path path to the JAR. Absolute or relative -on the target
- * system
- */
- public void insert(String path) {
- pathElements.add(0, path);
- }
-
- public void appendAll(Collection<String> paths) {
- pathElements.addAll(paths);
- }
-
- public void insertAll(Collection<String> paths) {
- pathElements.addAll(0, paths);
- }
-
-
- public void addLibDir(String pathToLibDir) {
- append(buildLibDir(pathToLibDir));
- }
-
- public void insertLibDir(String pathToLibDir) {
- insert(buildLibDir(pathToLibDir));
- }
-
- public void addClassDirectory(String pathToDir) {
- append(appendDirectoryTerminator(pathToDir));
- }
-
- public void insertClassDirectory(String pathToDir) {
- insert(buildLibDir(appendDirectoryTerminator(pathToDir)));
- }
-
-
- public void addRemoteClasspathEnvVar() {
- append(ApplicationConstants.Environment.CLASSPATH.$$());
- }
-
-
- public void insertRemoteClasspathEnvVar() {
- append(ApplicationConstants.Environment.CLASSPATH.$$());
- }
-
-
- /**
- * Build a lib dir path
- * @param pathToLibDir path to the directory; may or may not end with a
- * trailing space
- * @return a path to a lib dir that is compatible with the java classpath
- */
- public String buildLibDir(String pathToLibDir) {
- String dir = appendDirectoryTerminator(pathToLibDir);
- dir += "*";
- return dir;
- }
-
- private String appendDirectoryTerminator(String pathToLibDir) {
- String dir = pathToLibDir.trim();
- if (!dir.endsWith("/")) {
- dir += "/";
- }
- return dir;
- }
-
- /**
- * Split a classpath. This uses the local path separator so MUST NOT
- * be used to work with remote classpaths
- * @param localpath local path
- * @return a splite
- */
- public Collection<String> splitClasspath(String localpath) {
- String separator = System.getProperty("path.separator");
- return StringUtils.getStringCollection(localpath, separator);
- }
-
- /**
- * Get the local JVM classpath split up
- * @return the list of entries on the JVM classpath env var
- */
- public Collection<String> localJVMClasspath() {
- return splitClasspath(System.getProperty("java.class.path"));
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[83/86] [abbrv] hadoop git commit: YARN-7165. Miscellaneous fixes in
yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
YARN-7165. Miscellaneous fixes in yarn-native-services. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/803eb069
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/803eb069
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/803eb069
Branch: refs/heads/yarn-native-services
Commit: 803eb069d3bea41bac62915c4ad1b58f1221a663
Parents: e8af82c
Author: Gour Saha <go...@apache.org>
Authored: Thu Sep 7 23:25:31 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 25 16:37:25 2017 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 4 +-
hadoop-project/src/site/site.xml | 10 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 +-
...RN-Simplified-V1-API-Layer-For-Services.yaml | 11 +-
.../hadoop/yarn/service/ServiceMaster.java | 10 +-
.../hadoop/yarn/service/client/ServiceCLI.java | 5 +-
.../yarn/service/client/ServiceClient.java | 20 +-
.../client/params/AbstractActionArgs.java | 33 +-
.../AbstractClusterBuildingActionArgs.java | 22 +-
.../service/client/params/ActionBuildArgs.java | 5 +
.../service/client/params/ActionCreateArgs.java | 5 +
.../client/params/ActionDependencyArgs.java | 4 +-
.../service/client/params/ActionExamples.java | 26 +
.../service/client/params/ActionUpdateArgs.java | 7 +-
.../yarn/service/client/params/Arguments.java | 9 +-
.../yarn/service/client/params/ClientArgs.java | 10 -
.../yarn/service/client/params/CommonArgs.java | 28 +-
.../service/client/params/ServiceAMArgs.java | 50 ++
.../client/params/ServiceAMCreateAction.java | 37 ++
.../service/client/params/SliderAMArgs.java | 57 --
.../client/params/SliderAMCreateAction.java | 73 ---
.../service/client/params/SliderActions.java | 9 +-
.../ServiceTimelinePublisher.java | 5 +-
.../client/TestBuildExternalComponents.java | 6 +-
.../yarn/service/client/TestServiceCLI.java | 30 +-
.../hadoop/yarn/service/conf/examples/app.json | 2 +-
.../registry/client/api/RegistryConstants.java | 2 +-
.../src/site/markdown/YarnCommands.md | 56 +-
.../native-services/NativeServicesAPI.md | 606 -------------------
.../native-services/NativeServicesDiscovery.md | 144 -----
.../native-services/NativeServicesIntro.md | 107 ----
.../src/site/markdown/yarn-service/Concepts.md | 77 +++
.../src/site/markdown/yarn-service/Overview.md | 58 ++
.../site/markdown/yarn-service/QuickStart.md | 218 +++++++
.../markdown/yarn-service/ServiceDiscovery.md | 150 +++++
.../markdown/yarn-service/YarnServiceAPI.md | 592 ++++++++++++++++++
36 files changed, 1362 insertions(+), 1130 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 1697744..93475a5 100755
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -141,10 +141,10 @@
<!-- the version of Hadoop declared in the version resources; can be overridden
so that Hadoop 3.x can declare itself a 2.x artifact. -->
<declared.hadoop.version>${project.version}</declared.hadoop.version>
-
+
<swagger-annotations-version>1.5.4</swagger-annotations-version>
<snakeyaml.version>1.16</snakeyaml.version>
- <jcommander.version>1.30</jcommander.version>
+ <jcommander.version>1.48</jcommander.version>
</properties>
<dependencyManagement>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 755bc61..4b4b13f 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -152,10 +152,12 @@
<item name="Timeline Service V.2" href="hadoop-yarn/hadoop-yarn-site/TimelineServiceV2.html#Timeline_Service_v.2_REST_API"/>
</menu>
- <menu name="YARN Native Services" inherit="top">
- <item name="Introduction" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesIntro.html"/>
- <item name="Native Services API" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesAPI.html"/>
- <item name="Native Services Discovery" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesDiscovery.html"/>
+ <menu name="YARN Service" inherit="top">
+ <item name="Overview" href="hadoop-yarn/hadoop-yarn-site/yarn-service/Overview.html"/>
+ <item name="QuickStart" href="hadoop-yarn/hadoop-yarn-site/yarn-service/QuickStart.html"/>
+ <item name="Concepts" href="hadoop-yarn/hadoop-yarn-site/yarn-service/Concepts.html"/>
+ <item name="Yarn Service API" href="hadoop-yarn/hadoop-yarn-site/yarn-service/YarnServiceAPI.html"/>
+ <item name="Service Discovery" href="hadoop-yarn/hadoop-yarn-site/yarn-service/ServiceDiscovery.html"/>
</menu>
<menu name="Hadoop Compatible File Systems" inherit="top">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 43bcb6b..022287a 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -31,7 +31,7 @@ function hadoop_usage
hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
hadoop_add_option "--workers" "turn on worker mode"
- hadoop_add_subcommand "apiserver" "run yarn-native-service rest server"
+ hadoop_add_subcommand "apiserver" daemon "run the api-server for deploying/managing services on YARN"
hadoop_add_subcommand "application" client "prints application(s) report/kill application"
hadoop_add_subcommand "applicationattempt" client "prints applicationattempt(s) report"
hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries"
@@ -49,7 +49,7 @@ function hadoop_usage
hadoop_add_subcommand "rmadmin" admin "admin tools"
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
- hadoop_add_subcommand "service" "run a service"
+ hadoop_add_subcommand "service" client "deploy/manage a service on YARN"
hadoop_add_subcommand "sharedcachemanager" admin "run the SharedCacheManager daemon"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
hadoop_add_subcommand "timelineserver" daemon "run the timeline server"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index b084be7..b9b5b3a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -48,17 +48,16 @@ info:
url: http://www.apache.org/licenses/LICENSE-2.0.html
# the domain of the service
host: host.mycompany.com
+port: 9191(default)
# array of all schemes that your API supports
schemes:
- http
-# will be prefixed to all paths
-basePath: /ws/v1/
consumes:
- application/json
produces:
- application/json
paths:
- /services/version:
+ /ws/v1/services/version:
get:
summary: Get current version of the API server.
description: Get current version of the API server.
@@ -66,7 +65,7 @@ paths:
200:
description: Successful request
- /services:
+ /ws/v1/services:
get:
summary: (TBD) List of services running in the cluster.
description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name.
@@ -103,7 +102,7 @@ paths:
schema:
$ref: '#/definitions/ServiceStatus'
- /services/{service_name}:
+ /ws/v1/services/{service_name}:
put:
summary: Update a service or upgrade the binary version of the components of a running service
description: Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service.
@@ -174,7 +173,7 @@ paths:
description: Unexpected error
schema:
$ref: '#/definitions/ServiceStatus'
- /services/{service_name}/components/{component_name}:
+ /ws/v1/services/{service_name}/components/{component_name}:
put:
summary: Flex a component's number of instances.
description: Set a component's desired number of instanes
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
index e700961..2abdae1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
-import org.apache.hadoop.yarn.service.client.params.SliderAMArgs;
+import org.apache.hadoop.yarn.service.client.params.ServiceAMArgs;
import org.apache.hadoop.yarn.service.monitor.ServiceMonitor;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
@@ -51,7 +51,7 @@ public class ServiceMaster extends CompositeService {
private static final Logger LOG =
LoggerFactory.getLogger(ServiceMaster.class);
- private static SliderAMArgs amArgs;
+ private static ServiceAMArgs amArgs;
protected ServiceContext context;
public ServiceMaster(String name) {
@@ -108,7 +108,7 @@ public class ServiceMaster extends CompositeService {
}
protected Path getAppDir() {
- return new Path(amArgs.getAppDefPath()).getParent();
+ return new Path(amArgs.getServiceDefPath()).getParent();
}
protected ServiceScheduler createServiceScheduler(ServiceContext context)
@@ -119,7 +119,7 @@ public class ServiceMaster extends CompositeService {
protected void loadApplicationJson(ServiceContext context,
SliderFileSystem fs) throws IOException {
context.service = ServiceApiUtil
- .loadServiceFrom(fs, new Path(amArgs.getAppDefPath()));
+ .loadServiceFrom(fs, new Path(amArgs.getServiceDefPath()));
LOG.info(context.service.toString());
}
@@ -138,7 +138,7 @@ public class ServiceMaster extends CompositeService {
public static void main(String[] args) throws Exception {
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
StringUtils.startupShutdownMessage(ServiceMaster.class, args, LOG);
- amArgs = new SliderAMArgs(args);
+ amArgs = new ServiceAMArgs(args);
amArgs.parse();
try {
ServiceMaster serviceMaster = new ServiceMaster("Service Master");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
index cb27021..928c06f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
@@ -18,11 +18,12 @@
package org.apache.hadoop.yarn.service.client;
+import com.beust.jcommander.ParameterException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -101,7 +102,7 @@ public class ServiceCLI {
ClientArgs clientArgs = new ClientArgs(args);
try {
clientArgs.parse();
- } catch (BadCommandArgumentsException e) {
+ } catch (ParameterException | SliderException e) {
System.err.println(e.getMessage());
System.exit(-1);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index c3a2752..bfddc44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -158,7 +158,7 @@ public class ServiceClient extends CompositeService
private Service loadAppJsonFromLocalFS(
AbstractClusterBuildingActionArgs args) throws IOException {
- File file = args.getAppDef();
+ File file = args.getFile();
Path filePath = new Path(file.getAbsolutePath());
LOG.info("Loading app json from: " + filePath);
Service service = jsonSerDeser
@@ -166,7 +166,6 @@ public class ServiceClient extends CompositeService
if (args.lifetime > 0) {
service.setLifetime(args.lifetime);
}
- service.setName(args.getClusterName());
return service;
}
@@ -407,7 +406,8 @@ public class ServiceClient extends CompositeService
public int actionDestroy(String serviceName) throws Exception {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
- verifyNoLiveAppInRM(serviceName, "Destroy");
+ verifyNoLiveAppInRM(serviceName, "destroy");
+
Path appDir = fs.buildClusterDirPath(serviceName);
FileSystem fileSystem = fs.getFileSystem();
// remove from the appId cache
@@ -498,9 +498,15 @@ public class ServiceClient extends CompositeService
request.setApplicationStates(liveStates);
List<ApplicationReport> reports = yarnClient.getApplications(request);
if (!reports.isEmpty()) {
- throw new YarnException(
- "Failed to " + action + " service, as " + serviceName
- + " already exists.");
+ String message = "";
+ if (action.equals("destroy")) {
+ message = "Failed to destroy service " + serviceName
+ + ", because it is still running.";
+ } else {
+ message = "Failed to " + action + " service " + serviceName
+ + ", because it already exists.";
+ }
+ throw new YarnException(message);
}
}
@@ -592,7 +598,7 @@ public class ServiceClient extends CompositeService
CLI.add(ServiceMaster.class.getCanonicalName());
CLI.add(ACTION_CREATE, serviceName);
//TODO debugAM CLI.add(Arguments.ARG_DEBUG)
- CLI.add(Arguments.ARG_CLUSTER_URI, new Path(appRootDir, serviceName + ".json"));
+ CLI.add(Arguments.ARG_SERVICE_DEF_PATH, new Path(appRootDir, serviceName + ".json"));
// pass the registry binding
CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
index ea3bb0a..8b2d93e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
@@ -41,22 +41,10 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
}
/**
- * URI/binding to the filesystem
- */
- @Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG},
- description = "Filesystem Binding")
- public String filesystemBinding;
-
- @Parameter(names = {ARG_BASE_PATH},
- description = "Service base path on the filesystem",
- converter = PathArgumentConverter.class)
- public Path basePath;
-
- /**
* This is the default parameter
*/
@Parameter
- public final List<String> parameters = new ArrayList<>();
+ public List<String> parameters = new ArrayList<>();
/**
* get the name: relies on arg 1 being the cluster name in all operations
@@ -76,29 +64,22 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
*/
- @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions")
- public final List<String> definitions = new ArrayList<>();
+ @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions", hidden = true)
+ public List<String> definitions = new ArrayList<>();
/**
* System properties
*/
@Parameter(names = {ARG_SYSPROP}, arity = 1,
description = "system properties in the form name value" +
- " These are set after the JVM is started.")
- public final List<String> sysprops = new ArrayList<>(0);
+ " These are set after the JVM is started.",
+ hidden = true)
+ public List<String> sysprops = new ArrayList<>(0);
- @Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER},
- description = "Binding (usually hostname:port) of the YARN resource manager")
- public String manager;
-
-
- @Parameter(names = ARG_DEBUG, description = "Debug mode")
+ @Parameter(names = ARG_DEBUG, description = "Debug mode", hidden = true)
public boolean debug = false;
- @Parameter(names = {ARG_HELP}, description = "Help", help = true)
- public boolean help = false;
-
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
index 3a3a19a..d215436 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
@@ -19,13 +19,8 @@
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParametersDelegate;
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import java.io.File;
-import java.util.List;
-import java.util.Map;
/**
* Abstract Action to build things; shares args across build and
@@ -33,12 +28,12 @@ import java.util.Map;
*/
public abstract class AbstractClusterBuildingActionArgs
extends AbstractActionArgs {
- @Parameter(names = {ARG_APPDEF},
- description = "Template service definition file in JSON format.")
- public File appDef;
+ @Parameter(names = { ARG_FILE, ARG_FILE_SHORT }, required = true,
+ description = "The path to the service definition file in JSON format.")
+ public File file;
- public File getAppDef() {
- return appDef;
+ public File getFile() {
+ return file;
}
@Parameter(names = {
@@ -48,11 +43,4 @@ public abstract class AbstractClusterBuildingActionArgs
@Parameter(names = {
ARG_LIFETIME }, description = "Lifetime of the service from the time of request")
public long lifetime;
-
- @ParametersDelegate
- public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
-
- @ParametersDelegate
- public OptionArgsDelegate optionsDelegate =
- new OptionArgsDelegate();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
index c2ff545..cf61525 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
@@ -28,4 +28,9 @@ public class ActionBuildArgs extends AbstractClusterBuildingActionArgs {
public String getActionName() {
return SliderActions.ACTION_BUILD;
}
+
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
index eecffb6..c43f4fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
@@ -29,5 +29,10 @@ public class ActionCreateArgs extends AbstractClusterBuildingActionArgs {
public String getActionName() {
return SliderActions.ACTION_CREATE;
}
+
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
index 51e07c9..699e6ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
@@ -32,8 +32,8 @@ public class ActionDependencyArgs extends AbstractActionArgs {
return SliderActions.ACTION_DEPENDENCY;
}
- @Parameter(names = { ARG_UPLOAD },
- description = "Upload AM and agent libraries to HDFS for this client")
+ @Parameter(names = { ARG_UPLOAD }, required = true,
+ description = "Upload AM libraries to HDFS for this client version")
public boolean upload;
@Parameter(names = { ARG_OVERWRITE },
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java
new file mode 100644
index 0000000..e489e17
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_EXAMPLES},
+ commandDescription = SliderActions.DESCRIBE_ACTION_EXAMPLES)
+public class ActionExamples {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
index e310f45..00af69a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
@@ -18,12 +18,17 @@
package org.apache.hadoop.yarn.service.client.params;
+import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = { SliderActions.ACTION_UPDATE},
commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE)
-public class ActionUpdateArgs extends AbstractClusterBuildingActionArgs {
+public class ActionUpdateArgs extends AbstractActionArgs {
+
+ @Parameter(names = {
+ ARG_LIFETIME }, description = "Lifetime of the service from the time of request")
+ public long lifetime;
@Override
public String getActionName() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
index 204149b..911440d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
@@ -27,7 +27,8 @@ package org.apache.hadoop.yarn.service.client.params;
*/
public interface Arguments {
- String ARG_APPDEF = "--appdef";
+ String ARG_FILE = "--file";
+ String ARG_FILE_SHORT = "-f";
String ARG_BASE_PATH = "--basepath";
String ARG_COMPONENT = "--component";
String ARG_COMPONENT_SHORT = "--comp";
@@ -95,9 +96,7 @@ public interface Arguments {
RIGHT PLACE IN THE LIST
*/
- /**
- * server: URI for the cluster
- */
- String ARG_CLUSTER_URI = "-cluster-uri";
+ // Tha path in hdfs to be read by Service AM
+ String ARG_SERVICE_DEF_PATH = "-cluster-uri";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
index 09cae24..f479cd2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
@@ -85,16 +85,6 @@ public class ClientArgs extends CommonArgs {
public void applyDefinitions(Configuration conf) throws
BadCommandArgumentsException {
super.applyDefinitions(conf);
- //RM
- if (getManager() != null) {
- log.debug("Setting RM to {}", getManager());
- conf.set(YarnConfiguration.RM_ADDRESS, getManager());
- }
- if (getBasePath() != null) {
- log.debug("Setting basePath to {}", getBasePath());
- conf.set(YarnServiceConf.YARN_SERVICE_BASE_PATH,
- getBasePath().toString());
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
index e1197ea..a5638a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
@@ -156,12 +156,8 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
try {
commander.parse(args);
} catch (ParameterException e) {
- throw new BadCommandArgumentsException(e, "%s in %s",
- e.toString(),
- (args != null
- ? (SliderUtils.join(args,
- " ", false))
- : "[]"));
+ commander.usage(args[0]);
+ throw e;
}
//now copy back to this class some of the attributes that are common to all
//actions
@@ -251,31 +247,11 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
applyDefinitions(definitionMap, conf);
}
-
- /**
- * If the Filesystem binding was provided, it overrides anything in
- * the configuration
- * @param conf configuration
- */
- public void applyFileSystemBinding(Configuration conf) {
- ArgOps.applyFileSystemBinding(getFilesystemBinding(), conf);
- }
-
public boolean isDebug() {
return coreAction.debug;
}
- public String getFilesystemBinding() {
- return coreAction.filesystemBinding;
- }
-
- public Path getBasePath() { return coreAction.basePath; }
-
- public String getManager() {
- return coreAction.manager;
- }
-
public String getAction() {
return commander.getParsedCommand();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java
new file mode 100644
index 0000000..800bb24
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.yarn.service.client.params;
+
+/**
+ * Parameters sent by the Client to the AM
+ */
+public class ServiceAMArgs extends CommonArgs {
+
+ ServiceAMCreateAction createAction = new ServiceAMCreateAction();
+
+ public ServiceAMArgs(String[] args) {
+ super(args);
+ }
+
+ @Override
+ protected void addActionArguments() {
+ addActions(createAction);
+ }
+
+ // This is the path in hdfs to the service definition JSON file
+ public String getServiceDefPath() {
+ return createAction.serviceDefPath;
+ }
+
+ /**
+ * Am binding is simple: there is only one action
+ */
+ @Override
+ public void applyAction() {
+ bindCoreAction(createAction);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java
new file mode 100644
index 0000000..384e8ba
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_CREATE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
+
+public class ServiceAMCreateAction extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_CREATE;
+ }
+
+ @Parameter(names = ARG_SERVICE_DEF_PATH,
+ description = "Path to the service definition JSON file", required = true)
+ public String serviceDefPath;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
deleted file mode 100644
index 1c38213..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.hadoop.yarn.service.client.params;
-
-/**
- * Parameters sent by the Client to the AM
- */
-public class SliderAMArgs extends CommonArgs {
-
- SliderAMCreateAction createAction = new SliderAMCreateAction();
-
- public SliderAMArgs(String[] args) {
- super(args);
- }
-
- @Override
- protected void addActionArguments() {
- addActions(createAction);
- }
-
- public String getImage() {
- return createAction.image;
- }
-
- /**
- * This is the URI in the FS to the Slider cluster; the conf file (and any
- * other cluster-specifics) can be picked up here
- */
- public String getAppDefPath() {
- return createAction.sliderClusterURI;
- }
-
- /**
- * Am binding is simple: there is only one action
- */
- @Override
- public void applyAction() {
- bindCoreAction(createAction);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
deleted file mode 100644
index a446665..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-
-import java.io.File;
-
-
-@Parameters(commandNames = { SliderActions.ACTION_CREATE},
- commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
-
-public class SliderAMCreateAction extends AbstractActionArgs implements
- LaunchArgsAccessor {
-
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_CREATE;
- }
-
- @Parameter(names = ARG_IMAGE, description = "image", required = false)
- public String image;
-
- /**
- * This is the URI in the FS to the Slider cluster; the conf file (and any
- * other cluster-specifics) can be picked up here
- */
- @Parameter(names = ARG_CLUSTER_URI,
- description = "URI to the Slider cluster", required = true)
- public String sliderClusterURI;
-
- @ParametersDelegate LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
-
- @Override
- public String getRmAddress() {
- return launchArgs.getRmAddress();
- }
-
- @Override
- public int getWaittime() {
- return launchArgs.getWaittime();
- }
-
- @Override
- public void setWaittime(int waittime) {
- launchArgs.setWaittime(waittime);
- }
-
- @Override
- public File getOutputFile() {
- return launchArgs.getOutputFile();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
index 1b2a92d..fa05f2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
@@ -19,9 +19,7 @@
package org.apache.hadoop.yarn.service.client.params;
/**
- * Actions.
- * Only some of these are supported by specific Slider Services; they
- * are listed here to ensure the names are consistent
+ * Actions by client
*/
public interface SliderActions {
String ACTION_BUILD = "build";
@@ -32,11 +30,11 @@ public interface SliderActions {
String ACTION_UPGRADE = "upgrade";
String ACTION_DESTROY = "destroy";
String ACTION_EXISTS = "exists";
+ String ACTION_EXAMPLES = "examples";
String ACTION_FLEX = "flex";
String ACTION_STOP = "stop";
String ACTION_HELP = "help";
String ACTION_INSTALL_KEYTAB = "install-keytab";
- String ACTION_KDIAG = "kdiag";
String ACTION_KEYTAB = "keytab";
String ACTION_LIST = "list";
@@ -50,7 +48,7 @@ public interface SliderActions {
String DESCRIBE_ACTION_BUILD =
"Build a service specification, but do not start it";
String DESCRIBE_ACTION_CREATE =
- "Build and start a service, it's equivalent to first invoke build and then start";
+ "Create a service, it's equivalent to first invoke build and then start";
String DESCRIBE_ACTION_DEPENDENCY =
"Yarn service framework dependency (libraries) management";
String DESCRIBE_ACTION_UPDATE =
@@ -61,6 +59,7 @@ public interface SliderActions {
"Destroy a stopped service, service must be stopped first before destroying.";
String DESCRIBE_ACTION_EXISTS =
"Probe for a service running";
+ String DESCRIBE_ACTION_EXAMPLES = "Run an example service on YARN";
String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
String DESCRIBE_ACTION_FREEZE =
"Stop a running service";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
index 9e7d8e8..cfe7c1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.hadoop.yarn.service.api.records.Configuration;
import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.slf4j.Logger;
@@ -96,8 +97,8 @@ public class ServiceTimelinePublisher extends CompositeService {
// create info keys
Map<String, Object> entityInfos = new HashMap<String, Object>();
entityInfos.put(ServiceTimelineMetricsConstants.NAME, service.getName());
-// entityInfos.put(ServiceTimelineMetricsConstants.STATE,
-// service.getState().toString());
+ entityInfos.put(ServiceTimelineMetricsConstants.STATE,
+ ServiceState.STARTED.toString());
entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME,
currentTimeMillis);
entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
index 93a15cc..0498ebf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
@@ -36,7 +36,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
+import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
/**
@@ -56,12 +56,12 @@ public class TestBuildExternalComponents {
}
}
- // 1. Build the appDef and store on fs
+ // 1. Build the def file and store on fs
// 2. check component names
private void buildAndCheckComponents(String appName, String appDef,
SliderFileSystem sfs, Set<String> names) throws Throwable {
String[] args =
- { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
+ { "build", ARG_FILE, ExampleAppJson.resourceName(appDef) };
ClientArgs clientArgs = new ClientArgs(args);
clientArgs.parse();
ServiceCLI cli = new ServiceCLI() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
index c53ee2b..f22d487 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.ClientAMProtocol;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
@@ -43,7 +42,7 @@ import java.util.List;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS;
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
+import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
import static org.mockito.Mockito.*;
@@ -54,9 +53,9 @@ public class TestServiceCLI {
private ServiceCLI cli;
private SliderFileSystem fs;
- private void buildApp(String appName, String appDef) throws Throwable {
+ private void buildApp(String appDef) throws Throwable {
String[] args =
- { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
+ { "build", ARG_FILE, ExampleAppJson.resourceName(appDef)};
ClientArgs clientArgs = new ClientArgs(args);
clientArgs.parse();
cli.exec(clientArgs);
@@ -115,34 +114,35 @@ public class TestServiceCLI {
// Test flex components count are persisted.
@Test
public void testFlexComponents() throws Throwable {
- buildApp("service-1", ExampleAppJson.APP_JSON);
-
- checkCompCount("master", 1L);
+ String serviceName = "app-1";
+ buildApp(ExampleAppJson.APP_JSON);
+ checkCompCount("master",serviceName, 1L);
// increase by 2
- String[] flexUpArgs = {"flex", "service-1", "--component", "master" , "+2"};
+ String[] flexUpArgs = {"flex", serviceName, "--component", "master" , "+2"};
ClientArgs clientArgs = new ClientArgs(flexUpArgs);
clientArgs.parse();
cli.exec(clientArgs);
- checkCompCount("master", 3L);
+ checkCompCount("master", serviceName, 3L);
// decrease by 1
- String[] flexDownArgs = {"flex", "service-1", "--component", "master", "-1"};
+ String[] flexDownArgs = {"flex", serviceName, "--component", "master", "-1"};
clientArgs = new ClientArgs(flexDownArgs);
clientArgs.parse();
cli.exec(clientArgs);
- checkCompCount("master", 2L);
+ checkCompCount("master", serviceName, 2L);
- String[] flexAbsoluteArgs = {"flex", "service-1", "--component", "master", "10"};
+ String[] flexAbsoluteArgs = {"flex", serviceName, "--component", "master", "10"};
clientArgs = new ClientArgs(flexAbsoluteArgs);
clientArgs.parse();
cli.exec(clientArgs);
- checkCompCount("master", 10L);
+ checkCompCount("master", serviceName, 10L);
}
- private void checkCompCount(String compName, long count) throws IOException {
+ private void checkCompCount(String compName, String serviceName, long count)
+ throws IOException {
List<Component> components =
- ServiceApiUtil.getComponents(fs, "service-1");
+ ServiceApiUtil.getComponents(fs, serviceName);
for (Component component : components) {
if (component.getName().equals(compName)) {
Assert.assertEquals(count, component.getNumberOfContainers().longValue());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
index 12b51e4..43d3c39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
@@ -1,5 +1,5 @@
{
- "name": "service-1",
+ "name": "app-1",
"id" : "application_1503358878042_0011",
"lifetime": "3600",
"launch_command": "sleep 3600",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
index e66a761..0006dfd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
@@ -95,7 +95,7 @@ public interface RegistryConstants {
/**
* Default DNS port number.
*/
- int DEFAULT_DNS_PORT = 53;
+ int DEFAULT_DNS_PORT = 5353;
/**
* DNSSEC Enabled?
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 5f430ec..8968f13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -22,7 +22,7 @@ Overview
YARN commands are invoked by the bin/yarn script. Running the yarn script without any arguments prints the description for all commands.
-Usage: `yarn [SHELL_OPTIONS] COMMAND [GENERIC_OPTIONS] [COMMAND_OPTIONS]`
+Usage: `yarn [SHELL_OPTIONS] COMMAND [GENERIC_OPTIONS] [SUB_COMMAND] [COMMAND_OPTIONS]`
YARN has an option parsing framework that employs parsing generic options as well as running classes.
@@ -69,6 +69,56 @@ Usage: `yarn applicationattempt [options] `
prints applicationattempt(s) report
+### `service`
+Usage `yarn service [sub-command] [service-name] [options]`
+
+#### `SUB_COMMAND`
+
+* `build`: Build a service with its specifications, but do not start it.
+ ```
+ Usage: yarn service build --file [file]
+ ```
+
+ | COMMAND\_OPTIONS | Description |
+ |:---- |:---- |
+ | --file or -f | The local path to the service definition file |
+
+* `create`: create a service, it's equivalent to first invoke build and then start.
+ ```
+ Usage: yarn service create --file [file]
+ ```
+ | COMMAND\_OPTIONS | Description |
+ |:---- |:---- |
+ | --file or -f | The local path to the service definition file |
+
+* `dependency`: Yarn service framework dependency (libraries) management.
+ ```
+ Usage: yarn service dependency [options]
+ ```
+ | COMMAND\_OPTIONS | Description |
+ |:---- |:---- |
+ | --upload | Pre-upload the dependency jars onto HDFS to expediate service launch process. |
+
+* `destroy`: Destroy a stopped service, service must be stopped first before destroying.
+ ```
+ Usage: yarn service destroy [service-name]
+ ```
+* `flex`: Flex a service's component by increasing or decreasing the number of containers.
+ ```
+ Usage: yarn service flex [service-name] --component [component-name] [count]
+ ```
+ | COMMAND\_OPTIONS | Description |
+ |:---- |:---- |
+ | --component [component-name] [count] | Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.|
+* `status`: Get the status of a service.
+ ```
+ Usage: yarn service status [service-name]
+ ```
+* `start`: Start a service with pre-built specification or a previously stopped service.
+ ```
+ Usage: yarn service start [service-name]
+ ```
+
### `classpath`
Usage: `yarn classpath [--glob |--jar <path> |-h |--help]`
@@ -260,6 +310,10 @@ Usage: `yarn timelineserver`
Start the TimeLineServer
+### apiserver
+Usage: `yarn apiserver`
+Start the API-server for deploying/managing services on YARN
+
Files
-----
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
deleted file mode 100644
index d96d13e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.hadoop.yarn.service.provider.ProviderService;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-import org.apache.slider.server.appmaster.actions.QueueAccess;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.services.workflow.ServiceThreadFactory;
-import org.apache.slider.server.services.workflow.WorkflowExecutorService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import static org.apache.hadoop.yarn.service.conf.SliderKeys.KEY_CONTAINER_LAUNCH_DELAY;
-
-/**
- * A service for launching containers
- */
-public class RoleLaunchService
- extends WorkflowExecutorService<ExecutorService> {
- protected static final Logger log =
- LoggerFactory.getLogger(RoleLaunchService.class);
-
- public static final String ROLE_LAUNCH_SERVICE = "RoleLaunchService";
-
-
- /**
- * Queue submission API
- */
- private QueueAccess actionQueue;
-
- /**
- * Filesystem to use for the launch
- */
- private SliderFileSystem fs;
-
-
- private Map<String, String> envVars = new HashMap<>();
-
- /**
- * Construct an instance of the launcher
- * @param queueAccess
- * @param fs filesystem
- * @param envVars environment variables
- */
- public RoleLaunchService(QueueAccess queueAccess, SliderFileSystem fs,
- Map<String, String> envVars) {
- super(ROLE_LAUNCH_SERVICE);
- this.actionQueue = queueAccess;
- this.fs = fs;
- this.envVars = envVars;
- }
-
- public RoleLaunchService(SliderFileSystem fs) {
- super(ROLE_LAUNCH_SERVICE);
- this.fs = fs;
- }
-
- @Override
- public void init(Configuration conf) {
- super.init(conf);
- setExecutor(Executors.newCachedThreadPool(
- new ServiceThreadFactory(ROLE_LAUNCH_SERVICE, true)));
- }
-
- /**
- * Start an asychronous launch operation
- * @param assignment container assignment
- * @param credentials credentials to use
- */
- public void launchRole(ContainerAssignment assignment,
- Application application, Credentials credentials) {
- }
-
- public void launchComponent(Application application,
- ComponentInstance instance, Container container) {
- RoleLaunchService.RoleLauncher launcher =
- new RoleLaunchService.RoleLauncher(application, instance,
- container);
- execute(launcher);
- }
-
- /**
- * Thread that runs on the AM to launch a container
- */
- private class RoleLauncher implements Runnable {
- // Allocated container
- public final Container container;
- public final Application application;
- public ComponentInstance instance;
-
- public RoleLauncher(
- Application application,
- ComponentInstance instance, Container container) {
- this.container = container;
- this.application = application;
- this.instance = instance;
- }
-
- @Override
- public void run() {
- try {
- ContainerLauncher containerLauncher =
- new ContainerLauncher(null, fs, container, null);
- containerLauncher.putEnv(envVars);
-
- Component compSpec = instance.getCompSpec();
- ProviderService provider = ProviderFactory.getProviderService(
- compSpec.getArtifact());
- provider.buildContainerLaunchContext(containerLauncher, application,
- instance, fs, getConfig());
-
- long delay = compSpec.getConfiguration()
- .getPropertyLong(KEY_CONTAINER_LAUNCH_DELAY, 0);
- long maxDelay = getConfig()
- .getLong(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
- YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS);
- if (delay > maxDelay/1000) {
- log.warn("Container launch delay of {} exceeds the maximum allowed of"
- + " {} seconds. Delay will not be utilized.",
- delay, maxDelay/1000);
- delay = 0;
- }
- if (delay > 0) {
- Thread.sleep(delay * 1000);
- }
- instance.getComponent().getScheduler().getNmClient()
- .startContainerAsync(container,
- containerLauncher.completeContainerLaunch());
- } catch (Exception e) {
- log.error("Exception thrown while trying to start " + instance
- .getCompInstanceName()
- + " container = " + container.getId() + " on host " + container
- .getNodeId(), e);
- }
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
deleted file mode 100644
index fe8cce8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.net.URL;
-import java.util.Map;
-
-/**
- * Methods to aid in config, both in the Configuration class and
- * with other parts of setting up Slider-initated processes.
- *
- * Some of the methods take an argument of a map iterable for their sources; this allows
- * the same method
- */
-public class ConfigHelper {
- private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);
-
- /**
- * Set an entire map full of values
- *
- * @param config config to patch
- * @param map map of data
- * @param origin origin data
- */
- public static void addConfigMap(Configuration config,
- Map<String, String> map,
- String origin) throws BadConfigException {
- addConfigMap(config, map.entrySet(), origin);
- }
-
- /**
- * Set an entire map full of values
- *
- * @param config config to patch
- * @param map map of data
- * @param origin origin data
- */
- public static void addConfigMap(Configuration config,
- Iterable<Map.Entry<String, String>> map,
- String origin) throws BadConfigException {
- for (Map.Entry<String, String> mapEntry : map) {
- String key = mapEntry.getKey();
- String value = mapEntry.getValue();
- if (value == null) {
- throw new BadConfigException("Null value for property " + key);
- }
- config.set(key, value, origin);
- }
- }
-
- /**
- * Convert to an XML string
- * @param conf configuration
- * @return conf
- * @throws IOException
- */
- public static String toXml(Configuration conf) throws IOException {
- StringWriter writer = new StringWriter();
- conf.writeXml(writer);
- return writer.toString();
- }
-
-
- /**
- * Register a resource as a default resource.
- * Do not attempt to use this unless you understand that the
- * order in which default resources are loaded affects the outcome,
- * and that subclasses of Configuration often register new default
- * resources
- * @param resource the resource name
- * @return the URL or null
- */
- public static URL registerDefaultResource(String resource) {
- URL resURL = getResourceUrl(resource);
- if (resURL != null) {
- Configuration.addDefaultResource(resource);
- }
- return resURL;
- }
-
- /**
- * Load a configuration from a resource on this classpath.
- * If the resource is not found, an empty configuration is returned
- * @param resource the resource name
- * @return the loaded configuration.
- */
- public static Configuration loadFromResource(String resource) {
- Configuration conf = new Configuration(false);
- URL resURL = getResourceUrl(resource);
- if (resURL != null) {
- log.debug("loaded resources from {}", resURL);
- conf.addResource(resource);
- } else{
- log.debug("failed to find {} on the classpath", resource);
- }
- return conf;
-
- }
-
- /**
- * Get the URL to a resource, null if not on the CP
- * @param resource resource to look for
- * @return the URL or null
- */
- public static URL getResourceUrl(String resource) {
- return ConfigHelper.class.getClassLoader()
- .getResource(resource);
- }
-
- /**
- * This goes through the keyset of one configuration and retrieves each value
- * from a value source -a different or the same configuration. This triggers
- * the property resolution process of the value, resolving any variables against
- * in-config or inherited configurations
- * @param keysource source of keys
- * @param valuesource the source of values
- * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code>
- */
- public static Configuration resolveConfiguration(
- Iterable<Map.Entry<String, String>> keysource,
- Configuration valuesource) {
- Configuration result = new Configuration(false);
- for (Map.Entry<String, String> entry : keysource) {
- String key = entry.getKey();
- String value = valuesource.get(key);
- Preconditions.checkState(value != null,
- "no reference for \"%s\" in values", key);
- result.set(key, value);
- }
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java
deleted file mode 100644
index a969be9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class ConfigUtils {
- public static final String TEMPLATE_FILE = "template.file";
-
- public static String replaceProps(Map<String, String> config, String content) {
- Map<String, String> tokens = new HashMap<>();
- for (Entry<String, String> entry : config.entrySet()) {
- tokens.put("${" + entry.getKey() + "}", entry.getValue());
- tokens.put("{{" + entry.getKey() + "}}", entry.getValue());
- }
- String value = content;
- for (Map.Entry<String,String> token : tokens.entrySet()) {
- value = value.replaceAll(Pattern.quote(token.getKey()),
- Matcher.quoteReplacement(token.getValue()));
- }
- return value;
- }
-
- public static Map<String, String> replacePropsInConfig(
- Map<String, String> config, Map<String, String> env) {
- Map<String, String> tokens = new HashMap<>();
- for (Entry<String, String> entry : env.entrySet()) {
- tokens.put("${" + entry.getKey() + "}", entry.getValue());
- }
- Map<String, String> newConfig = new HashMap<>();
- for (Entry<String, String> entry : config.entrySet()) {
- String value = entry.getValue();
- for (Map.Entry<String,String> token : tokens.entrySet()) {
- value = value.replaceAll(Pattern.quote(token.getKey()),
- Matcher.quoteReplacement(token.getValue()));
- }
- newConfig.put(entry.getKey(), entry.getValue());
- }
- return newConfig;
- }
-
- public static void prepConfigForTemplateOutputter(ConfigFormat configFormat,
- Map<String, String> config, SliderFileSystem fileSystem,
- String clusterName, String fileName) throws IOException {
- if (!configFormat.equals(ConfigFormat.TEMPLATE)) {
- return;
- }
- Path templateFile = null;
- if (config.containsKey(TEMPLATE_FILE)) {
- templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE));
- if (!fileSystem.isFile(templateFile)) {
- templateFile = fileSystem.buildResourcePath(clusterName,
- config.get(TEMPLATE_FILE));
- }
- if (!fileSystem.isFile(templateFile)) {
- throw new IOException("config specified template file " + config
- .get(TEMPLATE_FILE) + " but " + templateFile + " doesn't exist");
- }
- }
- if (templateFile == null && fileName != null) {
- templateFile = fileSystem.buildResourcePath(fileName);
- if (!fileSystem.isFile(templateFile)) {
- templateFile = fileSystem.buildResourcePath(clusterName,
- fileName);
- }
- }
- if (fileSystem.isFile(templateFile)) {
- config.put("content", fileSystem.cat(templateFile));
- } else {
- config.put("content", "");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
deleted file mode 100644
index fa3b402..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
+++ /dev/null
@@ -1,521 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.URL;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.util.Records;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-public class CoreFileSystem {
- private static final Logger
- log = LoggerFactory.getLogger(CoreFileSystem.class);
-
- private static final String UTF_8 = "UTF-8";
-
- protected final FileSystem fileSystem;
- protected final Configuration configuration;
-
- public CoreFileSystem(FileSystem fileSystem, Configuration configuration) {
- Preconditions.checkNotNull(fileSystem,
- "Cannot create a CoreFileSystem with a null FileSystem");
- Preconditions.checkNotNull(configuration,
- "Cannot create a CoreFileSystem with a null Configuration");
- this.fileSystem = fileSystem;
- this.configuration = configuration;
- }
-
- public CoreFileSystem(Configuration configuration) throws IOException {
- Preconditions.checkNotNull(configuration,
- "Cannot create a CoreFileSystem with a null Configuration");
- this.fileSystem = FileSystem.get(configuration);
- this.configuration = configuration;
- }
-
- /**
- * Get the temp path for this cluster
- * @param clustername name of the cluster
- * @return path for temp files (is not purged)
- */
- public Path getTempPathForCluster(String clustername) {
- Path clusterDir = buildClusterDirPath(clustername);
- return new Path(clusterDir, YarnServiceConstants.TMP_DIR_PREFIX);
- }
-
- /**
- * Returns the underlying FileSystem for this object.
- *
- * @return filesystem
- */
- public FileSystem getFileSystem() {
- return fileSystem;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("CoreFileSystem{");
- sb.append("fileSystem=").append(fileSystem.getUri());
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * Build up the path string for a cluster instance -no attempt to
- * create the directory is made
- *
- * @param clustername name of the cluster
- * @return the path for persistent data
- */
- public Path buildClusterDirPath(String clustername) {
- Preconditions.checkNotNull(clustername);
- Path path = getBaseApplicationPath();
- return new Path(path, YarnServiceConstants.SERVICES_DIRECTORY + "/" + clustername);
- }
-
-
- /**
- * Build up the path string for keytab install location -no attempt to
- * create the directory is made
- *
- * @return the path for keytab
- */
- public Path buildKeytabInstallationDirPath(String keytabFolder) {
- Preconditions.checkNotNull(keytabFolder);
- Path path = getBaseApplicationPath();
- return new Path(path, YarnServiceConstants.KEYTAB_DIR + "/" + keytabFolder);
- }
-
- /**
- * Build up the path string for keytab install location -no attempt to
- * create the directory is made
- *
- * @return the path for keytab installation location
- */
- public Path buildKeytabPath(String keytabDir, String keytabName, String clusterName) {
- Path homePath = getHomeDirectory();
- Path baseKeytabDir;
- if (keytabDir != null) {
- baseKeytabDir = new Path(homePath, keytabDir);
- } else {
- baseKeytabDir = new Path(buildClusterDirPath(clusterName),
- YarnServiceConstants.KEYTAB_DIR);
- }
- return keytabName == null ? baseKeytabDir :
- new Path(baseKeytabDir, keytabName);
- }
-
- /**
- * Build up the path string for resource install location -no attempt to
- * create the directory is made
- *
- * @return the path for resource
- */
- public Path buildResourcePath(String resourceFolder) {
- Preconditions.checkNotNull(resourceFolder);
- Path path = getBaseApplicationPath();
- return new Path(path, YarnServiceConstants.RESOURCE_DIR + "/" + resourceFolder);
- }
-
- /**
- * Build up the path string for resource install location -no attempt to
- * create the directory is made
- *
- * @return the path for resource
- */
- public Path buildResourcePath(String dirName, String fileName) {
- Preconditions.checkNotNull(dirName);
- Preconditions.checkNotNull(fileName);
- Path path = getBaseApplicationPath();
- return new Path(path, YarnServiceConstants.RESOURCE_DIR + "/" + dirName + "/" + fileName);
- }
-
- /**
- * Create a directory with the given permissions.
- *
- * @param dir directory
- * @param clusterPerms cluster permissions
- * @throws IOException IO problem
- * @throws BadClusterStateException any cluster state problem
- */
- @SuppressWarnings("deprecation")
- public void createWithPermissions(Path dir, FsPermission clusterPerms) throws
- IOException,
- BadClusterStateException {
- if (fileSystem.isFile(dir)) {
- // HADOOP-9361 shows some filesystems don't correctly fail here
- throw new BadClusterStateException(
- "Cannot create a directory over a file %s", dir);
- }
- log.debug("mkdir {} with perms {}", dir, clusterPerms);
- //no mask whatoever
- fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");
- fileSystem.mkdirs(dir, clusterPerms);
- //and force set it anyway just to make sure
- fileSystem.setPermission(dir, clusterPerms);
- }
-
- /**
- * Verify that the cluster directory is not present
- *
- * @param clustername name of the cluster
- * @param clusterDirectory actual directory to look for
- * @throws IOException trouble with FS
- * @throws SliderException If the directory exists
- */
- public void verifyClusterDirectoryNonexistent(String clustername,
- Path clusterDirectory)
- throws IOException, SliderException {
- if (fileSystem.exists(clusterDirectory)) {
- throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
- ErrorStrings.PRINTF_E_INSTANCE_ALREADY_EXISTS, clustername,
- clusterDirectory);
- }
- }
- /**
- * Verify that the given directory is not present
- *
- * @param clusterDirectory actual directory to look for
- * @throws IOException trouble with FS
- * @throws SliderException If the directory exists
- */
- public void verifyDirectoryNonexistent(Path clusterDirectory) throws
- IOException,
- SliderException {
- if (fileSystem.exists(clusterDirectory)) {
-
- log.error("Dir {} exists: {}",
- clusterDirectory,
- listFSDir(clusterDirectory));
- throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
- ErrorStrings.PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS,
- clusterDirectory);
- }
- }
-
- /**
- * Verify that a user has write access to a directory.
- * It does this by creating then deleting a temp file
- *
- * @param dirPath actual directory to look for
- * @throws FileNotFoundException file not found
- * @throws IOException trouble with FS
- * @throws BadClusterStateException if the directory is not writeable
- */
- public void verifyDirectoryWriteAccess(Path dirPath) throws IOException,
- SliderException {
- verifyPathExists(dirPath);
- Path tempFile = new Path(dirPath, "tmp-file-for-checks");
- try {
- FSDataOutputStream out ;
- out = fileSystem.create(tempFile, true);
- IOUtils.closeStream(out);
- fileSystem.delete(tempFile, false);
- } catch (IOException e) {
- log.warn("Failed to create file {}: {}", tempFile, e);
- throw new BadClusterStateException(e,
- "Unable to write to directory %s : %s", dirPath, e.toString());
- }
- }
-
- /**
- * Verify that a path exists
- * @param path path to check
- * @throws FileNotFoundException file not found
- * @throws IOException trouble with FS
- */
- public void verifyPathExists(Path path) throws IOException {
- if (!fileSystem.exists(path)) {
- throw new FileNotFoundException(path.toString());
- }
- }
-
- /**
- * Verify that a path exists
- * @param path path to check
- * @throws FileNotFoundException file not found or is not a file
- * @throws IOException trouble with FS
- */
- public void verifyFileExists(Path path) throws IOException {
- FileStatus status = fileSystem.getFileStatus(path);
-
- if (!status.isFile()) {
- throw new FileNotFoundException("Not a file: " + path.toString());
- }
- }
-
- /**
- * Given a path, check if it exists and is a file
- *
- * @param path
- * absolute path to the file to check
- * @return true if and only if path exists and is a file, false for all other
- * reasons including if file check throws IOException
- */
- public boolean isFile(Path path) {
- boolean isFile = false;
- try {
- FileStatus status = fileSystem.getFileStatus(path);
- if (status.isFile()) {
- isFile = true;
- }
- } catch (IOException e) {
- // ignore, isFile is already set to false
- }
- return isFile;
- }
-
- /**
- * Get the base path
- *
- * @return the base path optionally configured by
- * {@link YarnServiceConf#YARN_SERVICE_BASE_PATH}
- */
- public Path getBaseApplicationPath() {
- String configuredBasePath = configuration
- .get(YarnServiceConf.YARN_SERVICE_BASE_PATH,
- getHomeDirectory() + "/" + YarnServiceConstants.SERVICE_BASE_DIRECTORY);
- return new Path(configuredBasePath);
- }
-
- /**
- * Get slider dependency parent dir in HDFS
- *
- * @return the parent dir path of slider.tar.gz in HDFS
- */
- public Path getDependencyPath() {
- String parentDir = YarnServiceConstants.DEPENDENCY_DIR;
- return new Path(String.format(parentDir, VersionInfo.getVersion()));
- }
-
- /**
- * Get slider.tar.gz absolute filepath in HDFS
- *
- * @return the absolute path to slider.tar.gz in HDFS
- */
- public Path getDependencyTarGzip() {
- Path dependencyLibAmPath = getDependencyPath();
- Path dependencyLibTarGzip = new Path(
- dependencyLibAmPath.toUri().toString(),
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
- + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
- return dependencyLibTarGzip;
- }
-
- public Path getHomeDirectory() {
- return fileSystem.getHomeDirectory();
- }
-
- /**
- * Create an AM resource from the
- *
- * @param destPath dest path in filesystem
- * @param resourceType resource type
- * @return the local resource for AM
- */
- public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException {
- FileStatus destStatus = fileSystem.getFileStatus(destPath);
- LocalResource amResource = Records.newRecord(LocalResource.class);
- amResource.setType(resourceType);
- // Set visibility of the resource
- // Setting to most private option
- amResource.setVisibility(LocalResourceVisibility.APPLICATION);
- // Set the resource to be copied over
- amResource.setResource(
- URL.fromPath(fileSystem.resolvePath(destStatus.getPath())));
- // Set timestamp and length of file so that the framework
- // can do basic sanity checks for the local resource
- // after it has been copied over to ensure it is the same
- // resource the client intended to use with the application
- amResource.setTimestamp(destStatus.getModificationTime());
- amResource.setSize(destStatus.getLen());
- return amResource;
- }
-
- /**
- * Register all files under a fs path as a directory to push out
- *
- * @param srcDir src dir
- * @param destRelativeDir dest dir (no trailing /)
- * @return the map of entries
- */
- public Map<String, LocalResource> submitDirectory(Path srcDir, String destRelativeDir) throws IOException {
- //now register each of the files in the directory to be
- //copied to the destination
- FileStatus[] fileset = fileSystem.listStatus(srcDir);
- Map<String, LocalResource> localResources =
- new HashMap<String, LocalResource>(fileset.length);
- for (FileStatus entry : fileset) {
-
- LocalResource resource = createAmResource(entry.getPath(),
- LocalResourceType.FILE);
- String relativePath = destRelativeDir + "/" + entry.getPath().getName();
- localResources.put(relativePath, resource);
- }
- return localResources;
- }
-
- /**
- * Submit a JAR containing a specific class, returning
- * the resource to be mapped in
- *
- * @param clazz class to look for
- * @param subdir subdirectory (expected to end in a "/")
- * @param jarName <i>At the destination</i>
- * @return the local resource ref
- * @throws IOException trouble copying to HDFS
- */
- public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName)
- throws IOException, SliderException {
- File localFile = SliderUtils.findContainingJarOrFail(clazz);
- return submitFile(localFile, tempPath, subdir, jarName);
- }
-
- /**
- * Submit a local file to the filesystem references by the instance's cluster
- * filesystem
- *
- * @param localFile filename
- * @param subdir subdirectory (expected to end in a "/")
- * @param destFileName destination filename
- * @return the local resource ref
- * @throws IOException trouble copying to HDFS
- */
- public LocalResource submitFile(File localFile, Path tempPath, String subdir, String destFileName)
- throws IOException {
- Path src = new Path(localFile.toString());
- Path subdirPath = new Path(tempPath, subdir);
- fileSystem.mkdirs(subdirPath);
- Path destPath = new Path(subdirPath, destFileName);
- log.debug("Copying {} (size={} bytes) to {}", localFile, localFile.length(), destPath);
-
- fileSystem.copyFromLocalFile(false, true, src, destPath);
-
- // Set the type of resource - file or archive
- // archives are untarred at destination
- // we don't need the jar file to be untarred for now
- return createAmResource(destPath, LocalResourceType.FILE);
- }
-
- /**
- * Submit the AM tar.gz resource referenced by the instance's cluster
- * filesystem. Also, update the providerResources object with the new
- * resource.
- *
- * @param providerResources
- * the provider resource map to be updated
- * @throws IOException
- * trouble copying to HDFS
- */
- public void submitTarGzipAndUpdate(
- Map<String, LocalResource> providerResources) throws IOException,
- BadClusterStateException {
- Path dependencyLibTarGzip = getDependencyTarGzip();
- LocalResource lc = createAmResource(dependencyLibTarGzip,
- LocalResourceType.ARCHIVE);
- providerResources.put(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK, lc);
- }
-
- public void copyLocalFileToHdfs(File localPath,
- Path destPath, FsPermission fp)
- throws IOException {
- if (localPath == null || destPath == null) {
- throw new IOException("Either localPath or destPath is null");
- }
- fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
- "000");
- fileSystem.mkdirs(destPath.getParent(), fp);
- log.info("Copying file {} to {}", localPath.toURI(),
- fileSystem.getScheme() + ":/" + destPath.toUri());
-
- fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()),
- destPath);
- // set file permissions of the destPath
- fileSystem.setPermission(destPath, fp);
- }
-
- public void copyHdfsFileToLocal(Path hdfsPath, File destFile)
- throws IOException {
- if (hdfsPath == null || destFile == null) {
- throw new IOException("Either hdfsPath or destPath is null");
- }
- log.info("Copying file {} to {}", hdfsPath.toUri(), destFile.toURI());
-
- Path destPath = new Path(destFile.getPath());
- fileSystem.copyToLocalFile(hdfsPath, destPath);
- }
-
- /**
- * list entries in a filesystem directory
- *
- * @param path directory
- * @return a listing, one to a line
- * @throws IOException
- */
- public String listFSDir(Path path) throws IOException {
- FileStatus[] stats = fileSystem.listStatus(path);
- StringBuilder builder = new StringBuilder();
- for (FileStatus stat : stats) {
- builder.append(stat.getPath().toString())
- .append("\t")
- .append(stat.getLen())
- .append("\n");
- }
- return builder.toString();
- }
-
- public String cat(Path path) throws IOException {
- FileStatus status = fileSystem.getFileStatus(path);
- byte[] b = new byte[(int) status.getLen()];
- FSDataInputStream in = null;
- try {
- in = fileSystem.open(path);
- int count = in.read(b);
- return new String(b, 0, count, UTF_8);
- } finally {
- IOUtils.closeStream(in);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java
deleted file mode 100644
index 6fadfd3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import java.io.Closeable;
-
-/**
- * A duration in milliseconds. This class can be used
- * to count time, and to be polled to see if a time limit has
- * passed.
- */
-public class Duration implements Closeable {
- public long start, finish;
- public final long limit;
-
- /**
- * Create a duration instance with a limit of 0
- */
- public Duration() {
- this(0);
- }
-
- /**
- * Create a duration with a limit specified in millis
- * @param limit duration in milliseconds
- */
- public Duration(long limit) {
- this.limit = limit;
- }
-
- /**
- * Start
- * @return self
- */
- public Duration start() {
- start = now();
- return this;
- }
-
- /**
- * The close operation relays to {@link #finish()}.
- * Implementing it allows Duration instances to be automatically
- * finish()'d in Java7 try blocks for when used in measuring durations.
- */
- @Override
- public final void close() {
- finish();
- }
-
- public void finish() {
- finish = now();
- }
-
- protected long now() {
- return System.nanoTime()/1000000;
- }
-
- public long getInterval() {
- return finish - start;
- }
-
- /**
- * return true if the limit has been exceeded
- * @return true if a limit was set and the current time
- * exceeds it.
- */
- public boolean getLimitExceeded() {
- return limit >= 0 && ((now() - start) > limit);
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append("Duration");
- if (finish >= start) {
- builder.append(" finished at ").append(getInterval()).append(" millis;");
- } else {
- if (start > 0) {
- builder.append(" started but not yet finished;");
- } else {
- builder.append(" unstarted;");
- }
- }
- if (limit > 0) {
- builder.append(" limit: ").append(limit).append(" millis");
- if (getLimitExceeded()) {
- builder.append(" - exceeded");
- }
- }
- return builder.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
deleted file mode 100644
index 7b22e3e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.codehaus.jackson.JsonGenerationException;
-import org.codehaus.jackson.JsonParseException;
-import org.codehaus.jackson.map.DeserializationConfig;
-import org.codehaus.jackson.map.JsonMappingException;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.PropertyNamingStrategy;
-import org.codehaus.jackson.map.SerializationConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-/**
- * Support for marshalling objects to and from JSON.
- * This class is NOT thread safe; it constructs an object mapper
- * as an instance field.
- * @param <T>
- */
-public class JsonSerDeser<T> {
-
- private static final Logger log = LoggerFactory.getLogger(JsonSerDeser.class);
- private static final String UTF_8 = "UTF-8";
-
- private final Class<T> classType;
- private final ObjectMapper mapper;
-
- /**
- * Create an instance bound to a specific type
- * @param classType class type
- */
- public JsonSerDeser(Class<T> classType) {
- this.classType = classType;
- this.mapper = new ObjectMapper();
- mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- public JsonSerDeser(Class<T> classType, PropertyNamingStrategy namingStrategy) {
- this(classType);
- mapper.setPropertyNamingStrategy(namingStrategy);
- }
-
- /**
- * Convert from JSON
- * @param json input
- * @return the parsed JSON
- * @throws IOException IO
- * @throws JsonMappingException failure to map from the JSON to this class
- */
- public T fromJson(String json)
- throws IOException, JsonParseException, JsonMappingException {
- try {
- return mapper.readValue(json, classType);
- } catch (IOException e) {
- log.error("Exception while parsing json : " + e + "\n" + json, e);
- throw e;
- }
- }
-
- /**
- * Convert from a JSON file
- * @param jsonFile input file
- * @return the parsed JSON
- * @throws IOException IO problems
- * @throws JsonMappingException failure to map from the JSON to this class
- */
- public T fromFile(File jsonFile)
- throws IOException, JsonParseException, JsonMappingException {
- File absoluteFile = jsonFile.getAbsoluteFile();
- try {
- return mapper.readValue(absoluteFile, classType);
- } catch (IOException e) {
- log.error("Exception while parsing json file {}", absoluteFile, e);
- throw e;
- }
- }
-
- /**
- * Convert from a JSON file
- * @param resource input file
- * @return the parsed JSON
- * @throws IOException IO problems
- * @throws JsonMappingException failure to map from the JSON to this class
- */
- public T fromResource(String resource)
- throws IOException, JsonParseException, JsonMappingException {
- try(InputStream resStream = this.getClass().getResourceAsStream(resource)) {
- if (resStream == null) {
- throw new FileNotFoundException(resource);
- }
- return (T) (mapper.readValue(resStream, classType));
- } catch (IOException e) {
- log.error("Exception while parsing json resource {}", resource, e);
- throw e;
- }
- }
-
- /**
- * Convert from an input stream, closing the stream afterwards.
- * @param stream
- * @return the parsed JSON
- * @throws IOException IO problems
- */
- public T fromStream(InputStream stream) throws IOException {
- try {
- return (T) (mapper.readValue(stream, classType));
- } catch (IOException e) {
- log.error("Exception while parsing json input stream", e);
- throw e;
- } finally {
- IOUtils.closeStream(stream);
- }
- }
-
- /**
- * clone by converting to JSON and back again.
- * This is much less efficient than any Java clone process.
- * @param instance instance to duplicate
- * @return a new instance
- * @throws IOException problems.
- */
- public T fromInstance(T instance) throws IOException {
- return fromJson(toJson(instance));
- }
-
- /**
- * Deserialize from a byte array
- * @param b
- * @return the deserialized value
- * @throws IOException parse problems
- */
- public T fromBytes(byte[] b) throws IOException {
- String json = new String(b, 0, b.length, UTF_8);
- return fromJson(json);
- }
-
- /**
- * Load from a Hadoop filesystem
- * @param fs filesystem
- * @param path path
- * @return a loaded CD
- * @throws IOException IO problems
- * @throws JsonParseException parse problems
- * @throws JsonMappingException O/J mapping problems
- */
- public T load(FileSystem fs, Path path)
- throws IOException, JsonParseException, JsonMappingException {
- FileStatus status = fs.getFileStatus(path);
- long len = status.getLen();
- byte[] b = new byte[(int) len];
- FSDataInputStream dataInputStream = fs.open(path);
- int count = dataInputStream.read(b);
- if (count != len) {
- throw new EOFException("Read of " + path +" finished prematurely");
- }
- return fromBytes(b);
- }
-
-
- /**
- * Save to a hadoop filesystem
- * @param fs filesystem
- * @param path path
- * @param instance instance to save
- * @param overwrite should any existing file be overwritten
- * @throws IOException IO exception
- */
- public void save(FileSystem fs, Path path, T instance,
- boolean overwrite) throws
- IOException {
- FSDataOutputStream dataOutputStream = fs.create(path, overwrite);
- writeJsonAsBytes(instance, dataOutputStream);
- }
-
- /**
- * Save an instance to a file
- * @param instance instance to save
- * @param file file
- * @throws IOException
- */
- public void save(T instance, File file) throws
- IOException {
- writeJsonAsBytes(instance, new FileOutputStream(file.getAbsoluteFile()));
- }
-
- /**
- * Write the json as bytes -then close the file
- * @param dataOutputStream an outout stream that will always be closed
- * @throws IOException on any failure
- */
- private void writeJsonAsBytes(T instance,
- OutputStream dataOutputStream) throws IOException {
- try {
- String json = toJson(instance);
- byte[] b = json.getBytes(UTF_8);
- dataOutputStream.write(b);
- dataOutputStream.flush();
- dataOutputStream.close();
- } finally {
- IOUtils.closeStream(dataOutputStream);
- }
- }
-
- /**
- * Convert an object to a JSON string
- * @param instance instance to convert
- * @return a JSON string description
- * @throws JsonParseException parse problems
- * @throws JsonMappingException O/J mapping problems
- */
- public String toJson(T instance) throws IOException,
- JsonGenerationException,
- JsonMappingException {
- mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
- return mapper.writeValueAsString(instance);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
deleted file mode 100644
index c0712c3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
+++ /dev/null
@@ -1,680 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SaslPropertiesResolver;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.Shell;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.crypto.Cipher;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.lang.reflect.InvocationTargetException;
-import java.net.InetAddress;
-import java.security.NoSuchAlgorithmException;
-import java.util.Collection;
-import java.util.Date;
-import java.util.List;
-import java.util.regex.Pattern;
-
-import static org.apache.hadoop.security.UserGroupInformation.*;
-import static org.apache.hadoop.security.authentication.util.KerberosUtil.*;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
-
-/**
- * Kerberos diagnostics
- * At some point this may move to hadoop core, so please keep use of slider
- * methods and classes to ~0.
- *
- * This operation expands some of the diagnostic output of the security code,
- * but not all. For completeness
- *
- * Set the environment variable {@code HADOOP_JAAS_DEBUG=true}
- * Set the log level for {@code org.apache.hadoop.security=DEBUG}
- */
-public class KerberosDiags implements Closeable {
-
- private static final Logger LOG = LoggerFactory.getLogger(KerberosDiags.class);
- public static final String KRB5_CCNAME = "KRB5CCNAME";
- public static final String JAVA_SECURITY_KRB5_CONF
- = "java.security.krb5.conf";
- public static final String JAVA_SECURITY_KRB5_REALM
- = "java.security.krb5.realm";
- public static final String SUN_SECURITY_KRB5_DEBUG
- = "sun.security.krb5.debug";
- public static final String SUN_SECURITY_SPNEGO_DEBUG
- = "sun.security.spnego.debug";
- public static final String SUN_SECURITY_JAAS_FILE
- = "java.security.auth.login.config";
- public static final String KERBEROS_KINIT_COMMAND
- = "hadoop.kerberos.kinit.command";
- public static final String HADOOP_AUTHENTICATION_IS_DISABLED
- = "Hadoop authentication is disabled";
- public static final String UNSET = "(unset)";
- public static final String NO_DEFAULT_REALM = "Cannot locate default realm";
-
- private final Configuration conf;
- private final List<String> services;
- private final PrintStream out;
- private final File keytab;
- private final String principal;
- private final long minKeyLength;
- private final boolean securityRequired;
-
- public static final String CAT_JVM = "JVM";
- public static final String CAT_JAAS = "JAAS";
- public static final String CAT_CONFIG = "CONFIG";
- public static final String CAT_LOGIN = "LOGIN";
- public static final String CAT_KERBEROS = "KERBEROS";
- public static final String CAT_SASL = "SASL";
-
- @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
- public KerberosDiags(Configuration conf,
- PrintStream out,
- List<String> services,
- File keytab,
- String principal,
- long minKeyLength,
- boolean securityRequired) {
- this.conf = conf;
- this.services = services;
- this.keytab = keytab;
- this.principal = principal;
- this.out = out;
- this.minKeyLength = minKeyLength;
- this.securityRequired = securityRequired;
- }
-
- @Override
- public void close() throws IOException {
- flush();
- }
-
- /**
- * Execute diagnostics.
- * <p>
- * Things it would be nice if UGI made accessible
- * <ol>
- * <li>A way to enable JAAS debug programatically</li>
- * <li>Access to the TGT</li>
- * </ol>
- * @return true if security was enabled and all probes were successful
- * @throws KerberosDiagsFailure explicitly raised failure
- * @throws Exception other security problems
- */
- @SuppressWarnings("deprecation")
- public boolean execute() throws Exception {
-
- title("Kerberos Diagnostics scan at %s",
- new Date(System.currentTimeMillis()));
-
- // check that the machine has a name
- println("Hostname: %s",
- InetAddress.getLocalHost().getCanonicalHostName());
-
- // Fail fast on a JVM without JCE installed.
- validateKeyLength();
-
- // look at realm
- println("JVM Kerberos Login Module = %s", getKrb5LoginModuleName());
- printDefaultRealm();
-
- title("System Properties");
- for (String prop : new String[]{
- JAVA_SECURITY_KRB5_CONF,
- JAVA_SECURITY_KRB5_REALM,
- SUN_SECURITY_KRB5_DEBUG,
- SUN_SECURITY_SPNEGO_DEBUG,
- SUN_SECURITY_JAAS_FILE
- }) {
- printSysprop(prop);
- }
-
- title("Environment Variables");
- for (String env : new String[]{
- "HADOOP_JAAS_DEBUG",
- KRB5_CCNAME,
- "HADOOP_USER_NAME",
- "HADOOP_PROXY_USER",
- HADOOP_TOKEN_FILE_LOCATION,
- }) {
- printEnv(env);
- }
-
- for (String prop : new String[]{
- KERBEROS_KINIT_COMMAND,
- HADOOP_SECURITY_AUTHENTICATION,
- HADOOP_SECURITY_AUTHORIZATION,
- "hadoop.kerberos.min.seconds.before.relogin", // not in 2.6
- "hadoop.security.dns.interface", // not in 2.6
- "hadoop.security.dns.nameserver", // not in 2.6
- HADOOP_RPC_PROTECTION,
- HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
- HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX,
- HADOOP_SECURITY_GROUP_MAPPING,
- "hadoop.security.impersonation.provider.class", // not in 2.6
- "dfs.data.transfer.protection" // HDFS
- }) {
- printConfOpt(prop);
- }
-
- // check that authentication is enabled
- if (SecurityUtil.getAuthenticationMethod(conf)
- .equals(AuthenticationMethod.SIMPLE)) {
- println(HADOOP_AUTHENTICATION_IS_DISABLED);
- failif(securityRequired, CAT_CONFIG, HADOOP_AUTHENTICATION_IS_DISABLED);
- // no security, skip rest of test
- return false;
- }
-
- validateKrb5File();
- validateSasl(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS);
- validateSasl("dfs.data.transfer.saslproperties.resolver.class");
- validateKinitExecutable();
- validateJAAS();
- // now the big test: login, then try again
- boolean krb5Debug = getAndSet(SUN_SECURITY_KRB5_DEBUG);
- boolean spnegoDebug = getAndSet(SUN_SECURITY_SPNEGO_DEBUG);
- try {
- title("Logging in");
-
- if (keytab != null) {
- dumpKeytab(keytab);
- loginFromKeytab();
- } else {
- UserGroupInformation loginUser = getLoginUser();
- dumpUGI("Log in user", loginUser);
- validateUGI("Login user", loginUser);
- println("Ticket based login: %b", isLoginTicketBased());
- println("Keytab based login: %b", isLoginKeytabBased());
- }
-
- return true;
- } finally {
- // restore original system properties
- System.setProperty(SUN_SECURITY_KRB5_DEBUG,
- Boolean.toString(krb5Debug));
- System.setProperty(SUN_SECURITY_SPNEGO_DEBUG,
- Boolean.toString(spnegoDebug));
- }
- }
-
- /**
- * Fail fast on a JVM without JCE installed.
- *
- * This is a recurrent problem
- * (that is: it keeps creeping back with JVM updates);
- * a fast failure is the best tactic
- * @throws NoSuchAlgorithmException
- */
-
- protected void validateKeyLength() throws NoSuchAlgorithmException {
- int aesLen = Cipher.getMaxAllowedKeyLength("AES");
- println("Maximum AES encryption key length %d bits", aesLen);
- failif (aesLen < minKeyLength,
- CAT_JVM,
- "Java Cryptography Extensions are not installed on this JVM."
- +" Maximum supported key length %s - minimum required %d",
- aesLen, minKeyLength);
- }
-
- /**
- * Get the default realm.
- * <p>
- * Not having a default realm may be harmless, so is noted at info.
- * All other invocation failures are downgraded to warn, as
- * follow-on actions may still work.
- * failure to invoke the method via introspection is rejected,
- * as it's a sign of JVM compatibility issues that may have other
- * consequences
- */
- protected void printDefaultRealm() {
- try {
- println("Default Realm = %s",
- getDefaultRealm());
- } catch (ClassNotFoundException
- | IllegalAccessException
- | NoSuchMethodException e) {
-
- throw new KerberosDiagsFailure(CAT_JVM, e,
- "Failed to invoke krb5.Config.getDefaultRealm: %s", e);
- } catch (InvocationTargetException e) {
- Throwable cause = e.getCause() != null ? e.getCause() : e;
- if (cause.toString().contains(NO_DEFAULT_REALM)) {
- // exception raised if there is no default realm. This is not
- // always a problem, so downgrade to a message.
- println("Host has no default realm");
- LOG.debug(cause.toString(), cause);
- } else {
- println("Kerberos.getDefaultRealm() failed: %s\n%s",
- cause,
- org.apache.hadoop.util.StringUtils.stringifyException(cause));
- }
- }
- }
-
- /**
- * Locate the krb5.conf file and dump it.
- * No-op on windows.
- * @throws IOException
- */
- private void validateKrb5File() throws IOException {
- if (!Shell.WINDOWS) {
- title("Locating Kerberos configuration file");
- String krbPath = "/etc/krb5.conf";
- String jvmKrbPath = System.getProperty(JAVA_SECURITY_KRB5_CONF);
- if (jvmKrbPath != null) {
- println("Setting kerberos path from sysprop %s: %s",
- JAVA_SECURITY_KRB5_CONF, jvmKrbPath);
- krbPath = jvmKrbPath;
- }
-
- String krb5name = System.getenv(KRB5_CCNAME);
- if (krb5name != null) {
- println("Setting kerberos path from environment variable %s: %s",
- KRB5_CCNAME, krb5name);
- krbPath = krb5name;
- if (jvmKrbPath != null) {
- println("Warning - both %s and %s were set - %s takes priority",
- JAVA_SECURITY_KRB5_CONF, KRB5_CCNAME, KRB5_CCNAME);
- }
- }
-
- File krbFile = new File(krbPath);
- println("Kerberos configuration file = %s", krbFile);
- failif(!krbFile.exists(),
- CAT_KERBEROS,
- "Kerberos configuration file %s not found", krbFile);
- dump(krbFile);
- }
- }
-
- /**
- * Dump a keytab: list all principals.
- * @param keytabFile the keytab file
- * @throws IOException IO problems
- */
- public void dumpKeytab(File keytabFile) throws IOException {
- title("Examining keytab %s", keytabFile);
- File kt = keytabFile.getCanonicalFile();
- failif(!kt.exists(), CAT_CONFIG, "Keytab not found: %s", kt);
- failif(!kt.isFile(), CAT_CONFIG, "Keytab is not a valid file: %s", kt);
-
- String[] names = getPrincipalNames(keytabFile.getCanonicalPath(),
- Pattern.compile(".*"));
- println("keytab entry count: %d", names.length);
- for (String name : names) {
- println(" %s", name);
- }
- println("-----");
- }
-
- /**
- * Log in from a keytab, dump the UGI, validate it, then try and log in again.
- * That second-time login catches JVM/Hadoop compatibility problems.
- * @throws IOException
- */
- private void loginFromKeytab() throws IOException {
- UserGroupInformation ugi;
- String identity;
- if (keytab != null) {
- File kt = keytab.getCanonicalFile();
- println("Using keytab %s principal %s", kt, principal);
- identity = principal;
-
- failif(StringUtils.isEmpty(principal), CAT_KERBEROS,
- "No principal defined");
- ugi = loginUserFromKeytabAndReturnUGI(principal, kt.getPath());
- dumpUGI(identity, ugi);
- validateUGI(principal, ugi);
-
- title("Attempting to log in from keytab again");
- // package scoped -hence the reason why this class must be in the
- // hadoop.security package
- setShouldRenewImmediatelyForTests(true);
- // attempt a new login
- ugi.reloginFromKeytab();
- } else {
- println("No keytab: logging is as current user");
- }
- }
-
- /**
- * Dump a UGI.
- * @param title title of this section
- * @param ugi UGI to dump
- * @throws IOException
- */
- private void dumpUGI(String title, UserGroupInformation ugi)
- throws IOException {
- title(title);
- println("UGI instance = %s", ugi);
- println("Has kerberos credentials: %b", ugi.hasKerberosCredentials());
- println("Authentication method: %s", ugi.getAuthenticationMethod());
- println("Real Authentication method: %s",
- ugi.getRealAuthenticationMethod());
- title("Group names");
- for (String name : ugi.getGroupNames()) {
- println(name);
- }
- title("Credentials");
- Credentials credentials = ugi.getCredentials();
- List<Text> secretKeys = credentials.getAllSecretKeys();
- title("Secret keys");
- if (!secretKeys.isEmpty()) {
- for (Text secret: secretKeys) {
- println("%s", secret);
- }
- } else {
- println("(none)");
- }
-
- dumpTokens(ugi);
- }
-
- /**
- * Validate the UGI: verify it is kerberized.
- * @param messagePrefix message in exceptions
- * @param user user to validate
- */
- private void validateUGI(String messagePrefix, UserGroupInformation user) {
- failif(!user.hasKerberosCredentials(),
- CAT_LOGIN, "%s: No kerberos credentials for %s", messagePrefix, user);
- failif(user.getAuthenticationMethod() == null,
- CAT_LOGIN, "%s: Null AuthenticationMethod for %s", messagePrefix, user);
- }
-
- /**
- * A cursory look at the {@code kinit} executable.
- * If it is an absolute path: it must exist with a size > 0.
- * If it is just a command, it has to be on the path. There's no check
- * for that -but the PATH is printed out.
- */
- private void validateKinitExecutable() {
- String kinit = conf.getTrimmed(KERBEROS_KINIT_COMMAND, "");
- if (!kinit.isEmpty()) {
- File kinitPath = new File(kinit);
- println("%s = %s", KERBEROS_KINIT_COMMAND, kinitPath);
- if (kinitPath.isAbsolute()) {
- failif(!kinitPath.exists(), CAT_KERBEROS,
- "%s executable does not exist: %s",
- KERBEROS_KINIT_COMMAND, kinitPath);
- failif(!kinitPath.isFile(), CAT_KERBEROS,
- "%s path does not refer to a file: %s",
- KERBEROS_KINIT_COMMAND, kinitPath);
- failif(kinitPath.length() == 0, CAT_KERBEROS,
- "%s file is empty: %s",
- KERBEROS_KINIT_COMMAND, kinitPath);
- } else {
- println("Executable %s is relative -must be on the PATH", kinit);
- printEnv("PATH");
- }
- }
- }
-
- /**
- * Try to load the SASL resolver.
- * @param saslPropsResolverKey key for the SASL resolver
- */
- private void validateSasl(String saslPropsResolverKey) {
- title("Resolving SASL property %s", saslPropsResolverKey);
- String saslPropsResolver = conf.getTrimmed(saslPropsResolverKey);
- try {
- Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass(
- saslPropsResolverKey,
- SaslPropertiesResolver.class, SaslPropertiesResolver.class);
- println("Resolver is %s", resolverClass);
- } catch (RuntimeException e) {
- throw new KerberosDiagsFailure(CAT_SASL, e,
- "Failed to load %s class %s",
- saslPropsResolverKey, saslPropsResolver);
- }
- }
-
- /**
- * Validate any JAAS entry referenced in the {@link #SUN_SECURITY_JAAS_FILE}
- * property.
- */
- private void validateJAAS() {
- String jaasFilename = System.getProperty(SUN_SECURITY_JAAS_FILE);
- if (jaasFilename != null) {
- title("JAAS");
- File jaasFile = new File(jaasFilename);
- println("JAAS file is defined in %s: %s",
- SUN_SECURITY_JAAS_FILE, jaasFile);
- failif(!jaasFile.exists(), CAT_JAAS,
- "JAAS file does not exist: %s", jaasFile);
- failif(!jaasFile.isFile(), CAT_JAAS,
- "Specified JAAS file is not a file: %s", jaasFile);
- }
- }
-
- /**
- * Dump all tokens of a user
- * @param user user
- */
- public void dumpTokens(UserGroupInformation user) {
- Collection<Token<? extends TokenIdentifier>> tokens
- = user.getCredentials().getAllTokens();
- title("Token Count: %d", tokens.size());
- for (Token<? extends TokenIdentifier> token : tokens) {
- println("Token %s", token.getKind());
- }
- }
-
- /**
- * Set the System property to true; return the old value for caching
- * @param sysprop property
- * @return the previous value
- */
- private boolean getAndSet(String sysprop) {
- boolean old = Boolean.getBoolean(sysprop);
- System.setProperty(sysprop, "true");
- return old;
- }
-
- /**
- * Flush all active output channels, including {@Code System.err},
- * so as to stay in sync with any JRE log messages.
- */
- private void flush() {
- if (out != null) {
- out.flush();
- } else {
- System.out.flush();
- }
- System.err.flush();
- }
-
- /**
- * Format and print a line of output.
- * This goes to any output file, or
- * is logged at info. The output is flushed before and after, to
- * try and stay in sync with JRE logging.
- * @param format format string
- * @param args any arguments
- */
- @VisibleForTesting
- public void println(String format, Object... args) {
- println(format(format, args));
- }
-
- /**
- * Print a line of output. This goes to any output file, or
- * is logged at info. The output is flushed before and after, to
- * try and stay in sync with JRE logging.
- * @param msg message string
- */
- @VisibleForTesting
- private void println(String msg) {
- flush();
- if (out != null) {
- out.println(msg);
- } else {
- LOG.info(msg);
- }
- flush();
- }
-
- /**
- * Print a title entry
- * @param format format string
- * @param args any arguments
- */
- private void title(String format, Object... args) {
- println("");
- println("");
- String msg = "== " + format(format, args) + " ==";
- println(msg);
- println("");
- }
-
- /**
- * Print a system property, or {@link #UNSET} if unset.
- * @param property property to print
- */
- private void printSysprop(String property) {
- println("%s = \"%s\"", property,
- System.getProperty(property, UNSET));
- }
-
- /**
- * Print a configuration option, or {@link #UNSET} if unset.
- * @param option option to print
- */
- private void printConfOpt(String option) {
- println("%s = \"%s\"", option, conf.get(option, UNSET));
- }
-
- /**
- * Print an environment variable's name and value; printing
- * {@link #UNSET} if it is not set
- * @param variable environment variable
- */
- private void printEnv(String variable) {
- String env = System.getenv(variable);
- println("%s = \"%s\"", variable, env != null ? env : UNSET);
- }
-
- /**
- * Dump any file to standard out; add a trailing newline
- * @param file file to dump
- * @throws IOException IO problems
- */
- public void dump(File file) throws IOException {
- try (FileInputStream in = new FileInputStream(file)) {
- for (String line : IOUtils.readLines(in)) {
- println("%s", line);
- }
- }
- println("");
- }
-
- /**
- * Format and raise a failure
- *
- * @param category category for exception
- * @param message string formatting message
- * @param args any arguments for the formatting
- * @throws KerberosDiagsFailure containing the formatted text
- */
- private void fail(String category, String message, Object... args)
- throws KerberosDiagsFailure {
- throw new KerberosDiagsFailure(category, message, args);
- }
-
- /**
- * Conditional failure with string formatted arguments
- * @param condition failure condition
- * @param category category for exception
- * @param message string formatting message
- * @param args any arguments for the formatting
- * @throws KerberosDiagsFailure containing the formatted text
- * if the condition was met
- */
- private void failif(boolean condition,
- String category,
- String message,
- Object... args)
- throws KerberosDiagsFailure {
- if (condition) {
- fail(category, message, args);
- }
- }
-
- /**
- * Format a string, treating a call where there are no varags values
- * as a string to pass through unformatted.
- * @param message message, which is either a format string + args, or
- * a general string
- * @param args argument array
- * @return a string for printing.
- */
- public static String format(String message, Object... args) {
- if (args.length == 0) {
- return message;
- } else {
- return String.format(message, args);
- }
- }
-
- /**
- * Diagnostics failures return the exit code 41, "unauthorized".
- *
- * They have a category, initially for testing: the category can be
- * validated without having to match on the entire string.
- */
- public static class KerberosDiagsFailure extends ExitUtil.ExitException {
- private final String category;
-
- public KerberosDiagsFailure(String category, String message) {
- super(41, category + ": " + message);
- this.category = category;
- }
-
- public KerberosDiagsFailure(String category, String message, Object... args) {
- this(category, format(message, args));
- }
-
- public KerberosDiagsFailure(String category, Throwable throwable,
- String message, Object... args) {
- this(category, message, args);
- initCause(throwable);
- }
-
- public String getCategory() {
- return category;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java
deleted file mode 100644
index 6efa880..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import java.util.regex.Pattern;
-
-/**
- * Utility class to validate strings against a predefined pattern.
- */
-public class PatternValidator {
-
- public static final String E_INVALID_NAME =
- "Invalid name %s does not match the pattern pattern %s ";
- private final Pattern valid;
- private final String pattern;
-
- public PatternValidator(String pattern) {
- this.pattern = pattern;
- valid = Pattern.compile(pattern);
- }
-
- /**
- * Validate the name -restricting it to the set defined in
- * @param name name to validate
- * @throws IllegalArgumentException if not a valid name
- */
- public void validate(String name) {
- if (!matches(name)) {
- throw new IllegalArgumentException(
- String.format(E_INVALID_NAME, name, pattern));
- }
- }
-
- /**
- * Query to see if the pattern matches
- * @param name name to validate
- * @return true if the string matches the pattern
- */
- public boolean matches(String name) {
- return valid.matcher(name).matches();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
deleted file mode 100644
index 2dbf37f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * a scanner which can take an input string for a range or scan the lot.
- */
-public class PortScanner {
- private static Pattern NUMBER_RANGE = Pattern.compile("^(\\d+)\\s*-\\s*(\\d+)$");
- private static Pattern SINGLE_NUMBER = Pattern.compile("^\\d+$");
-
- private List<Integer> remainingPortsToCheck;
-
- public PortScanner() {
- }
-
- public void setPortRange(String input) throws BadConfigException {
- // first split based on commas
- Set<Integer> inputPorts= new TreeSet<Integer>();
- String[] ranges = input.split(",");
- for ( String range : ranges ) {
- if (range.trim().isEmpty()) {
- continue;
- }
- Matcher m = SINGLE_NUMBER.matcher(range.trim());
- if (m.find()) {
- inputPorts.add(Integer.parseInt(m.group()));
- continue;
- }
- m = NUMBER_RANGE.matcher(range.trim());
- if (m.find()) {
- String[] boundaryValues = m.group(0).split("-");
- int start = Integer.parseInt(boundaryValues[0].trim());
- int end = Integer.parseInt(boundaryValues[1].trim());
- if (end < start) {
- throw new BadConfigException("End of port range is before start: "
- + range + " in input: " + input);
- }
- for (int i = start; i < end + 1; i++) {
- inputPorts.add(i);
- }
- continue;
- }
- throw new BadConfigException("Bad port range: " + range + " in input: "
- + input);
- }
- if (inputPorts.size() == 0) {
- throw new BadConfigException("No ports found in range: " + input);
- }
- this.remainingPortsToCheck = new ArrayList<Integer>(inputPorts);
- }
-
- public List<Integer> getRemainingPortsToCheck() {
- return remainingPortsToCheck;
- }
-
- public int getAvailablePort() throws SliderException, IOException {
- if (remainingPortsToCheck != null) {
- return getAvailablePortViaPortArray();
- } else {
- return SliderUtils.getOpenPort();
- }
- }
-
- private int getAvailablePortViaPortArray() throws SliderException {
- boolean found = false;
- int availablePort = -1;
- Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator();
- while (portsToCheck.hasNext() && !found) {
- int portToCheck = portsToCheck.next();
- found = SliderUtils.isPortAvailable(portToCheck);
- if (found) {
- availablePort = portToCheck;
- portsToCheck.remove();
- }
- }
-
- if (availablePort < 0) {
- throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
- "No available ports found in configured range {}",
- remainingPortsToCheck);
- }
-
- return availablePort;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java
deleted file mode 100644
index 9d00b3c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.SerializationConfig;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.io.IOException;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-/**
- * JSON-serializable description of a published key-val configuration.
- *
- * The values themselves are not serialized in the external view; they have
- * to be served up by the far end
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class PublishedConfiguration {
-
- public String description;
- public long updated;
-
- public String updatedTime;
-
- public Map<String, String> entries = new HashMap<>();
-
- public PublishedConfiguration() {
- }
-
- /**
- * build an empty published configuration
- * @param description configuration description
- */
- public PublishedConfiguration(String description) {
- this.description = description;
- }
-
- /**
- * Build a configuration from the entries
- * @param description configuration description
- * @param entries entries to put
- */
- public PublishedConfiguration(String description,
- Iterable<Map.Entry<String, String>> entries) {
- this.description = description;
- putValues(entries);
- }
-
- /**
- * Build a published configuration, using the keys from keysource,
- * but resolving the values from the value source, via Configuration.get()
- * @param description configuration description
- * @param keysource source of keys
- * @param valuesource source of values
- */
- public PublishedConfiguration(String description,
- Iterable<Map.Entry<String, String>> keysource,
- Configuration valuesource) {
- this.description = description;
- putValues(ConfigHelper.resolveConfiguration(keysource, valuesource));
- }
-
-
- /**
- * Is the configuration empty. This means either that it has not
- * been given any values, or it is stripped down copy set down over the
- * wire.
- * @return true if it is empty
- */
- public boolean isEmpty() {
- return entries.isEmpty();
- }
-
-
- public void setUpdated(long updated) {
- this.updated = updated;
- this.updatedTime = new Date(updated).toString();
- }
-
- public long getUpdated() {
- return updated;
- }
-
- /**
- * Set the values from an iterable (this includes a Hadoop Configuration
- * and Java properties object).
- * Any existing value set is discarded
- * @param entries entries to put
- */
- public void putValues(Iterable<Map.Entry<String, String>> entries) {
- this.entries = new HashMap<String, String>();
- for (Map.Entry<String, String> entry : entries) {
- this.entries.put(entry.getKey(), entry.getValue());
- }
-
- }
-
- /**
- * Convert to Hadoop XML
- * @return the configuration as a Hadoop Configuratin
- */
- public Configuration asConfiguration() {
- Configuration conf = new Configuration(false);
- try {
- ConfigHelper.addConfigMap(conf, entries, "");
- } catch (BadConfigException e) {
- // triggered on a null value; switch to a runtime (and discard the stack)
- throw new RuntimeException(e.toString());
- }
- return conf;
- }
-
- public String asConfigurationXML() throws IOException {
- return ConfigHelper.toXml(asConfiguration());
- }
-
- /**
- * Convert values to properties
- * @return a property file
- */
- public Properties asProperties() {
- Properties props = new Properties();
- props.putAll(entries);
- return props;
- }
-
- /**
- * Return the values as json string
- * @return the JSON representation
- * @throws IOException marshalling failure
- */
- public String asJson() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
- String json = mapper.writeValueAsString(entries);
- return json;
- }
-
-
- /**
- * This makes a copy without the nested content -so is suitable
- * for returning as part of the list of a parent's values
- * @return the copy
- */
- public PublishedConfiguration shallowCopy() {
- PublishedConfiguration that = new PublishedConfiguration();
- that.description = this.description;
- that.updated = this.updated;
- that.updatedTime = this.updatedTime;
- return that;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("PublishedConfiguration{");
- sb.append("description='").append(description).append('\'');
- sb.append(" entries = ").append(entries.size());
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * Create an outputter for a given format
- * @param format format to use
- * @return an instance of output
- */
- public PublishedConfigurationOutputter createOutputter(ConfigFormat format) {
- return PublishedConfigurationOutputter.createOutputter(format, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
deleted file mode 100644
index 88ecf2c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-import org.yaml.snakeyaml.DumperOptions;
-import org.yaml.snakeyaml.DumperOptions.FlowStyle;
-import org.yaml.snakeyaml.Yaml;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.StringWriter;
-import java.util.Properties;
-
-/**
- * Output a published configuration
- */
-public abstract class PublishedConfigurationOutputter {
-
- private static final String COMMENTS = "Generated by Apache Slider";
-
- protected final PublishedConfiguration owner;
-
- protected PublishedConfigurationOutputter(PublishedConfiguration owner) {
- this.owner = owner;
- }
-
- /**
- * Save the config to a destination file, in the format of this outputter
- * @param dest destination file
- * @throws IOException
- */
-/* JDK7
- public void save(File dest) throws IOException {
- try(FileOutputStream out = new FileOutputStream(dest)) {
- save(out);
- out.close();
- }
- }
-*/
- public void save(File dest) throws IOException {
- FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
- }
-
- /**
- * Save the content. The default saves the asString() value
- * to the output stream
- * @param out output stream
- * @throws IOException
- */
- public void save(OutputStream out) throws IOException {
- IOUtils.write(asString(), out, Charsets.UTF_8);
- }
- /**
- * Convert to a string
- * @return the string form
- * @throws IOException
- */
- public abstract String asString() throws IOException;
-
- /**
- * Create an outputter for the chosen format
- * @param format format enumeration
- * @param owner owning config
- * @return the outputter
- */
-
- public static PublishedConfigurationOutputter createOutputter(ConfigFormat format,
- PublishedConfiguration owner) {
- Preconditions.checkNotNull(owner);
- switch (format) {
- case XML:
- case HADOOP_XML:
- return new XmlOutputter(owner);
- case PROPERTIES:
- return new PropertiesOutputter(owner);
- case JSON:
- return new JsonOutputter(owner);
- case ENV:
- return new EnvOutputter(owner);
- case TEMPLATE:
- return new TemplateOutputter(owner);
- case YAML:
- return new YamlOutputter(owner);
- default:
- throw new RuntimeException("Unsupported format :" + format);
- }
- }
-
- public static class XmlOutputter extends PublishedConfigurationOutputter {
-
-
- private final Configuration configuration;
-
- public XmlOutputter(PublishedConfiguration owner) {
- super(owner);
- configuration = owner.asConfiguration();
- }
-
- @Override
- public void save(OutputStream out) throws IOException {
- configuration.writeXml(out);
- }
-
- @Override
- public String asString() throws IOException {
- return ConfigHelper.toXml(configuration);
- }
-
- public Configuration getConfiguration() {
- return configuration;
- }
- }
-
- public static class PropertiesOutputter extends PublishedConfigurationOutputter {
-
- private final Properties properties;
-
- public PropertiesOutputter(PublishedConfiguration owner) {
- super(owner);
- properties = owner.asProperties();
- }
-
- @Override
- public void save(OutputStream out) throws IOException {
- properties.store(out, COMMENTS);
- }
-
-
- public String asString() throws IOException {
- StringWriter sw = new StringWriter();
- properties.store(sw, COMMENTS);
- return sw.toString();
- }
- }
-
-
- public static class JsonOutputter extends PublishedConfigurationOutputter {
-
- public JsonOutputter(PublishedConfiguration owner) {
- super(owner);
- }
-
- @Override
- public String asString() throws IOException {
- return owner.asJson();
- }
- }
-
-
- public static class EnvOutputter extends PublishedConfigurationOutputter {
-
- public EnvOutputter(PublishedConfiguration owner) {
- super(owner);
- }
-
- @Override
- public String asString() throws IOException {
- if (!owner.entries.containsKey("content")) {
- throw new IOException("Configuration has no content field and cannot " +
- "be retrieved as type 'env'");
- }
- String content = owner.entries.get("content");
- return ConfigUtils.replaceProps(owner.entries, content);
- }
- }
-
- public static class TemplateOutputter extends EnvOutputter {
- public TemplateOutputter(PublishedConfiguration owner) {
- super(owner);
- }
- }
-
- public static class YamlOutputter extends PublishedConfigurationOutputter {
-
- private final Yaml yaml;
-
- public YamlOutputter(PublishedConfiguration owner) {
- super(owner);
- DumperOptions options = new DumperOptions();
- options.setDefaultFlowStyle(FlowStyle.BLOCK);
- yaml = new Yaml(options);
- }
-
- public String asString() throws IOException {
- return yaml.dump(owner.entries);
- }
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
deleted file mode 100644
index 7712191..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ /dev/null
@@ -1,2783 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.slider.client;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.exceptions.NoRecordException;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.RegistryPathStatus;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.KerberosDiags;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
-import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.hadoop.yarn.util.Times;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-import org.apache.slider.api.SliderClusterProtocol;
-import org.apache.slider.api.proto.Messages;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.slider.api.types.NodeInformationList;
-import org.apache.slider.client.ipc.SliderClusterOperations;
-import org.apache.slider.common.Constants;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
-import org.apache.slider.common.params.ActionAMSuicideArgs;
-import org.apache.slider.common.params.ActionClientArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
-import org.apache.slider.common.params.ActionDiagnosticArgs;
-import org.apache.slider.common.params.ActionExistsArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
-import org.apache.slider.common.params.ActionFreezeArgs;
-import org.apache.slider.common.params.ActionKDiagArgs;
-import org.apache.slider.common.params.ActionKeytabArgs;
-import org.apache.slider.common.params.ActionKillContainerArgs;
-import org.apache.slider.common.params.ActionListArgs;
-import org.apache.slider.common.params.ActionLookupArgs;
-import org.apache.slider.common.params.ActionNodesArgs;
-import org.apache.slider.common.params.ActionRegistryArgs;
-import org.apache.slider.common.params.ActionResolveArgs;
-import org.apache.slider.common.params.ActionResourceArgs;
-import org.apache.slider.common.params.ActionStatusArgs;
-import org.apache.slider.common.params.ActionThawArgs;
-import org.apache.slider.common.params.ActionTokensArgs;
-import org.apache.slider.common.params.ActionUpgradeArgs;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.client.params.CommonArgs;
-import org.apache.slider.common.tools.ConfigHelper;
-import org.apache.slider.common.tools.Duration;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.common.tools.SliderVersionInfo;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.NoSuchNodeException;
-import org.apache.slider.core.exceptions.NotFoundException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
-import org.apache.slider.core.exceptions.UsageException;
-import org.apache.slider.core.launch.ClasspathConstructor;
-import org.apache.slider.core.launch.CredentialUtils;
-import org.apache.slider.core.launch.JavaCommandLineBuilder;
-import org.apache.slider.core.launch.SerializedApplicationReport;
-import org.apache.slider.core.main.RunService;
-import org.apache.slider.core.persist.ApplicationReportSerDeser;
-import org.apache.slider.core.persist.JsonSerDeser;
-import org.apache.slider.core.registry.SliderRegistryUtils;
-import org.apache.slider.core.registry.YarnAppListClient;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.docstore.PublishedExports;
-import org.apache.slider.core.registry.docstore.PublishedExportsOutputter;
-import org.apache.slider.core.registry.docstore.PublishedExportsSet;
-import org.apache.slider.core.registry.retrieve.RegistryRetriever;
-import org.apache.slider.core.zk.BlockingZKWatcher;
-import org.apache.slider.core.zk.ZKIntegration;
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.provider.ProviderUtils;
-import org.apache.slider.server.appmaster.rpc.RpcBinder;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
-import org.apache.hadoop.yarn.service.client.ClientAMProxy;
-import org.apache.hadoop.yarn.service.ServiceMaster;
-import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.data.ACL;
-import org.codehaus.jackson.map.PropertyNamingStrategy;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.io.OutputStreamWriter;
-import java.io.PrintStream;
-import java.io.PrintWriter;
-import java.net.InetSocketAddress;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.*;
-import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
-import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.*;
-import static org.apache.slider.common.tools.SliderUtils.*;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
-
-/**
- * Client service for Slider
- */
-
-public class SliderClient extends AbstractSliderLaunchedService implements RunService,
- SliderExitCodes, SliderKeys, ErrorStrings, SliderClientAPI {
- private static final Logger log = LoggerFactory.getLogger(SliderClient.class);
- public static final String E_MUST_BE_A_VALID_JSON_FILE
- = "Invalid configuration. Must be a valid json file.";
- public static final String E_INVALID_INSTALL_LOCATION
- = "A valid install location must be provided for the client.";
- public static final String E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE
- = "Unable to read supplied package file";
- public static final String E_INVALID_APPLICATION_PACKAGE_LOCATION
- = "A valid application package location required.";
- public static final String E_INVALID_INSTALL_PATH = "Install path is not a valid directory";
- public static final String E_INSTALL_PATH_DOES_NOT_EXIST = "Install path does not exist";
- public static final String E_INVALID_APPLICATION_TYPE_NAME
- = "A valid application type name is required (e.g. HBASE).";
- public static final String E_USE_REPLACEPKG_TO_OVERWRITE = "Use --replacepkg to overwrite.";
- public static final String E_PACKAGE_DOES_NOT_EXIST = "Package does not exist";
- public static final String E_NO_ZOOKEEPER_QUORUM = "No Zookeeper quorum defined";
- public static final String E_NO_RESOURCE_MANAGER = "No valid Resource Manager address provided";
- public static final String E_PACKAGE_EXISTS = "Package exists";
- private static PrintStream clientOutputStream = System.out;
- private static final JsonSerDeser<Application> jsonSerDeser =
- new JsonSerDeser<Application>(Application.class,
- PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
-
- // value should not be changed without updating string find in slider.py
- private static final String PASSWORD_PROMPT = "Enter password for";
-
- private ClientArgs serviceArgs;
- public ApplicationId applicationId;
-
- private String deployedClusterName;
- /**
- * Cluster operations against the deployed cluster -will be null
- * if no bonding has yet taken place
- */
- private SliderClusterOperations sliderClusterOperations;
-
- protected SliderFileSystem sliderFileSystem;
- private YarnRPC rpc;
- /**
- * Yarn client service
- */
- private SliderYarnClientImpl yarnClient;
- private YarnAppListClient yarnAppListClient;
- ResourceCalculator calculator;
- /**
- * The YARN registry service
- */
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private RegistryOperations registryOperations;
-
- private static EnumSet<YarnApplicationState> terminatedStates =
- EnumSet.of(FINISHED, FAILED, KILLED);
- private static EnumSet<YarnApplicationState> waitingStates =
- EnumSet.of(NEW, NEW_SAVING, SUBMITTED, RUNNING);
-
- /**
- * Constructor
- */
- public SliderClient() {
- super("Slider Client");
- new HdfsConfiguration();
- new YarnConfiguration();
- }
-
- /**
- * This is called <i>Before serviceInit is called</i>
- * @param config the initial configuration build up by the
- * service launcher.
- * @param args argument list list of arguments passed to the command line
- * after any launcher-specific commands have been stripped.
- * @return the post-binding configuration to pass to the <code>init()</code>
- * operation.
- * @throws Exception
- */
- @Override
- public Configuration bindArgs(Configuration config, String... args) throws Exception {
- config = super.bindArgs(config, args);
- serviceArgs = new ClientArgs(args);
- serviceArgs.parse();
- // yarn-ify
- YarnConfiguration yarnConfiguration = new YarnConfiguration(config);
- return patchConfiguration(yarnConfiguration);
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- Configuration clientConf = loadSliderClientXML();
- ConfigHelper.mergeConfigurations(conf, clientConf, SLIDER_CLIENT_XML, true);
- serviceArgs.applyDefinitions(conf);
- serviceArgs.applyFileSystemBinding(conf);
- AbstractActionArgs coreAction = serviceArgs.getCoreAction();
- // init security with our conf
- if (!coreAction.disableSecureLogin() && isHadoopClusterSecure(conf)) {
- forceLogin();
- initProcessSecurity(conf);
- }
- if (coreAction.getHadoopServicesRequired()) {
- initHadoopBinding();
- }
- rpc = YarnRPC.create(conf);
- super.serviceInit(conf);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
- }
-
- /**
- * Launched service execution. This runs {@link #exec()}
- * then catches some exceptions and converts them to exit codes
- * @return an exit code
- * @throws Throwable
- */
- @Override
- public int runService() throws Throwable {
- try {
- return exec();
- } catch (FileNotFoundException | PathNotFoundException nfe) {
- throw new NotFoundException(nfe, nfe.toString());
- }
- }
-
- /**
- * Execute the command line
- * @return an exit code
- * @throws Throwable on a failure
- */
- public int exec() throws Throwable {
-
- // choose the action
- String action = serviceArgs.getAction();
- if (isUnset(action)) {
- throw new SliderException(EXIT_USAGE, serviceArgs.usage());
- }
-
- int exitCode = EXIT_SUCCESS;
- String clusterName = serviceArgs.getClusterName();
- // actions
-
- switch (action) {
- case ACTION_AM_SUICIDE:
- exitCode = actionAmSuicide(clusterName,
- serviceArgs.getActionAMSuicideArgs());
- break;
-
- case ACTION_BUILD:
- exitCode = actionBuild(getApplicationFromArgs(clusterName,
- serviceArgs.getActionBuildArgs()));
- break;
-
- case ACTION_CLIENT:
- exitCode = actionClient(serviceArgs.getActionClientArgs());
- break;
-
- case ACTION_CREATE:
- actionCreate(getApplicationFromArgs(clusterName,
- serviceArgs.getActionCreateArgs()));
- break;
-
- case ACTION_DEPENDENCY:
- exitCode = actionDependency(serviceArgs.getActionDependencyArgs());
- break;
-
- case ACTION_DESTROY:
- actionDestroy(clusterName);
- break;
-
- case ACTION_DIAGNOSTICS:
- exitCode = actionDiagnostic(serviceArgs.getActionDiagnosticArgs());
- break;
-
- case ACTION_EXISTS:
- exitCode = actionExists(clusterName,
- serviceArgs.getActionExistsArgs());
- break;
-
- case ACTION_FLEX:
- actionFlex(clusterName, serviceArgs.getActionFlexArgs());
- break;
-
- case ACTION_STOP:
- actionStop(clusterName, serviceArgs.getActionFreezeArgs());
- break;
-
- case ACTION_HELP:
- log.info(serviceArgs.usage());
- break;
-
- case ACTION_KDIAG:
- exitCode = actionKDiag(serviceArgs.getActionKDiagArgs());
- break;
-
- case ACTION_KILL_CONTAINER:
- exitCode = actionKillContainer(clusterName,
- serviceArgs.getActionKillContainerArgs());
- break;
-
- case ACTION_KEYTAB:
- exitCode = actionKeytab(serviceArgs.getActionKeytabArgs());
- break;
-
- case ACTION_LIST:
- exitCode = actionList(clusterName, serviceArgs.getActionListArgs());
- break;
-
- case ACTION_LOOKUP:
- exitCode = actionLookup(serviceArgs.getActionLookupArgs());
- break;
-
- case ACTION_NODES:
- exitCode = actionNodes("", serviceArgs.getActionNodesArgs());
- break;
-
- case ACTION_REGISTRY:
- exitCode = actionRegistry(serviceArgs.getActionRegistryArgs());
- break;
-
- case ACTION_RESOLVE:
- exitCode = actionResolve(serviceArgs.getActionResolveArgs());
- break;
-
- case ACTION_RESOURCE:
- exitCode = actionResource(serviceArgs.getActionResourceArgs());
- break;
-
- case ACTION_STATUS:
- exitCode = actionStatus(clusterName, serviceArgs.getActionStatusArgs());
- break;
-
- case ACTION_START:
- exitCode = actionStart(clusterName, serviceArgs.getActionThawArgs());
- break;
-
- case ACTION_TOKENS:
- exitCode = actionTokens(serviceArgs.getActionTokenArgs());
- break;
-
- case ACTION_UPDATE:
- exitCode = actionUpdate(clusterName, serviceArgs.getActionUpdateArgs());
- break;
-
- case ACTION_UPGRADE:
- exitCode = actionUpgrade(clusterName, serviceArgs.getActionUpgradeArgs());
- break;
-
- case ACTION_VERSION:
- exitCode = actionVersion();
- break;
-
- default:
- throw new SliderException(EXIT_UNIMPLEMENTED,
- "Unimplemented: " + action);
- }
-
- return exitCode;
- }
-
- /**
- * Perform everything needed to init the hadoop binding.
- * This assumes that the service is already in inited or started state
- * @throws IOException
- * @throws SliderException
- */
- protected void initHadoopBinding() throws IOException, SliderException {
- // validate the client
- validateSliderClientEnvironment(null);
- //create the YARN client
- yarnClient = new SliderYarnClientImpl();
- yarnClient.init(getConfig());
- if (getServiceState() == STATE.STARTED) {
- yarnClient.start();
- }
- addService(yarnClient);
- yarnAppListClient =
- new YarnAppListClient(yarnClient, getUsername(), getConfig());
- // create the filesystem
- sliderFileSystem = new SliderFileSystem(getConfig());
- }
-
- /**
- * Delete the zookeeper node associated with the calling user and the cluster
- * TODO: YARN registry operations
- **/
- @VisibleForTesting
- public boolean deleteZookeeperNode(String clusterName) throws YarnException, IOException {
- String user = getUsername();
- String zkPath = ZKIntegration.mkClusterPath(user, clusterName);
- Exception e = null;
- try {
- ZKIntegration client = getZkClient(clusterName, user);
- if (client != null) {
- if (client.exists(zkPath)) {
- log.info("Deleting zookeeper path {}", zkPath);
- }
- client.deleteRecursive(zkPath);
- return true;
- }
- } catch (InterruptedException | BadConfigException | KeeperException ex) {
- e = ex;
- }
- if (e != null) {
- log.warn("Unable to recursively delete zk node {}", zkPath, e);
- }
-
- return false;
- }
-
- /**
- * Create the zookeeper node associated with the calling user and the cluster
- *
- * @param clusterName slider application name
- * @param nameOnly should the name only be created (i.e. don't create ZK node)
- * @return the path, using the policy implemented in
- * {@link ZKIntegration#mkClusterPath(String, String)}
- * @throws YarnException
- * @throws IOException
- */
- @VisibleForTesting
- public String createZookeeperNode(String clusterName, Boolean nameOnly) throws YarnException, IOException {
- try {
- return createZookeeperNodeInner(clusterName, nameOnly);
- } catch (KeeperException.NodeExistsException e) {
- return null;
- } catch (KeeperException e) {
- return null;
- } catch (InterruptedException e) {
- throw new InterruptedIOException(e.toString());
- }
- }
-
- /**
- * Create the zookeeper node associated with the calling user and the cluster
- * -throwing exceptions on any failure
- * @param clusterName cluster name
- * @param nameOnly create the path, not the node
- * @return the path, using the policy implemented in
- * {@link ZKIntegration#mkClusterPath(String, String)}
- * @throws YarnException
- * @throws IOException
- * @throws KeeperException
- * @throws InterruptedException
- */
- @VisibleForTesting
- public String createZookeeperNodeInner(String clusterName, Boolean nameOnly)
- throws YarnException, IOException, KeeperException, InterruptedException {
- String user = getUsername();
- String zkPath = ZKIntegration.mkClusterPath(user, clusterName);
- if (nameOnly) {
- return zkPath;
- }
- ZKIntegration client = getZkClient(clusterName, user);
- if (client != null) {
- // set up the permissions. This must be done differently on a secure cluster from an insecure
- // one
- List<ACL> zkperms = new ArrayList<>();
- if (UserGroupInformation.isSecurityEnabled()) {
- zkperms.add(new ACL(ZooDefs.Perms.ALL, ZooDefs.Ids.AUTH_IDS));
- zkperms.add(new ACL(ZooDefs.Perms.READ, ZooDefs.Ids.ANYONE_ID_UNSAFE));
- } else {
- zkperms.add(new ACL(ZooDefs.Perms.ALL, ZooDefs.Ids.ANYONE_ID_UNSAFE));
- }
- client.createPath(zkPath, "",
- zkperms,
- CreateMode.PERSISTENT);
- return zkPath;
- } else {
- return null;
- }
- }
-
- /**
- * Gets a zookeeper client, returns null if it cannot connect to zookeeper
- **/
- protected ZKIntegration getZkClient(String clusterName, String user) throws YarnException {
- String registryQuorum = lookupZKQuorum();
- ZKIntegration client = null;
- try {
- BlockingZKWatcher watcher = new BlockingZKWatcher();
- client = ZKIntegration.newInstance(registryQuorum, user, clusterName, true, false, watcher,
- ZKIntegration.SESSION_TIMEOUT);
- boolean fromCache = client.init();
- if (!fromCache) {
- watcher.waitForZKConnection(2 * 1000);
- }
- } catch (InterruptedException e) {
- client = null;
- log.warn("Interrupted - unable to connect to zookeeper quorum {}",
- registryQuorum, e);
- } catch (IOException e) {
- log.warn("Unable to connect to zookeeper quorum {}", registryQuorum, e);
- }
- return client;
- }
-
- /**
- * Keep this signature for backward compatibility with
- * force=true by default.
- */
- @Override
- public int actionDestroy(String appName)
- throws YarnException, IOException {
- validateClusterName(appName);
- verifyNoLiveApp(appName, "Destroy");
- Path appDir = sliderFileSystem.buildClusterDirPath(appName);
- FileSystem fs = sliderFileSystem.getFileSystem();
- if (fs.exists(appDir)) {
- if (fs.delete(appDir, true)) {
- log.info("Successfully deleted application dir for " + appName);
- } else {
- String message =
- "Failed to delete application + " + appName + " at: " + appDir;
- log.info(message);
- throw new YarnException(message);
- }
- }
- if (!deleteZookeeperNode(appName)) {
- String message =
- "Failed to cleanup cleanup application " + appName + " in zookeeper";
- log.warn(message);
- throw new YarnException(message);
- }
-
- //TODO clean registry?
- String registryPath = SliderRegistryUtils.registryPathForInstance(
- appName);
- try {
- getRegistryOperations().delete(registryPath, true);
- } catch (IOException e) {
- log.warn("Error deleting registry entry {}: {} ", registryPath, e, e);
- } catch (SliderException e) {
- log.warn("Error binding to registry {} ", e, e);
- }
-
- log.info("Destroyed cluster {}", appName);
- return EXIT_SUCCESS;
- }
-
-
- @Override
- public int actionAmSuicide(String clustername,
- ActionAMSuicideArgs args) throws YarnException, IOException {
- SliderClusterOperations clusterOperations =
- createClusterOperations(clustername);
- clusterOperations.amSuicide(args.message, args.exitcode, args.waittime);
- return EXIT_SUCCESS;
- }
-
- private Application getApplicationFromArgs(String clusterName,
- AbstractClusterBuildingActionArgs args) throws IOException {
- File file = args.getAppDef();
- Path filePath = new Path(file.getAbsolutePath());
- log.info("Loading app definition from: " + filePath);
- Application application =
- jsonSerDeser.load(FileSystem.getLocal(getConfig()), filePath);
- if(args.lifetime > 0) {
- application.setLifetime(args.lifetime);
- }
- application.setName(clusterName);
- return application;
- }
-
- public int actionBuild(Application application) throws YarnException,
- IOException {
- Path appDir = checkAppNotExistOnHdfs(application);
- ServiceApiUtil.validateAndResolveApplication(application,
- sliderFileSystem, getConfig());
- persistApp(appDir, application);
- deployedClusterName = application.getName();
- return EXIT_SUCCESS;
- }
-
- public ApplicationId actionCreate(Application application)
- throws IOException, YarnException {
- String appName = application.getName();
- validateClusterName(appName);
- ServiceApiUtil.validateAndResolveApplication(application,
- sliderFileSystem, getConfig());
- verifyNoLiveApp(appName, "Create");
- Path appDir = checkAppNotExistOnHdfs(application);
-
- ApplicationId appId = submitApp(application);
- application.setId(appId.toString());
- // write app definition on to hdfs
- persistApp(appDir, application);
- return appId;
- //TODO deal with registry
- }
-
- private ApplicationId submitApp(Application app)
- throws IOException, YarnException {
- String appName = app.getName();
- Configuration conf = getConfig();
- Path appRootDir = sliderFileSystem.buildClusterDirPath(app.getName());
- deployedClusterName = appName;
-
- YarnClientApplication yarnApp = yarnClient.createApplication();
- ApplicationSubmissionContext submissionContext =
- yarnApp.getApplicationSubmissionContext();
- ServiceApiUtil.validateCompResourceSize(
- yarnApp.getNewApplicationResponse().getMaximumResourceCapability(),
- app);
-
- applicationId = submissionContext.getApplicationId();
- submissionContext.setKeepContainersAcrossApplicationAttempts(true);
- if (app.getLifetime() > 0) {
- Map<ApplicationTimeoutType, Long> appTimeout = new HashMap<>();
- appTimeout.put(ApplicationTimeoutType.LIFETIME, app.getLifetime());
- submissionContext.setApplicationTimeouts(appTimeout);
- }
- submissionContext.setMaxAppAttempts(conf.getInt(KEY_AM_RESTART_LIMIT, 2));
-
- Map<String, LocalResource> localResources = new HashMap<>();
-
- // copy local slideram-log4j.properties to hdfs and add to localResources
- boolean hasSliderAMLog4j =
- addAMLog4jResource(appName, conf, localResources);
- // copy jars to hdfs and add to localResources
- addJarResource(appName, localResources);
- // add keytab if in secure env
- addKeytabResourceIfSecure(sliderFileSystem, localResources, conf, appName);
- printLocalResources(localResources);
-
- //TODO SliderAMClientProvider#copyEnvVars
- //TODO localResource putEnv
-
- Map<String, String> env = addAMEnv(conf);
-
- // create AM CLI
- String cmdStr =
- buildCommandLine(appName, conf, appRootDir, hasSliderAMLog4j);
-
- //TODO set log aggregation context
- //TODO set retry window
- submissionContext.setResource(Resource.newInstance(
- conf.getLong(KEY_AM_RESOURCE_MEM, DEFAULT_KEY_AM_RESOURCE_MEM), 1));
- submissionContext.setQueue(conf.get(KEY_YARN_QUEUE, app.getQueue()));
- submissionContext.setApplicationName(appName);
- submissionContext.setApplicationType(SliderKeys.APP_TYPE);
- Set<String> appTags =
- AbstractClientProvider.createApplicationTags(appName, null, null);
- if (!appTags.isEmpty()) {
- submissionContext.setApplicationTags(appTags);
- }
- ContainerLaunchContext amLaunchContext =
- Records.newRecord(ContainerLaunchContext.class);
- amLaunchContext.setCommands(Collections.singletonList(cmdStr));
- amLaunchContext.setEnvironment(env);
- amLaunchContext.setLocalResources(localResources);
- addCredentialsIfSecure(conf, amLaunchContext);
- submissionContext.setAMContainerSpec(amLaunchContext);
- submitApplication(submissionContext);
- return submissionContext.getApplicationId();
- }
-
- @VisibleForTesting
- public ApplicationId submitApplication(ApplicationSubmissionContext context)
- throws IOException, YarnException {
- return yarnClient.submitApplication(context);
- }
-
- private void printLocalResources(Map<String, LocalResource> map) {
- log.info("Added LocalResource for localization: ");
- StringBuilder builder = new StringBuilder();
- for (Map.Entry<String, LocalResource> entry : map.entrySet()) {
- builder.append(entry.getKey()).append(" -> ")
- .append(entry.getValue().getResource().getFile())
- .append(System.lineSeparator());
- }
- log.info(builder.toString());
- }
-
- private void addCredentialsIfSecure(Configuration conf,
- ContainerLaunchContext amLaunchContext) throws IOException {
- if (UserGroupInformation.isSecurityEnabled()) {
- // pick up oozie credentials
- Credentials credentials =
- CredentialUtils.loadTokensFromEnvironment(System.getenv(), conf);
- if (credentials == null) {
- // nothing from oozie, so build up directly
- credentials = new Credentials(
- UserGroupInformation.getCurrentUser().getCredentials());
- CredentialUtils.addRMRenewableFSDelegationTokens(conf,
- sliderFileSystem.getFileSystem(), credentials);
- } else {
- log.info("Using externally supplied credentials to launch AM");
- }
- amLaunchContext.setTokens(CredentialUtils.marshallCredentials(credentials));
- }
- }
-
- private String buildCommandLine(String appName, Configuration conf,
- Path appRootDir, boolean hasSliderAMLog4j) throws BadConfigException {
- JavaCommandLineBuilder CLI = new JavaCommandLineBuilder();
- CLI.forceIPv4().headless();
- //TODO CLI.setJVMHeap
- //TODO CLI.addJVMOPTS
- if (hasSliderAMLog4j) {
- CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME);
- CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
- }
- CLI.add(ServiceMaster.class.getCanonicalName());
- CLI.add(ACTION_CREATE, appName);
- //TODO debugAM CLI.add(Arguments.ARG_DEBUG)
- CLI.add(Arguments.ARG_CLUSTER_URI, new Path(appRootDir, appName + ".json"));
- // InetSocketAddress rmSchedulerAddress = getRmSchedulerAddress(conf);
-// String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress);
-// CLI.add(Arguments.ARG_RM_ADDR, rmAddr);
- // pass the registry binding
- CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
- RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
- CLI.addMandatoryConfOption(conf, RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
- if(isHadoopClusterSecure(conf)) {
- //TODO Is this required ??
- // if the cluster is secure, make sure that
- // the relevant security settings go over
- CLI.addConfOption(conf, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
- }
-// // copy over any/all YARN RM client values, in case the server-side XML conf file
-// // has the 0.0.0.0 address
-// CLI.addConfOptions(conf, YarnConfiguration.RM_ADDRESS,
-// YarnConfiguration.RM_CLUSTER_ID, YarnConfiguration.RM_HOSTNAME,
-// YarnConfiguration.RM_PRINCIPAL);
-
- // write out the path output
- CLI.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
- String cmdStr = CLI.build();
- log.info("Completed setting up app master command: {}", cmdStr);
- return cmdStr;
- }
-
- private Map<String, String> addAMEnv(Configuration conf)
- throws IOException {
- Map<String, String> env = new HashMap<>();
- ClasspathConstructor classpath =
- buildClasspath(SliderKeys.SUBMITTED_CONF_DIR, "lib",
- sliderFileSystem, getUsingMiniMRCluster());
- env.put("CLASSPATH", classpath.buildClasspath());
- env.put("LANG", "en_US.UTF-8");
- env.put("LC_ALL", "en_US.UTF-8");
- env.put("LANGUAGE", "en_US.UTF-8");
- String jaas = System.getenv(HADOOP_JAAS_DEBUG);
- if (jaas != null) {
- env.put(HADOOP_JAAS_DEBUG, jaas);
- }
- if (!UserGroupInformation.isSecurityEnabled()) {
- String userName = UserGroupInformation.getCurrentUser().getUserName();
- log.info("Run as user " + userName);
- // HADOOP_USER_NAME env is used by UserGroupInformation when log in
- // This env makes AM run as this user
- env.put("HADOOP_USER_NAME", userName);
- }
- env.putAll(getAmLaunchEnv(conf));
- log.info("AM env: \n{}", stringifyMap(env));
- return env;
- }
-
- protected Path addJarResource(String appName,
- Map<String, LocalResource> localResources)
- throws IOException, SliderException {
- Path libPath = sliderFileSystem.buildClusterDirPath(appName);
- ProviderUtils
- .addProviderJar(localResources, ServiceMaster.class, SLIDER_JAR,
- sliderFileSystem, libPath, "lib", false);
- Path dependencyLibTarGzip = sliderFileSystem.getDependencyTarGzip();
- if (sliderFileSystem.isFile(dependencyLibTarGzip)) {
- log.info("Loading lib tar from " + sliderFileSystem.getFileSystem()
- .getScheme() + ": " + dependencyLibTarGzip);
- SliderUtils.putAmTarGzipAndUpdate(localResources, sliderFileSystem);
- } else {
- String[] libs = SliderUtils.getLibDirs();
- log.info("Loading dependencies from local file system: " + Arrays
- .toString(libs));
- for (String libDirProp : libs) {
- ProviderUtils
- .addAllDependencyJars(localResources, sliderFileSystem, libPath,
- "lib", libDirProp);
- }
- }
- return libPath;
- }
-
- private boolean addAMLog4jResource(String appName, Configuration conf,
- Map<String, LocalResource> localResources)
- throws IOException, BadClusterStateException {
- boolean hasSliderAMLog4j = false;
- String hadoopConfDir =
- System.getenv(ApplicationConstants.Environment.HADOOP_CONF_DIR.name());
- if (hadoopConfDir != null) {
- File localFile =
- new File(hadoopConfDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
- if (localFile.exists()) {
- Path localFilePath = createLocalPath(localFile);
- Path appDirPath = sliderFileSystem.buildClusterDirPath(appName);
- Path remoteConfPath =
- new Path(appDirPath, SliderKeys.SUBMITTED_CONF_DIR);
- Path remoteFilePath =
- new Path(remoteConfPath, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
- copy(conf, localFilePath, remoteFilePath);
- LocalResource localResource = sliderFileSystem
- .createAmResource(remoteConfPath, LocalResourceType.FILE);
- localResources.put(localFilePath.getName(), localResource);
- hasSliderAMLog4j = true;
- }
- }
- return hasSliderAMLog4j;
- }
-
- private Path checkAppNotExistOnHdfs(Application application)
- throws IOException, SliderException {
- Path appDir = sliderFileSystem.buildClusterDirPath(application.getName());
- sliderFileSystem.verifyDirectoryNonexistent(
- new Path(appDir, application.getName() + ".json"));
- return appDir;
- }
-
- private Path checkAppExistOnHdfs(String appName)
- throws IOException, SliderException {
- Path appDir = sliderFileSystem.buildClusterDirPath(appName);
- sliderFileSystem.verifyPathExists(
- new Path(appDir, appName + ".json"));
- return appDir;
- }
-
- private void persistApp(Path appDir, Application application)
- throws IOException, SliderException {
- FsPermission appDirPermission = new FsPermission("750");
- sliderFileSystem.createWithPermissions(appDir, appDirPermission);
- Path appJson = new Path(appDir, application.getName() + ".json");
- jsonSerDeser
- .save(sliderFileSystem.getFileSystem(), appJson, application, true);
- log.info(
- "Persisted application " + application.getName() + " at " + appJson);
- }
-
- private void addKeytabResourceIfSecure(SliderFileSystem fileSystem,
- Map<String, LocalResource> localResource, Configuration conf,
- String appName) throws IOException, BadConfigException {
- if (!UserGroupInformation.isSecurityEnabled()) {
- return;
- }
- String keytabPreInstalledOnHost =
- conf.get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
- if (StringUtils.isEmpty(keytabPreInstalledOnHost)) {
- String amKeytabName =
- conf.get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
- String keytabDir = conf.get(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
- Path keytabPath =
- fileSystem.buildKeytabPath(keytabDir, amKeytabName, appName);
- if (fileSystem.getFileSystem().exists(keytabPath)) {
- LocalResource keytabRes =
- fileSystem.createAmResource(keytabPath, LocalResourceType.FILE);
- localResource
- .put(SliderKeys.KEYTAB_DIR + "/" + amKeytabName, keytabRes);
- log.info("Adding AM keytab on hdfs: " + keytabPath);
- } else {
- log.warn("No keytab file was found at {}.", keytabPath);
- if (conf.getBoolean(KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) {
- throw new BadConfigException("No keytab file was found at %s.",
- keytabPath);
- } else {
- log.warn("The AM will be "
- + "started without a kerberos authenticated identity. "
- + "The application is therefore not guaranteed to remain "
- + "operational beyond 24 hours.");
- }
- }
- }
- }
-
- @Override
- public int actionUpgrade(String clustername, ActionUpgradeArgs upgradeArgs)
- throws YarnException, IOException {
- //TODO
- return 0;
- }
-
- @Override
- public int actionKeytab(ActionKeytabArgs keytabInfo)
- throws YarnException, IOException {
- if (keytabInfo.install) {
- return actionInstallKeytab(keytabInfo);
- } else if (keytabInfo.delete) {
- return actionDeleteKeytab(keytabInfo);
- } else if (keytabInfo.list) {
- return actionListKeytab(keytabInfo);
- } else {
- throw new BadCommandArgumentsException(
- "Keytab option specified not found.\n"
- + CommonArgs.usage(serviceArgs, ACTION_KEYTAB));
- }
- }
-
- private int actionListKeytab(ActionKeytabArgs keytabInfo) throws IOException {
- String folder = keytabInfo.folder != null ? keytabInfo.folder : StringUtils.EMPTY;
- Path keytabPath = sliderFileSystem.buildKeytabInstallationDirPath(folder);
- RemoteIterator<LocatedFileStatus> files =
- sliderFileSystem.getFileSystem().listFiles(keytabPath, true);
- log.info("Keytabs:");
- while (files.hasNext()) {
- log.info("\t" + files.next().getPath().toString());
- }
-
- return EXIT_SUCCESS;
- }
-
- private int actionDeleteKeytab(ActionKeytabArgs keytabInfo)
- throws BadCommandArgumentsException, IOException {
- if (StringUtils.isEmpty(keytabInfo.folder)) {
- throw new BadCommandArgumentsException(
- "A valid destination keytab sub-folder name is required (e.g. 'security').\n"
- + CommonArgs.usage(serviceArgs, ACTION_KEYTAB));
- }
-
- if (StringUtils.isEmpty(keytabInfo.keytab)) {
- throw new BadCommandArgumentsException("A keytab name is required.");
- }
-
- Path pkgPath = sliderFileSystem.buildKeytabInstallationDirPath(keytabInfo.folder);
-
- Path fileInFs = new Path(pkgPath, keytabInfo.keytab );
- log.info("Deleting keytab {}", fileInFs);
- FileSystem sfs = sliderFileSystem.getFileSystem();
- require(sfs.exists(fileInFs), "No keytab to delete found at %s",
- fileInFs.toUri());
- sfs.delete(fileInFs, false);
-
- return EXIT_SUCCESS;
- }
-
- private int actionInstallKeytab(ActionKeytabArgs keytabInfo)
- throws BadCommandArgumentsException, IOException {
- Path srcFile = null;
- require(isSet(keytabInfo.folder),
- "A valid destination keytab sub-folder name is required (e.g. 'security').\n"
- + CommonArgs.usage(serviceArgs, ACTION_KEYTAB));
-
- requireArgumentSet(Arguments.ARG_KEYTAB, keytabInfo.keytab);
- File keytabFile = new File(keytabInfo.keytab);
- require(keytabFile.isFile(),
- "Unable to access supplied keytab file at %s", keytabFile.getAbsolutePath());
- srcFile = new Path(keytabFile.toURI());
-
- Path pkgPath = sliderFileSystem.buildKeytabInstallationDirPath(keytabInfo.folder);
- FileSystem sfs = sliderFileSystem.getFileSystem();
- sfs.mkdirs(pkgPath);
- sfs.setPermission(pkgPath, new FsPermission(
- FsAction.ALL, FsAction.NONE, FsAction.NONE));
-
- Path fileInFs = new Path(pkgPath, srcFile.getName());
- log.info("Installing keytab {} at {} and overwrite is {}.",
- srcFile, fileInFs, keytabInfo.overwrite);
- require(!(sfs.exists(fileInFs) && !keytabInfo.overwrite),
- "Keytab exists at %s. Use --overwrite to overwrite.", fileInFs.toUri());
-
- sfs.copyFromLocalFile(false, keytabInfo.overwrite, srcFile, fileInFs);
- sfs.setPermission(fileInFs,
- new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE));
-
- return EXIT_SUCCESS;
- }
-
- @Override
- public int actionResource(ActionResourceArgs resourceInfo)
- throws YarnException, IOException {
- if (resourceInfo.help) {
- actionHelp(ACTION_RESOURCE);
- return EXIT_SUCCESS;
- } else if (resourceInfo.install) {
- return actionInstallResource(resourceInfo);
- } else if (resourceInfo.delete) {
- return actionDeleteResource(resourceInfo);
- } else if (resourceInfo.list) {
- return actionListResource(resourceInfo);
- } else {
- throw new BadCommandArgumentsException(
- "Resource option specified not found.\n"
- + CommonArgs.usage(serviceArgs, ACTION_RESOURCE));
- }
- }
-
- private int actionListResource(ActionResourceArgs resourceInfo) throws IOException {
- String folder = resourceInfo.folder != null ? resourceInfo.folder : StringUtils.EMPTY;
- Path path = sliderFileSystem.buildResourcePath(folder);
- RemoteIterator<LocatedFileStatus> files =
- sliderFileSystem.getFileSystem().listFiles(path, true);
- log.info("Resources:");
- while (files.hasNext()) {
- log.info("\t" + files.next().getPath().toString());
- }
-
- return EXIT_SUCCESS;
- }
-
- private int actionDeleteResource(ActionResourceArgs resourceInfo)
- throws BadCommandArgumentsException, IOException {
- if (StringUtils.isEmpty(resourceInfo.resource)) {
- throw new BadCommandArgumentsException("A file name is required.");
- }
-
- Path fileInFs;
- if (resourceInfo.folder == null) {
- fileInFs = sliderFileSystem.buildResourcePath(resourceInfo.resource);
- } else {
- fileInFs = sliderFileSystem.buildResourcePath(resourceInfo.folder,
- resourceInfo.resource);
- }
-
- log.info("Deleting resource {}", fileInFs);
- FileSystem sfs = sliderFileSystem.getFileSystem();
- require(sfs.exists(fileInFs), "No resource to delete found at %s", fileInFs.toUri());
- sfs.delete(fileInFs, true);
-
- return EXIT_SUCCESS;
- }
-
- private int actionInstallResource(ActionResourceArgs resourceInfo)
- throws BadCommandArgumentsException, IOException {
- Path srcFile = null;
- String folder = resourceInfo.folder != null ? resourceInfo.folder : StringUtils.EMPTY;
-
- requireArgumentSet(Arguments.ARG_RESOURCE, resourceInfo.resource);
- File file = new File(resourceInfo.resource);
- require(file.isFile() || file.isDirectory(),
- "Unable to access supplied file at %s", file.getAbsolutePath());
-
- File[] files;
- if (file.isDirectory()) {
- files = file.listFiles();
- } else {
- files = new File[] { file };
- }
-
- Path pkgPath = sliderFileSystem.buildResourcePath(folder);
- FileSystem sfs = sliderFileSystem.getFileSystem();
-
- if (!sfs.exists(pkgPath)) {
- sfs.mkdirs(pkgPath);
- sfs.setPermission(pkgPath, new FsPermission(
- FsAction.ALL, FsAction.NONE, FsAction.NONE));
- } else {
- require(sfs.isDirectory(pkgPath), "Specified folder %s exists and is " +
- "not a directory", folder);
- }
-
- if (files != null) {
- for (File f : files) {
- srcFile = new Path(f.toURI());
-
- Path fileInFs = new Path(pkgPath, srcFile.getName());
- log.info("Installing file {} at {} and overwrite is {}.",
- srcFile, fileInFs, resourceInfo.overwrite);
- require(!(sfs.exists(fileInFs) && !resourceInfo.overwrite),
- "File exists at %s. Use --overwrite to overwrite.", fileInFs.toUri());
-
- sfs.copyFromLocalFile(false, resourceInfo.overwrite, srcFile, fileInFs);
- sfs.setPermission(fileInFs,
- new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE));
- }
- }
-
- return EXIT_SUCCESS;
- }
-
- @Override
- public int actionClient(ActionClientArgs clientInfo) throws
- YarnException,
- IOException {
- if (clientInfo.install) {
- // TODO implement client install
- throw new UnsupportedOperationException("Client install not yet " +
- "supported");
- } else {
- throw new BadCommandArgumentsException(
- "Only install, keystore, and truststore commands are supported for the client.\n"
- + CommonArgs.usage(serviceArgs, ACTION_CLIENT));
-
- }
- }
-
- @Override
- public int actionUpdate(String clustername,
- AbstractClusterBuildingActionArgs buildInfo) throws
- YarnException, IOException {
- if (buildInfo.lifetime > 0) {
- updateLifetime(clustername, buildInfo.lifetime);
- } else {
- //TODO upgrade
- }
- return EXIT_SUCCESS;
- }
-
- public String updateLifetime(String appName, long lifetime)
- throws YarnException, IOException {
- EnumSet<YarnApplicationState> appStates = EnumSet.range(NEW, RUNNING);
- ApplicationReport report = findInstance(appName, appStates);
- if (report == null) {
- throw new YarnException("Application not found for " + appName);
- }
- ApplicationId appId = report.getApplicationId();
- log.info("Updating lifetime of an application: appName = " + appName
- + ", appId = " + appId+ ", lifetime = " + lifetime);
- Map<ApplicationTimeoutType, String> map = new HashMap<>();
- String newTimeout =
- Times.formatISO8601(System.currentTimeMillis() + lifetime * 1000);
- map.put(ApplicationTimeoutType.LIFETIME, newTimeout);
- UpdateApplicationTimeoutsRequest request =
- UpdateApplicationTimeoutsRequest.newInstance(appId, map);
- yarnClient.updateApplicationTimeouts(request);
- log.info("Successfully updated lifetime for an application: appName = "
- + appName + ", appId = " + appId
- + ". New expiry time in ISO8601 format is " + newTimeout);
- return newTimeout;
- }
-
- protected Map<String, String> getAmLaunchEnv(Configuration config) {
- String sliderAmLaunchEnv = config.get(KEY_AM_LAUNCH_ENV);
- log.debug("{} = {}", KEY_AM_LAUNCH_ENV, sliderAmLaunchEnv);
- // Multiple env variables can be specified with a comma (,) separator
- String[] envs = StringUtils.isEmpty(sliderAmLaunchEnv) ? null
- : sliderAmLaunchEnv.split(",");
- if (ArrayUtils.isEmpty(envs)) {
- return Collections.emptyMap();
- }
- Map<String, String> amLaunchEnv = new HashMap<>();
- for (String env : envs) {
- if (StringUtils.isNotEmpty(env)) {
- // Each env name/value is separated by equals sign (=)
- String[] tokens = env.split("=");
- if (tokens != null && tokens.length == 2) {
- String envKey = tokens[0];
- String envValue = tokens[1];
- for (Map.Entry<String, String> placeholder : generatePlaceholderKeyValueMap(
- env).entrySet()) {
- if (StringUtils.isNotEmpty(placeholder.getValue())) {
- envValue = envValue.replaceAll(
- Pattern.quote(placeholder.getKey()), placeholder.getValue());
- }
- }
- if (Shell.WINDOWS) {
- envValue = "%" + envKey + "%;" + envValue;
- } else {
- envValue = "$" + envKey + ":" + envValue;
- }
- log.info("Setting AM launch env {}={}", envKey, envValue);
- amLaunchEnv.put(envKey, envValue);
- }
- }
- }
- return amLaunchEnv;
- }
-
- protected Map<String, String> generatePlaceholderKeyValueMap(String env) {
- String PLACEHOLDER_PATTERN = "\\$\\{[^{]+\\}";
- Pattern placeholderPattern = Pattern.compile(PLACEHOLDER_PATTERN);
- Matcher placeholderMatcher = placeholderPattern.matcher(env);
- Map<String, String> placeholderKeyValueMap = new HashMap<>();
- if (placeholderMatcher.find()) {
- String placeholderKey = placeholderMatcher.group();
- String systemKey = placeholderKey
- .substring(2, placeholderKey.length() - 1).toUpperCase(Locale.ENGLISH)
- .replaceAll("\\.", "_");
- String placeholderValue = getSystemEnv(systemKey);
- log.debug("Placeholder {}={}", placeholderKey, placeholderValue);
- placeholderKeyValueMap.put(placeholderKey, placeholderValue);
- }
- return placeholderKeyValueMap;
- }
-
- /**
- * verify that a live cluster isn't there
- * @param clustername cluster name
- * @param action
- * @throws SliderException with exit code EXIT_CLUSTER_LIVE
- * if a cluster of that name is either live or starting up.
- */
- public void verifyNoLiveApp(String clustername, String action) throws
- IOException,
- YarnException {
- List<ApplicationReport> existing = findAllLiveInstances(clustername);
-
- if (!existing.isEmpty()) {
- throw new SliderException(EXIT_APPLICATION_IN_USE,
- action +" failed for "
- + clustername
- + ": "
- + E_CLUSTER_RUNNING + " :" +
- existing.get(0));
- }
- }
-
- public String getUsername() throws IOException {
- return RegistryUtils.currentUser();
- }
-
- /**
- * Get the name of any deployed cluster
- * @return the cluster name
- */
- public String getDeployedClusterName() {
- return deployedClusterName;
- }
-
- /**
- * ask if the client is using a mini MR cluster
- * @return true if they are
- */
- private boolean getUsingMiniMRCluster() {
- return getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER,
- false);
- }
-
-
- /**
- * List Slider instances belonging to a specific user with a specific app
- * name and within a set of app states.
- * @param user user: "" means all users, null means "default"
- * @param appName name of the application set as a tag
- * @param appStates a set of states the applications should be in
- * @return a possibly empty list of Slider AMs
- */
- public List<ApplicationReport> listSliderInstances(String user,
- String appName, EnumSet<YarnApplicationState> appStates)
- throws YarnException, IOException {
- return yarnAppListClient.listInstances(user, appName, appStates);
- }
-
- /**
- * A basic list action to list live instances
- * @param clustername cluster name
- * @return success if the listing was considered successful
- * @throws IOException
- * @throws YarnException
- */
- public int actionList(String clustername) throws IOException, YarnException {
- ActionListArgs args = new ActionListArgs();
- args.live = true;
- return actionList(clustername, args);
- }
-
- /**
- * Implement the list action.
- * @param clustername List out specific instance name
- * @param args Action list arguments
- * @return 0 if one or more entries were listed
- * @throws IOException
- * @throws YarnException
- * @throws UnknownApplicationInstanceException if a specific instance
- * was named but it was not found
- */
- @Override
- public int actionList(String clustername, ActionListArgs args)
- throws IOException, YarnException {
- Set<ApplicationReport> appInstances = getApplicationList(clustername, args);
- if (!appInstances.isEmpty()) {
- return EXIT_SUCCESS;
- } else {
- return EXIT_FALSE;
- }
- }
-
- /**
- * Retrieve a list of application instances satisfying the query criteria.
- *
- * @param clustername
- * List out specific instance name (set null for all)
- * @param args
- * Action list arguments
- * @return the list of application names which satisfies the list criteria
- * @throws IOException
- * @throws YarnException
- * @throws UnknownApplicationInstanceException
- * if a specific instance was named but it was not found
- */
- public Set<ApplicationReport> getApplicationList(String clustername,
- ActionListArgs args) throws IOException, YarnException {
- if (args.help) {
- actionHelp(ACTION_LIST);
- // the above call throws an exception so the return is not really required
- return Collections.emptySet();
- }
- boolean live = args.live;
- String state = args.state;
- boolean listContainers = args.containers;
- boolean verbose = args.verbose;
- String version = args.version;
- Set<String> components = args.components;
-
- if (live && !state.isEmpty()) {
- throw new BadCommandArgumentsException(
- Arguments.ARG_LIVE + " and " + Arguments.ARG_STATE + " are exclusive");
- }
- if (listContainers && isUnset(clustername)) {
- throw new BadCommandArgumentsException(
- "Should specify an application instance with "
- + Arguments.ARG_CONTAINERS);
- }
- // specifying both --version and --components with --containers is okay
- if (StringUtils.isNotEmpty(version) && !listContainers) {
- throw new BadCommandArgumentsException(Arguments.ARG_VERSION
- + " can be specified only with " + Arguments.ARG_CONTAINERS);
- }
- if (!components.isEmpty() && !listContainers) {
- throw new BadCommandArgumentsException(Arguments.ARG_COMPONENTS
- + " can be specified only with " + Arguments.ARG_CONTAINERS);
- }
-
- // flag to indicate only services in a specific state are to be listed
- boolean listOnlyInState = live || !state.isEmpty();
-
- YarnApplicationState min, max;
- if (live) {
- min = NEW;
- max = RUNNING;
- } else if (!state.isEmpty()) {
- YarnApplicationState stateVal = extractYarnApplicationState(state);
- min = max = stateVal;
- } else {
- min = NEW;
- max = KILLED;
- }
- // get the complete list of persistent instances
- Map<String, Path> persistentInstances = sliderFileSystem.listPersistentInstances();
-
- if (persistentInstances.isEmpty() && isUnset(clustername)) {
- // an empty listing is a success if no cluster was named
- log.debug("No application instances found");
- return Collections.emptySet();
- }
-
- // and those the RM knows about
- EnumSet<YarnApplicationState> appStates = EnumSet.range(min, max);
- List<ApplicationReport> instances = listSliderInstances(null, clustername,
- appStates);
- sortApplicationsByMostRecent(instances);
- Map<String, ApplicationReport> reportMap =
- buildApplicationReportMap(instances, min, max);
- log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}",
- persistentInstances.size(),
- instances.size(),
- min, max,
- reportMap.size() );
-
- List<ContainerInformation> containers = null;
- if (isSet(clustername)) {
- // only one instance is expected
- // resolve the persistent value
- Path persistent = persistentInstances.get(clustername);
- if (persistent == null) {
- throw unknownClusterException(clustername);
- }
- // create a new map with only that instance in it.
- // this restricts the output of results to this instance
- persistentInstances = new HashMap<>();
- persistentInstances.put(clustername, persistent);
- if (listContainers) {
- containers = getContainers(clustername);
- }
- }
-
- // at this point there is either the entire list or a stripped down instance
- Set<ApplicationReport> listedInstances = new HashSet<ApplicationReport>();
- for (String name : persistentInstances.keySet()) {
- ApplicationReport report = reportMap.get(name);
- if (!listOnlyInState || report != null) {
- // list the details if all were requested, or the filtering contained
- // a report
- listedInstances.add(report);
- // containers will be non-null when only one instance is requested
- String details = instanceDetailsToString(name, report,
- containers, version, components, verbose);
- print(details);
- }
- }
-
- return listedInstances;
- }
-
- public List<ContainerInformation> getContainers(String name)
- throws YarnException, IOException {
- SliderClusterOperations clusterOps = new SliderClusterOperations(
- bondToCluster(name));
- try {
- return clusterOps.getContainers();
- } catch (NoSuchNodeException e) {
- throw new BadClusterStateException(
- "Containers not found for application instance %s", name);
- }
- }
-
-
- /**
- * Extract the state of a Yarn application --state argument
- * @param state state argument
- * @return the application state
- * @throws BadCommandArgumentsException if the argument did not match
- * any known state
- */
- private YarnApplicationState extractYarnApplicationState(String state) throws
- BadCommandArgumentsException {
- YarnApplicationState stateVal;
- try {
- stateVal = YarnApplicationState.valueOf(state.toUpperCase(Locale.ENGLISH));
- } catch (IllegalArgumentException e) {
- throw new BadCommandArgumentsException("Unknown state: " + state);
-
- }
- return stateVal;
- }
-
- /**
- * Is an application active: accepted or running
- * @param report the application report
- * @return true if it is running or scheduled to run.
- */
- public boolean isApplicationActive(ApplicationReport report) {
- return report.getYarnApplicationState() == RUNNING
- || report.getYarnApplicationState() == YarnApplicationState.ACCEPTED;
- }
-
- /**
- * Implement the islive action: probe for a cluster of the given name existing
- * @return exit code
- */
-
- @Override
- @VisibleForTesting
- public int actionFlex(String appName, ActionFlexArgs args)
- throws YarnException, IOException {
- Map<String, Long> componentCounts = new HashMap<>(args.getComponentMap()
- .size());
- for (Entry<String, String> entry : args.getComponentMap().entrySet()) {
- long numberOfContainers = Long.parseLong(entry.getValue());
- componentCounts.put(entry.getKey(), numberOfContainers);
- }
- // throw usage exception if no changes proposed
- if (componentCounts.size() == 0) {
- actionHelp(ACTION_FLEX);
- }
- flex(appName, componentCounts);
- return EXIT_SUCCESS;
- }
-
- @Override
- public int actionExists(String name, boolean checkLive) throws YarnException, IOException {
- ActionExistsArgs args = new ActionExistsArgs();
- args.live = checkLive;
- return actionExists(name, args);
- }
-
- public int actionExists(String name, ActionExistsArgs args) throws YarnException, IOException {
- validateClusterName(name);
- boolean checkLive = args.live;
- log.debug("actionExists({}, {}, {})", name, checkLive, args.state);
-
- //initial probe for a cluster in the filesystem
- Path clusterDirectory = sliderFileSystem.buildClusterDirPath(name);
- if (!sliderFileSystem.getFileSystem().exists(clusterDirectory)) {
- throw unknownClusterException(name);
- }
- String state = args.state;
- if (!checkLive && isUnset(state)) {
- log.info("Application {} exists", name);
- return EXIT_SUCCESS;
- }
-
- //test for liveness/state
- boolean inDesiredState = false;
- ApplicationReport instance;
- instance = findInstance(name);
- if (instance == null) {
- log.info("Application {} not running", name);
- return EXIT_FALSE;
- }
- if (checkLive) {
- // the app exists, check that it is not in any terminated state
- YarnApplicationState appstate = instance.getYarnApplicationState();
- log.debug(" current app state = {}", appstate);
- inDesiredState = appstate.ordinal() < FINISHED.ordinal();
- } else {
- // scan for instance in single --state state
- state = state.toUpperCase(Locale.ENGLISH);
- YarnApplicationState desiredState = extractYarnApplicationState(state);
- List<ApplicationReport> userInstances = yarnClient
- .listDeployedInstances("", EnumSet.of(desiredState), name);
- ApplicationReport foundInstance =
- yarnClient.findAppInInstanceList(userInstances, name, desiredState);
- if (foundInstance != null) {
- // found in selected state: success
- inDesiredState = true;
- // mark this as the instance to report
- instance = foundInstance;
- }
- }
-
- OnDemandReportStringifier report =
- new OnDemandReportStringifier(instance);
- if (!inDesiredState) {
- //cluster in the list of apps but not running
- log.info("Application {} found but is in wrong state {}", name,
- instance.getYarnApplicationState());
- log.debug("State {}", report);
- return EXIT_FALSE;
- } else {
- log.debug("Application instance is in desired state");
- log.info("Application {} is {}\n{}", name,
- instance.getYarnApplicationState(), report);
- return EXIT_SUCCESS;
- }
- }
-
-
- @Override
- public int actionKillContainer(String name,
- ActionKillContainerArgs args) throws YarnException, IOException {
- String id = args.id;
- if (isUnset(id)) {
- throw new BadCommandArgumentsException("Missing container id");
- }
- log.info("killingContainer {}:{}", name, id);
- SliderClusterOperations clusterOps =
- new SliderClusterOperations(bondToCluster(name));
- try {
- clusterOps.killContainer(id);
- } catch (NoSuchNodeException e) {
- throw new BadClusterStateException("Container %s not found in cluster %s",
- id, name);
- }
- return EXIT_SUCCESS;
- }
-
- /**
- * Find an instance of an application belonging to the current user.
- * @param appname application name
- * @return the app report or null if none is found
- * @throws YarnException YARN issues
- * @throws IOException IO problems
- */
- public ApplicationReport findInstance(String appname)
- throws YarnException, IOException {
- return findInstance(appname, null);
- }
-
- /**
- * Find an instance of an application belonging to the current user and in
- * specific app states.
- * @param appname application name
- * @param appStates app states in which the application should be in
- * @return the app report or null if none is found
- * @throws YarnException YARN issues
- * @throws IOException IO problems
- */
- public ApplicationReport findInstance(String appname,
- EnumSet<YarnApplicationState> appStates)
- throws YarnException, IOException {
- return yarnAppListClient.findInstance(appname, appStates);
- }
-
- /**
- * find all live instances of a specific app -if there is >1 in the cluster,
- * this returns them all. State should be running or less
- * @param appname application name
- * @return the list of all matching application instances
- */
- private List<ApplicationReport> findAllLiveInstances(String appname)
- throws YarnException, IOException {
-
- return yarnAppListClient.findAllLiveInstances(appname);
- }
-
- /**
- * Connect to a Slider AM
- * @param app application report providing the details on the application
- * @return an instance
- * @throws YarnException
- * @throws IOException
- */
- private SliderClusterProtocol connect(ApplicationReport app)
- throws YarnException, IOException {
-
- try {
- return RpcBinder.getProxy(getConfig(),
- yarnClient.getRmClient(),
- app,
- Constants.CONNECT_TIMEOUT,
- Constants.RPC_TIMEOUT);
- } catch (InterruptedException e) {
- throw new SliderException(SliderExitCodes.EXIT_TIMED_OUT,
- e,
- "Interrupted waiting for communications with the Slider AM");
- }
- }
-
- @Override
- @VisibleForTesting
- public int actionStatus(String clustername, ActionStatusArgs statusArgs)
- throws YarnException, IOException {
- if (statusArgs.lifetime) {
- queryAndPrintLifetime(clustername);
- return EXIT_SUCCESS;
- }
-
- Application application = getApplication(clustername);
- String outfile = statusArgs.getOutput();
- if (outfile == null) {
- log.info(application.toString());
- } else {
- jsonSerDeser.save(application, new File(statusArgs.getOutput()));
- }
- return EXIT_SUCCESS;
- }
-
- @Override
- public Application actionStatus(String clustername)
- throws YarnException, IOException {
- return getApplication(clustername);
- }
-
- private void queryAndPrintLifetime(String appName)
- throws YarnException, IOException {
- ApplicationReport appReport = findInstance(appName);
- if (appReport == null) {
- throw new YarnException("No application found for " + appName);
- }
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- PrintWriter timeoutStr =
- new PrintWriter(new OutputStreamWriter(baos, Charset.forName("UTF-8")));
- try {
- ApplicationTimeout lifetime = appReport.getApplicationTimeouts()
- .get(ApplicationTimeoutType.LIFETIME);
- if (lifetime.getRemainingTime() == -1L) {
- timeoutStr.append(appName + " has no lifetime configured.");
- } else {
- timeoutStr.append("\t" + ApplicationTimeoutType.LIFETIME);
- timeoutStr.print(" expires at : " + lifetime.getExpiryTime());
- timeoutStr.println(
- ".\tRemaining Time : " + lifetime.getRemainingTime() + " seconds");
- }
- System.out.println(baos.toString("UTF-8"));
- } finally {
- timeoutStr.close();
- }
- }
-
- @Override
- public int actionVersion() {
- SliderVersionInfo.loadAndPrintVersionInfo(log);
- return EXIT_SUCCESS;
- }
-
- @Override
- public int actionStop(String appName, ActionFreezeArgs freezeArgs)
- throws YarnException, IOException {
- validateClusterName(appName);
- ApplicationReport app = findInstance(appName);
- if (app == null) {
- throw new ApplicationNotFoundException(
- "Application " + appName + " doesn't exist in RM.");
- }
-
- if (terminatedStates.contains(app.getYarnApplicationState())) {
- log.info("Application {} is already in a terminated state {}", appName,
- app.getYarnApplicationState());
- return EXIT_SUCCESS;
- }
-
- try {
- SliderClusterProtocol appMaster = connect(app);
- Messages.StopClusterRequestProto r =
- Messages.StopClusterRequestProto.newBuilder()
- .setMessage(freezeArgs.message).build();
- appMaster.stopCluster(r);
- log.info("Application " + appName + " is being gracefully stopped...");
- long startTime = System.currentTimeMillis();
- int pollCount = 0;
- while (true) {
- Thread.sleep(200);
- ApplicationReport report =
- yarnClient.getApplicationReport(app.getApplicationId());
- if (terminatedStates.contains(report.getYarnApplicationState())) {
- log.info("Application " + appName + " is stopped.");
- break;
- }
- // kill after 10 seconds.
- if ((System.currentTimeMillis() - startTime) > 10000) {
- log.info("Stop operation timeout stopping, forcefully kill the app "
- + appName);
- yarnClient
- .killApplication(app.getApplicationId(), freezeArgs.message);
- break;
- }
- if (++pollCount % 10 == 0) {
- log.info("Waiting for application " + appName + " to be stopped.");
- }
- }
- } catch (IOException | YarnException | InterruptedException e) {
- log.info("Failed to stop " + appName
- + " gracefully, forcefully kill the app.");
- yarnClient.killApplication(app.getApplicationId(), freezeArgs.message);
- }
- return EXIT_SUCCESS;
- }
-
- @Override
- public int actionStart(String appName, ActionThawArgs thaw)
- throws YarnException, IOException {
- validateClusterName(appName);
- Path appDir = checkAppExistOnHdfs(appName);
- Application application = ServiceApiUtil.loadApplication(sliderFileSystem,
- appName);
- ServiceApiUtil.validateAndResolveApplication(application,
- sliderFileSystem, getConfig());
- // see if it is actually running and bail out;
- verifyNoLiveApp(appName, "Thaw");
- ApplicationId appId = submitApp(application);
- application.setId(appId.toString());
- // write app definition on to hdfs
- persistApp(appDir, application);
- return 0;
- }
-
- public Map<String, Long> flex(String appName, Map<String, Long>
- componentCounts) throws YarnException, IOException {
- validateClusterName(appName);
- Application persistedApp = ServiceApiUtil.loadApplication(sliderFileSystem,
- appName);
- Map<String, Long> original = new HashMap<>(componentCounts.size());
- for (Component persistedComp : persistedApp.getComponents()) {
- String name = persistedComp.getName();
- if (componentCounts.containsKey(persistedComp.getName())) {
- original.put(name, persistedComp.getNumberOfContainers());
- persistedComp.setNumberOfContainers(componentCounts.get(name));
- }
- }
- if (original.size() < componentCounts.size()) {
- componentCounts.keySet().removeAll(original.keySet());
- throw new YarnException("Components " + componentCounts.keySet()
- + " do not exist in app definition.");
- }
- jsonSerDeser
- .save(sliderFileSystem.getFileSystem(), ServiceApiUtil.getAppJsonPath(
- sliderFileSystem, appName), persistedApp, true);
- log.info("Updated app definition file for components " + componentCounts
- .keySet());
-
- ApplicationReport instance = findInstance(appName);
- if (instance != null) {
- log.info("Flexing running app " + appName);
- SliderClusterProtocol appMaster = connect(instance);
- SliderClusterOperations clusterOps =
- new SliderClusterOperations(appMaster);
- clusterOps.flex(componentCounts);
- for (Entry<String, Long> componentCount : componentCounts.entrySet()) {
- log.info(
- "Application name = " + appName + ", Component name = " +
- componentCount.getKey() + ", number of containers updated " +
- "from " + original.get(componentCount.getKey()) + " to " +
- componentCount.getValue());
- }
- } else {
- String message = "Application " + appName + "does not exist in RM. ";
- throw new YarnException(message);
- }
- return original;
- }
-
- /**
- * Connect to a live cluster and get its current state
- *
- * @param appName the cluster name
- * @return its description
- */
- @VisibleForTesting
- public Application getApplication(String appName)
- throws YarnException, IOException {
- validateClusterName(appName);
- SliderClusterOperations clusterOperations =
- createClusterOperations(appName);
- return clusterOperations.getApplication();
- }
-
- private ClientAMProtocol connectToAM(String appName)
- throws IOException, YarnException {
- if (applicationId == null) {
- Application persistedApp = ServiceApiUtil.loadApplication(sliderFileSystem,
- appName);
- if (persistedApp == null) {
- throw new YarnException("Application " + appName
- + " doesn't exist on hdfs. Please check if the app exists in RM");
- }
- applicationId = ApplicationId.fromString(persistedApp.getId());
- }
- // Wait until app becomes running.
- long startTime = System.currentTimeMillis();
- int pollCount = 0;
- ApplicationReport appReport = null;
- while (true) {
- appReport = yarnClient.getApplicationReport(applicationId);
- YarnApplicationState state = appReport.getYarnApplicationState();
- if (state == RUNNING) {
- break;
- }
- if (terminatedStates.contains(state)) {
- throw new YarnException(
- "Failed to getStatus " + applicationId + ": " + appReport
- .getDiagnostics());
- }
- long elapsedMillis = System.currentTimeMillis() - startTime;
- // if over 5 min, quit
- if (elapsedMillis >= 300000) {
- throw new YarnException(
- "Timed out while waiting for application " + applicationId
- + " to be running");
- }
-
- if (++pollCount % 10 == 0) {
- log.info("Waiting for application {} to be running, current state is {}",
- applicationId, state);
- }
- try {
- Thread.sleep(3000);
- } catch (InterruptedException ie) {
- String msg =
- "Interrupted while waiting for application " + applicationId
- + " to be running.";
- throw new YarnException(msg, ie);
- }
- }
-
- // Make the connection
- InetSocketAddress address = NetUtils
- .createSocketAddrForHost(appReport.getHost(), appReport.getRpcPort());
- return ClientAMProxy
- .createProxy(getConfig(), ClientAMProtocol.class,
- UserGroupInformation.getCurrentUser(), rpc, address);
- }
-
- public Application getStatus(String appName)
- throws IOException, YarnException {
- ClientAMProtocol proxy = connectToAM(appName);
- GetStatusResponseProto response =
- proxy.getStatus(GetStatusRequestProto.newBuilder().build());
- Application app = jsonSerDeser.fromJson(response.getStatus());
- return app;
- }
-
-
- /**
- * Bond to a running cluster
- * @param clustername cluster name
- * @return the AM RPC client
- * @throws SliderException if the cluster is unkown
- */
- private SliderClusterProtocol bondToCluster(String clustername) throws
- YarnException,
- IOException {
- if (clustername == null) {
- throw unknownClusterException("(undefined)");
- }
- ApplicationReport instance = findInstance(clustername,
- SliderUtils.getAllLiveAppStates());
- if (null == instance) {
- throw unknownClusterException(clustername);
- }
- return connect(instance);
- }
-
- /**
- * Create a cluster operations instance against a given cluster
- * @param clustername cluster name
- * @return a bonded cluster operations instance
- * @throws YarnException YARN issues
- * @throws IOException IO problems
- */
- private SliderClusterOperations createClusterOperations(String clustername) throws
- YarnException,
- IOException {
- SliderClusterProtocol sliderAM = bondToCluster(clustername);
- return new SliderClusterOperations(sliderAM);
- }
-
- /**
- * Generate an exception for an unknown cluster
- * @param clustername cluster name
- * @return an exception with text and a relevant exit code
- */
- public UnknownApplicationInstanceException unknownClusterException(String clustername) {
- return UnknownApplicationInstanceException.unknownInstance(clustername);
- }
-
- @Override
- public String toString() {
- return "Slider Client in state " + getServiceState()
- + " and Slider Application Instance " + deployedClusterName;
- }
-
- /**
- * Get all YARN applications
- * @return a possibly empty list
- * @throws YarnException
- * @throws IOException
- */
- @VisibleForTesting
- public List<ApplicationReport> getApplications()
- throws YarnException, IOException {
- return yarnClient.getApplications();
- }
-
- @Override
- public int actionResolve(ActionResolveArgs args)
- throws YarnException, IOException {
- // as this is an API entry point, validate
- // the arguments
- args.validate();
- String path = SliderRegistryUtils.resolvePath(args.path);
- ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal();
- try {
- if (args.list) {
- File destDir = args.destdir;
- if (destDir != null && !destDir.exists() && !destDir.mkdirs()) {
- throw new IOException("Failed to create directory: " + destDir);
- }
-
-
- Map<String, ServiceRecord> recordMap;
- Map<String, RegistryPathStatus> znodes;
- try {
- znodes = statChildren(registryOperations, path);
- recordMap = extractServiceRecords(registryOperations,
- path,
- znodes.values());
- } catch (PathNotFoundException e) {
- // treat the root directory as if if is always there
-
- if ("/".equals(path)) {
- znodes = new HashMap<>(0);
- recordMap = new HashMap<>(0);
- } else {
- throw e;
- }
- }
- // subtract all records from the znodes map to get pure directories
- log.info("Entries: {}", znodes.size());
-
- for (String name : znodes.keySet()) {
- println(" " + name);
- }
- println("");
-
- log.info("Service records: {}", recordMap.size());
- for (Entry<String, ServiceRecord> recordEntry : recordMap.entrySet()) {
- String name = recordEntry.getKey();
- ServiceRecord instance = recordEntry.getValue();
- String json = serviceRecordMarshal.toJson(instance);
- if (destDir == null) {
- println(name);
- println(json);
- } else {
- String filename = RegistryPathUtils.lastPathEntry(name) + ".json";
- File jsonFile = new File(destDir, filename);
- write(jsonFile, serviceRecordMarshal.toBytes(instance));
- }
- }
- } else {
- // resolve single entry
- ServiceRecord instance = resolve(path);
- File outFile = args.out;
- if (args.destdir != null) {
- outFile = new File(args.destdir, RegistryPathUtils.lastPathEntry(path));
- }
- if (outFile != null) {
- write(outFile, serviceRecordMarshal.toBytes(instance));
- } else {
- println(serviceRecordMarshal.toJson(instance));
- }
- }
- } catch (PathNotFoundException | NoRecordException e) {
- // no record at this path
- throw new NotFoundException(e, path);
- }
- return EXIT_SUCCESS;
- }
-
- @Override
- public int actionRegistry(ActionRegistryArgs registryArgs) throws
- YarnException,
- IOException {
- // as this is also a test entry point, validate
- // the arguments
- registryArgs.validate();
- try {
- if (registryArgs.list) {
- actionRegistryList(registryArgs);
- } else if (registryArgs.listConf) {
- // list the configurations
- actionRegistryListConfigsYarn(registryArgs);
- } else if (registryArgs.listExports) {
- // list the exports
- actionRegistryListExports(registryArgs);
- } else if (isSet(registryArgs.getConf)) {
- // get a configuration
- PublishedConfiguration publishedConfiguration =
- actionRegistryGetConfig(registryArgs);
- outputConfig(publishedConfiguration, registryArgs);
- } else if (isSet(registryArgs.getExport)) {
- // get a export group
- PublishedExports publishedExports =
- actionRegistryGetExport(registryArgs);
- outputExport(publishedExports, registryArgs);
- } else {
- // it's an unknown command
- log.info(CommonArgs.usage(serviceArgs, ACTION_DIAGNOSTICS));
- return EXIT_USAGE;
- }
-// JDK7
- } catch (FileNotFoundException e) {
- log.info("{}", e.toString());
- log.debug("{}", e, e);
- return EXIT_NOT_FOUND;
- } catch (PathNotFoundException e) {
- log.info("{}", e.toString());
- log.debug("{}", e, e);
- return EXIT_NOT_FOUND;
- }
- return EXIT_SUCCESS;
- }
-
- /**
- * Registry operation
- *
- * @param registryArgs registry Arguments
- * @return the instances (for tests)
- * @throws YarnException YARN problems
- * @throws IOException Network or other problems
- */
- @VisibleForTesting
- public Collection<ServiceRecord> actionRegistryList(
- ActionRegistryArgs registryArgs)
- throws YarnException, IOException {
- String serviceType = registryArgs.serviceType;
- String name = registryArgs.name;
- RegistryOperations operations = getRegistryOperations();
- Collection<ServiceRecord> serviceRecords;
- if (StringUtils.isEmpty(name)) {
- String path = serviceclassPath(currentUser(), serviceType);
-
- try {
- Map<String, ServiceRecord> recordMap =
- listServiceRecords(operations, path);
- if (recordMap.isEmpty()) {
- throw new UnknownApplicationInstanceException(
- "No applications registered under " + path);
- }
- serviceRecords = recordMap.values();
- } catch (PathNotFoundException e) {
- throw new NotFoundException(path, e);
- }
- } else {
- ServiceRecord instance = lookupServiceRecord(registryArgs);
- serviceRecords = new ArrayList<>(1);
- serviceRecords.add(instance);
- }
-
- for (ServiceRecord serviceRecord : serviceRecords) {
- logInstance(serviceRecord, registryArgs.verbose);
- }
- return serviceRecords;
- }
-
- @Override
- public int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) {
- try {
- if (diagnosticArgs.client) {
- actionDiagnosticClient(diagnosticArgs);
- } else if (diagnosticArgs.application) {
- // TODO print configs of application - get from AM
- } else if (diagnosticArgs.yarn) {
- // This method prints yarn nodes info and yarn configs.
- // We can just use yarn node CLI instead which is much more richful
- // for yarn configs, this method reads local config which is only client
- // config not cluster configs.
-// actionDiagnosticYarn(diagnosticArgs);
- } else if (diagnosticArgs.credentials) {
- // actionDiagnosticCredentials internall only runs a bare 'klist' command...
- // IMHO, the user can just run klist on their own with extra options supported, don't
- // actually see the point of this method.
-// actionDiagnosticCredentials();
- } else if (diagnosticArgs.all) {
- actionDiagnosticAll(diagnosticArgs);
- } else if (diagnosticArgs.level) {
- // agent is removed
- } else {
- // it's an unknown option
- log.info(CommonArgs.usage(serviceArgs, ACTION_DIAGNOSTICS));
- return EXIT_USAGE;
- }
- } catch (Exception e) {
- log.error(e.toString());
- return EXIT_FALSE;
- }
- return EXIT_SUCCESS;
- }
-
- private void actionDiagnosticAll(ActionDiagnosticArgs diagnosticArgs)
- throws IOException, YarnException {
- // assign application name from param to each sub diagnostic function
- actionDiagnosticClient(diagnosticArgs);
- // actionDiagnosticSlider only prints the agent location on hdfs,
- // which is invalid now.
- // actionDiagnosticCredentials only runs 'klist' command, IMHO, the user
- // can just run klist on its own with extra options supported, don't
- // actually see the point of this method.
- }
-
- private void actionDiagnosticClient(ActionDiagnosticArgs diagnosticArgs)
- throws SliderException, IOException {
- try {
- String currentCommandPath = getCurrentCommandPath();
- SliderVersionInfo.loadAndPrintVersionInfo(log);
- String clientConfigPath = getClientConfigPath();
- String jdkInfo = getJDKInfo();
- println("The slider command path: %s", currentCommandPath);
- println("The slider-client.xml used by current running command path: %s",
- clientConfigPath);
- println(jdkInfo);
-
- // security info
- Configuration config = getConfig();
- if (isHadoopClusterSecure(config)) {
- println("Hadoop Cluster is secure");
- println("Login user is %s", UserGroupInformation.getLoginUser());
- println("Current user is %s", UserGroupInformation.getCurrentUser());
-
- } else {
- println("Hadoop Cluster is insecure");
- }
-
- // verbose?
- if (diagnosticArgs.verbose) {
- // do the environment
- Map<String, String> env = getSystemEnv();
- Set<String> envList = ConfigHelper.sortedConfigKeys(env.entrySet());
- StringBuilder builder = new StringBuilder("Environment variables:\n");
- for (String key : envList) {
- builder.append(key).append("=").append(env.get(key)).append("\n");
- }
- println(builder.toString());
-
- // Java properties
- builder = new StringBuilder("JVM Properties\n");
- Map<String, String> props =
- sortedMap(toMap(System.getProperties()));
- for (Entry<String, String> entry : props.entrySet()) {
- builder.append(entry.getKey()).append("=")
- .append(entry.getValue()).append("\n");
- }
-
- println(builder.toString());
-
- // then the config
- println("Slider client configuration:\n" + ConfigHelper.dumpConfigToString(config));
- }
-
- validateSliderClientEnvironment(log);
- } catch (SliderException | IOException e) {
- log.error(e.toString());
- throw e;
- }
-
- }
-
- /**
- * Kerberos Diagnostics
- * @param args CLI arguments
- * @return exit code
- * @throws SliderException
- * @throws IOException
- */
- @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
- private int actionKDiag(ActionKDiagArgs args)
- throws Exception {
- PrintStream out;
- boolean closeStream = false;
- if (args.out != null) {
- out = new PrintStream(args.out, "UTF-8");
- closeStream = true;
- } else {
- ou
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
deleted file mode 100644
index 3e8a3c3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import org.apache.hadoop.yarn.api.records.Container;
-
-/**
- * Static assignment structure
- */
-public class ContainerAssignment {
-
- /**
- * Container that has been allocated
- */
- public final Container container;
-
- /**
- * Role to assign to it
- */
- public final RoleStatus role;
-
- /**
- * Placement outcome: was this from history or not
- */
- public final ContainerAllocationOutcome placement;
-
- public ContainerAssignment(Container container,
- RoleStatus role,
- ContainerAllocationOutcome placement) {
- this.container = container;
- this.role = role;
- this.placement = placement;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder("ContainerAssignment{");
- sb.append("container=").append(container);
- sb.append(", role=").append(role);
- sb.append(", placement=").append(placement);
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java
deleted file mode 100644
index 6df4bf4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
-
-/**
- * Container outcomes we care about; slightly simplified from
- * {@link ContainerExitStatus} -and hopefully able to handle
- * any new exit codes.
- */
-public enum ContainerOutcome {
- Completed,
- Failed,
- Failed_limits_exceeded,
- Disk_failure,
- Preempted;
-
- /**
- * Build a container outcome from an exit status.
- * The values in {@link ContainerExitStatus} are used
- * here.
- * @param exitStatus exit status
- * @return an enumeration of the outcome.
- */
- public static ContainerOutcome fromExitStatus(int exitStatus) {
- switch (exitStatus) {
- case ContainerExitStatus.ABORTED:
- case ContainerExitStatus.KILLED_BY_APPMASTER:
- case ContainerExitStatus.KILLED_BY_RESOURCEMANAGER:
- case ContainerExitStatus.KILLED_AFTER_APP_COMPLETION:
- // could either be a release or node failure. Treat as completion
- return Completed;
- case ContainerExitStatus.DISKS_FAILED:
- return Disk_failure;
- case ContainerExitStatus.PREEMPTED:
- return Preempted;
- case ContainerExitStatus.KILLED_EXCEEDED_PMEM:
- case ContainerExitStatus.KILLED_EXCEEDED_VMEM:
- return Failed_limits_exceeded;
- default:
- return exitStatus == 0 ? Completed : Failed;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
deleted file mode 100644
index df222fa..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.util.Records;
-
-import java.util.Locale;
-
-/**
- * Class containing the logic to build/split container priorities into the
- * different fields used by Slider
- *
- * The original design here had a requestID merged with the role, to
- * track outstanding requests. However, this isn't possible, so
- * the request ID has been dropped. A "location specified" flag was
- * added to indicate whether or not the request was for a specific location
- * -though this is currently unused.
- *
- * The methods are effectively surplus -but retained to preserve the
- * option of changing behavior in future
- */
-public final class ContainerPriority {
-
- // bit that represents whether location is specified
- static final int NOLOCATION = 1 << 30;
-
- public static int buildPriority(int role,
- boolean locationSpecified) {
- int location = locationSpecified ? 0 : NOLOCATION;
- return role | location;
- }
-
-
- public static Priority createPriority(int role,
- boolean locationSpecified) {
- Priority pri = Records.newRecord(Priority.class);
- pri.setPriority(ContainerPriority.buildPriority(role,
- locationSpecified));
- return pri;
- }
-
- public static int extractRole(int priority) {
- return priority >= NOLOCATION ? priority ^ NOLOCATION : priority;
- }
-
- /**
- * Does the priority have location
- * @param priority priority index
- * @return true if the priority has the location marker
- */
- public static boolean hasLocation(int priority) {
- return (priority ^ NOLOCATION ) == 0;
- }
-
- /**
- * Map from a container to a role key by way of its priority
- * @param container container
- * @return role key
- */
- public static int extractRole(Container container) {
- Priority priority = container.getPriority();
- return extractRole(priority);
- }
-
- /**
- * Priority record to role mapper
- * @param priorityRecord priority record
- * @return the role #
- */
- public static int extractRole(Priority priorityRecord) {
- Preconditions.checkNotNull(priorityRecord);
- return extractRole(priorityRecord.getPriority());
- }
-
- /**
- * Convert a priority record to a string, extracting role and locality
- * @param priorityRecord priority record. May be null
- * @return a string value
- */
- public static String toString(Priority priorityRecord) {
- if (priorityRecord==null) {
- return "(null)";
- } else {
- return String.format(Locale.ENGLISH,
- "role %d (locality=%b)",
- extractRole(priorityRecord),
- hasLocation(priorityRecord.getPriority()));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java
deleted file mode 100644
index fafbada..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import java.util.List;
-
-/**
- * Interface implemented by anything that must choose containers to release
- *
- */
-public interface ContainerReleaseSelector {
-
- /**
- * Given a list of candidate containers, return a sorted version of the priority
- * in which they should be released.
- * @param candidates candidate list ... everything considered suitable
- * @return the list of candidates
- */
- List<RoleInstance> sortCandidates(int roleId,
- List<RoleInstance> candidates);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
deleted file mode 100644
index 38c5b8e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import org.apache.slider.common.tools.Comparators;
-
-import java.io.Serializable;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-
-/**
- * Sort the candidate list by the most recent container first.
- */
-public class MostRecentContainerReleaseSelector implements ContainerReleaseSelector {
-
- @Override
- public List<RoleInstance> sortCandidates(int roleId,
- List<RoleInstance> candidates) {
- Collections.sort(candidates, new newerThan());
- return candidates;
- }
-
- private static class newerThan implements Comparator<RoleInstance>, Serializable {
- private final Comparator<Long> innerComparator =
- new Comparators.ComparatorReverser<>(new Comparators.LongComparator());
- public int compare(RoleInstance o1, RoleInstance o2) {
- return innerComparator.compare(o1.createTime, o2.createTime);
-
- }
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
deleted file mode 100644
index d57b6d2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
+++ /dev/null
@@ -1,325 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.slider.api.types.NodeEntryInformation;
-
-/**
- * Information about the state of a role on a specific node instance.
- * No fields are synchronized; sync on the instance to work with it
- * <p>
- * The two fields `releasing` and `requested` are used to track the ongoing
- * state of YARN requests; they do not need to be persisted across stop/start
- * cycles. They may be relevant across AM restart, but without other data
- * structures in the AM, not enough to track what the AM was up to before
- * it was restarted. The strategy will be to ignore unexpected allocation
- * responses (which may come from pre-restart) requests, while treating
- * unexpected container release responses as failures.
- * <p>
- * The `active` counter is only decremented after a container release response
- * has been received.
- * <p>
- *
- */
-public class NodeEntry implements Cloneable {
-
- public final int rolePriority;
-
- public NodeEntry(int rolePriority) {
- this.rolePriority = rolePriority;
- }
-
- /**
- * instance explicitly requested on this node: it's OK if an allocation
- * comes in that has not been (and when that happens, this count should
- * not drop).
- */
- private int requested;
-
- /** number of starting instances */
- private int starting;
-
- /** incrementing counter of instances that failed to start */
- private int startFailed;
-
- /** incrementing counter of instances that failed */
- private int failed;
-
- /**
- * Counter of "failed recently" events. These are all failures
- * which have happened since it was last reset.
- */
- private int failedRecently;
-
- /** incrementing counter of instances that have been pre-empted. */
- private int preempted;
-
- /**
- * Number of live nodes.
- */
- private int live;
-
- /** number of containers being released off this node */
- private int releasing;
-
- /** timestamp of last use */
- private long lastUsed;
-
- /**
- * Is the node available for assignments? That is, it is
- * not running any instances of this type, nor are there
- * any requests oustanding for it.
- * @return true if a new request could be issued without taking
- * the number of instances > 1.
- */
- public synchronized boolean isAvailable() {
- return live + requested + starting - releasing <= 0;
- }
-
- /**
- * Are the anti-affinity constraints held. That is, zero or one
- * node running or starting
- * @return true if the constraint holds.
- */
- public synchronized boolean isAntiAffinityConstraintHeld() {
- return (live - releasing + starting) <= 1;
- }
-
- /**
- * return no of active instances -those that could be released as they
- * are live and not already being released
- * @return a number, possibly 0
- */
- public synchronized int getActive() {
- return (live - releasing);
- }
-
- /**
- * Return true if the node is not busy, and it
- * has not been used since the absolute time
- * @param absoluteTime time
- * @return true if the node could be cleaned up
- */
- public synchronized boolean notUsedSince(long absoluteTime) {
- return isAvailable() && lastUsed < absoluteTime;
- }
-
- public synchronized int getLive() {
- return live;
- }
-
- public int getStarting() {
- return starting;
- }
-
- /**
- * Set the live value directly -used on AM restart
- * @param v value
- */
- public synchronized void setLive(int v) {
- live = v;
- }
-
- private synchronized void incLive() {
- ++live;
- }
-
- private synchronized void decLive() {
- live = RoleHistoryUtils.decToFloor(live);
- }
-
- public synchronized void onStarting() {
- ++starting;
- }
-
- private void decStarting() {
- starting = RoleHistoryUtils.decToFloor(starting);
- }
-
- public synchronized void onStartCompleted() {
- decStarting();
- incLive();
- }
-
- /**
- * start failed -decrement the starting flag.
- * @return true if the node is now available
- */
- public synchronized boolean onStartFailed() {
- decStarting();
- ++startFailed;
- return containerCompleted(false, ContainerOutcome.Failed);
- }
-
- /**
- * no of requests made of this role of this node. If it goes above
- * 1 there's a problem
- */
- public synchronized int getRequested() {
- return requested;
- }
-
- /**
- * request a node:
- */
- public synchronized void request() {
- ++requested;
- }
-
- /**
- * A request made explicitly to this node has completed
- */
- public synchronized void requestCompleted() {
- requested = RoleHistoryUtils.decToFloor(requested);
- }
-
- /**
- * No of instances in release state
- */
- public synchronized int getReleasing() {
- return releasing;
- }
-
- /**
- * Release an instance -which is no longer marked as active
- */
- public synchronized void release() {
- releasing++;
- }
-
- /**
- * completion event, which can be a planned or unplanned
- * planned: dec our release count
- * unplanned: dec our live count
- * @param wasReleased true if this was planned
- * @param outcome
- * @return true if this node is now available
- */
- public synchronized boolean containerCompleted(boolean wasReleased, ContainerOutcome outcome) {
- if (wasReleased) {
- releasing = RoleHistoryUtils.decToFloor(releasing);
- } else {
- // for the node, we use the outcome of the faiure to decide
- // whether this is potentially "node-related"
- switch(outcome) {
- // general "any reason" app failure
- case Failed:
- // specific node failure
- case Disk_failure:
-
- ++failed;
- ++failedRecently;
- break;
-
- case Preempted:
- preempted++;
- break;
-
- // failures which are node-independent
- case Failed_limits_exceeded:
- case Completed:
- default:
- break;
- }
- }
- decLive();
- return isAvailable();
- }
-
- /**
- * Time last used.
- */
- public synchronized long getLastUsed() {
- return lastUsed;
- }
-
- public synchronized void setLastUsed(long lastUsed) {
- this.lastUsed = lastUsed;
- }
-
- public synchronized int getStartFailed() {
- return startFailed;
- }
-
- public synchronized int getFailed() {
- return failed;
- }
-
- public synchronized int getFailedRecently() {
- return failedRecently;
- }
-
- @VisibleForTesting
- public synchronized void setFailedRecently(int failedRecently) {
- this.failedRecently = failedRecently;
- }
-
- public synchronized int getPreempted() {
- return preempted;
- }
-
-
- /**
- * Reset the failed recently count.
- */
- public synchronized void resetFailedRecently() {
- failedRecently = 0;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder("NodeEntry{");
- sb.append("priority=").append(rolePriority);
- sb.append(", requested=").append(requested);
- sb.append(", starting=").append(starting);
- sb.append(", live=").append(live);
- sb.append(", releasing=").append(releasing);
- sb.append(", lastUsed=").append(lastUsed);
- sb.append(", failedRecently=").append(failedRecently);
- sb.append(", preempted=").append(preempted);
- sb.append(", startFailed=").append(startFailed);
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * Produced a serialized form which can be served up as JSON
- * @return a summary of the current role status.
- */
- public synchronized NodeEntryInformation serialize() {
- NodeEntryInformation info = new NodeEntryInformation();
- info.priority = rolePriority;
- info.requested = requested;
- info.releasing = releasing;
- info.starting = starting;
- info.startFailed = startFailed;
- info.failed = failed;
- info.failedRecently = failedRecently;
- info.preempted = preempted;
- info.live = live;
- info.lastUsed = lastUsed;
- return info;
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException {
- return super.clone();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
deleted file mode 100644
index 120d402..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ /dev/null
@@ -1,419 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.common.tools.Comparators;
-import org.apache.slider.common.tools.SliderUtils;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-
-/**
- * A node instance -stores information about a node in the cluster.
- * <p>
- * Operations on the array/set of roles are synchronized.
- */
-public class NodeInstance {
-
- public final String hostname;
-
- private boolean blacklisted = false;
-
- /**
- * last state of node. Starts off as {@link NodeState#RUNNING},
- * on the assumption that it is live.
- */
- private NodeState nodeState = NodeState.RUNNING;
-
- /**
- * Last node report. If null: none
- */
- private NodeReport nodeReport = null;
-
- /**
- * time of state update
- */
- private long nodeStateUpdateTime = 0;
-
- /**
- * Node labels.
- *
- * IMPORTANT: we assume that there is one label/node, which is the policy
- * for Hadoop as of November 2015
- */
- private String nodeLabels = "";
-
- /**
- * An unordered list of node entries of specific roles. There's nothing
- * indexed so as to support sparser datastructures.
- */
- private final List<NodeEntry> nodeEntries;
-
- /**
- * Create an instance and the (empty) array of nodes
- * @param roles role count -the no. of roles
- */
- public NodeInstance(String hostname, int roles) {
- this.hostname = hostname;
- nodeEntries = new ArrayList<>(roles);
- }
-
- public synchronized void setBlacklisted(boolean blacklisted) {
- this.blacklisted = blacklisted;
- }
-
- public boolean isBlacklisted() {
- return blacklisted;
- }
-
- /**
- * Update the node status.
- * The return code is true if the node state changed enough to
- * trigger a re-evaluation of pending requests. That is, either a node
- * became available when it was previously not, or the label changed
- * on an available node.
- *
- * Transitions of a node from live to dead aren't treated as significant,
- * nor label changes on a dead node.
- *
- * @param report latest node report
- * @return true if the node state changed enough for a request evaluation.
- */
- public synchronized boolean updateNode(NodeReport report) {
- nodeStateUpdateTime = report.getLastHealthReportTime();
- nodeReport = report;
- NodeState oldState = nodeState;
- boolean oldStateUnusable = oldState.isUnusable();
- nodeState = report.getNodeState();
- boolean newUsable = !nodeState.isUnusable();
- boolean nodeNowAvailable = oldStateUnusable && newUsable;
- String labels = this.nodeLabels;
- nodeLabels = SliderUtils.extractNodeLabel(report);
- return nodeNowAvailable
- || newUsable && !this.nodeLabels.equals(labels);
- }
-
- public String getNodeLabels() {
- return nodeLabels;
- }
-
- /**
- * Get the entry for a role -if present
- * @param role role index
- * @return the entry
- * null if the role is out of range
- */
- public synchronized NodeEntry get(int role) {
- for (NodeEntry nodeEntry : nodeEntries) {
- if (nodeEntry.rolePriority == role) {
- return nodeEntry;
- }
- }
- return null;
- }
-
- /**
- * Get the entry for a role -if present
- * @param role role index
- * @return the entry
- * @throws ArrayIndexOutOfBoundsException if the role is out of range
- */
- public synchronized NodeEntry getOrCreate(int role) {
- NodeEntry entry = get(role);
- if (entry == null) {
- entry = new NodeEntry(role);
- nodeEntries.add(entry);
- }
- return entry;
- }
-
- /**
- * Get the node entry matching a container on this node
- * @param container container
- * @return matching node instance for the role
- */
- public NodeEntry getOrCreate(Container container) {
- return getOrCreate(ContainerPriority.extractRole(container));
- }
-
- /**
- * Count the number of active role instances on this node
- * @param role role index
- * @return 0 if there are none, otherwise the #of nodes that are running and
- * not being released already.
- */
- public int getActiveRoleInstances(int role) {
- NodeEntry nodeEntry = get(role);
- return (nodeEntry != null ) ? nodeEntry.getActive() : 0;
- }
-
- /**
- * Count the number of live role instances on this node
- * @param role role index
- * @return 0 if there are none, otherwise the #of nodes that are running
- */
- public int getLiveRoleInstances(int role) {
- NodeEntry nodeEntry = get(role);
- return (nodeEntry != null ) ? nodeEntry.getLive() : 0;
- }
-
- /**
- * Is the node considered online
- * @return the node
- */
- public boolean isOnline() {
- return !nodeState.isUnusable();
- }
-
- /**
- * Query for a node being considered unreliable
- * @param role role key
- * @param threshold threshold above which a node is considered unreliable
- * @return true if the node is considered unreliable
- */
- public boolean isConsideredUnreliable(int role, int threshold) {
- NodeEntry entry = get(role);
- return entry != null && entry.getFailedRecently() > threshold;
- }
-
- /**
- * Get the entry for a role -and remove it if present
- * @param role the role index
- * @return the entry that WAS there
- */
- public synchronized NodeEntry remove(int role) {
- NodeEntry nodeEntry = get(role);
- if (nodeEntry != null) {
- nodeEntries.remove(nodeEntry);
- }
- return nodeEntry;
- }
-
- public synchronized void set(int role, NodeEntry nodeEntry) {
- remove(role);
- nodeEntries.add(nodeEntry);
- }
-
- /**
- * run through each entry; gc'ing & removing old ones that don't have
- * a recent failure count (we care about those)
- * @param absoluteTime age in millis
- * @return true if there are still entries left
- */
- public synchronized boolean purgeUnusedEntries(long absoluteTime) {
- boolean active = false;
- ListIterator<NodeEntry> entries = nodeEntries.listIterator();
- while (entries.hasNext()) {
- NodeEntry entry = entries.next();
- if (entry.notUsedSince(absoluteTime) && entry.getFailedRecently() == 0) {
- entries.remove();
- } else {
- active = true;
- }
- }
- return active;
- }
-
-
- /**
- * run through each entry resetting the failure count
- */
- public synchronized void resetFailedRecently() {
- for (NodeEntry entry : nodeEntries) {
- entry.resetFailedRecently();
- }
- }
-
- @Override
- public String toString() {
- return hostname;
- }
-
- /**
- * Full dump of entry including children
- * @return a multi-line description fo the node
- */
- public String toFullString() {
- final StringBuilder sb =
- new StringBuilder(toString());
- sb.append("{ ");
- for (NodeEntry entry : nodeEntries) {
- sb.append(String.format("%n [%02d] ", entry.rolePriority));
- sb.append(entry.toString());
- }
- sb.append("} ");
- return sb.toString();
- }
-
- /**
- * Equality test is purely on the hostname of the node address
- * @param o other
- * @return true if the hostnames are equal
- */
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- NodeInstance that = (NodeInstance) o;
- return hostname.equals(that.hostname);
- }
-
- @Override
- public int hashCode() {
- return hostname.hashCode();
- }
-
-
- /**
- * Predicate to query if the number of recent failures of a role
- * on this node exceeds that role's failure threshold.
- * If there is no record of a deployment of that role on this
- * node, the failure count is taken as "0".
- * @param role role to look up
- * @return true if the failure rate is above the threshold.
- */
- public boolean exceedsFailureThreshold(RoleStatus role) {
- NodeEntry entry = get(role.getKey());
- int numFailuresOnLastHost = entry != null ? entry.getFailedRecently() : 0;
- int failureThreshold = role.getNodeFailureThreshold();
- return failureThreshold < 0 || numFailuresOnLastHost > failureThreshold;
- }
-
- /**
- * Produced a serialized form which can be served up as JSON
- * @param naming map of priority -> value for naming entries
- * @return a summary of the current role status.
- */
- public synchronized NodeInformation serialize(Map<Integer, String> naming) {
- NodeInformation info = new NodeInformation();
- info.hostname = hostname;
- // null-handling state constructor
- info.state = "" + nodeState;
- info.lastUpdated = nodeStateUpdateTime;
- info.labels = nodeLabels;
- if (nodeReport != null) {
- info.httpAddress = nodeReport.getHttpAddress();
- info.rackName = nodeReport.getRackName();
- info.healthReport = nodeReport.getHealthReport();
- }
- info.entries = new HashMap<>(nodeEntries.size());
- for (NodeEntry nodeEntry : nodeEntries) {
- String name = naming.get(nodeEntry.rolePriority);
- if (name == null) {
- name = Integer.toString(nodeEntry.rolePriority);
- }
- info.entries.put(name, nodeEntry.serialize());
- }
- return info;
- }
-
- /**
- * Is this node instance a suitable candidate for the specific role?
- * @param role role ID
- * @param label label which must match, or "" for no label checks
- * @return true if the node has space for this role, is running and the labels
- * match.
- */
- public boolean canHost(int role, String label) {
- return isOnline()
- && (SliderUtils.isUnset(label) || label.equals(nodeLabels)) // label match
- && getOrCreate(role).isAvailable(); // no live role
- }
-
- /**
- * A comparator for sorting entries where the node is preferred over another.
- *
- * The exact algorithm may change: current policy is "most recent first", so sorted
- * on the lastUsed
- *
- * the comparision is a positive int if left is preferred to right;
- * negative if right over left, 0 for equal
- */
- public static class Preferred implements Comparator<NodeInstance>, Serializable {
-
- private static final Comparators.InvertedLongComparator comparator =
- new Comparators.InvertedLongComparator();
- private final int role;
-
- public Preferred(int role) {
- this.role = role;
- }
-
- @Override
- public int compare(NodeInstance o1, NodeInstance o2) {
- NodeEntry left = o1.get(role);
- NodeEntry right = o2.get(role);
- long ageL = left != null ? left.getLastUsed() : -1;
- long ageR = right != null ? right.getLastUsed() : -1;
- return comparator.compare(ageL, ageR);
- }
- }
-
- /**
- * A comparator for sorting entries where the role is newer than
- * the other.
- * This sort only compares the lastUsed field, not whether the
- * node is in use or not
- */
- public static class MoreActiveThan implements Comparator<NodeInstance>,
- Serializable {
-
- private final int role;
-
- public MoreActiveThan(int role) {
- this.role = role;
- }
-
- @Override
- public int compare(NodeInstance left, NodeInstance right) {
- int activeLeft = left.getActiveRoleInstances(role);
- int activeRight = right.getActiveRoleInstances(role);
- return activeRight - activeLeft;
- }
- }
- /**
- * A comparator for sorting entries alphabetically
- */
- public static class CompareNames implements Comparator<NodeInstance>,
- Serializable {
-
- public CompareNames() {
- }
-
- @Override
- public int compare(NodeInstance left, NodeInstance right) {
- return left.hostname.compareTo(right.hostname);
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
deleted file mode 100644
index 3858b68..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Node map map -and methods to work with it.
- * Not Synchronized: caller is expected to lock access.
- */
-public class NodeMap extends HashMap<String, NodeInstance> {
- protected static final Logger log =
- LoggerFactory.getLogger(NodeMap.class);
-
- /**
- * number of roles
- */
- private final int roleSize;
-
- /**
- * Construct
- * @param roleSize number of roles
- */
- public NodeMap(int roleSize) {
- this.roleSize = roleSize;
- }
-
- /**
- * Get the node instance for the specific node -creating it if needed
- * @param hostname node
- * @return the instance
- */
- public NodeInstance getOrCreate(String hostname) {
- NodeInstance node = get(hostname);
- if (node == null) {
- node = new NodeInstance(hostname, roleSize);
- put(hostname, node);
- }
- return node;
- }
-
- /**
- * List the active nodes
- * @param role role
- * @return a possibly empty sorted list of all nodes that are active
- * in that role
- */
- public List<NodeInstance> listActiveNodes(int role) {
- List<NodeInstance> nodes = new ArrayList<>();
- for (NodeInstance instance : values()) {
- if (instance.getActiveRoleInstances(role) > 0) {
- nodes.add(instance);
- }
- }
- Collections.sort(nodes, new NodeInstance.MoreActiveThan(role));
- return nodes;
- }
-
- /**
- * reset the failed recently counters
- */
- public void resetFailedRecently() {
- for (Map.Entry<String, NodeInstance> entry : entrySet()) {
- NodeInstance ni = entry.getValue();
- ni.resetFailedRecently();
- }
- }
-
- /**
- * Update the node state. Return true if the node state changed: either by
- * being created, or by changing its internal state as defined
- * by {@link NodeInstance#updateNode(NodeReport)}.
- *
- * @param hostname host name
- * @param report latest node report
- * @return true if the node state changed enough for a request evaluation.
- */
- public boolean updateNode(String hostname, NodeReport report) {
- boolean nodeExisted = get(hostname) != null;
- boolean updated = getOrCreate(hostname).updateNode(report);
- return updated || !nodeExisted;
- }
-
- /**
- * Clone point
- * @return a shallow clone
- */
- @Override
- public Object clone() {
- return super.clone();
- }
-
- /**
- * Insert a list of nodes into the map; overwrite any with that name
- * This is a bulk operation for testing.
- * @param nodes collection of nodes.
- */
- @VisibleForTesting
- public void insert(Collection<NodeInstance> nodes) {
- for (NodeInstance node : nodes) {
- put(node.hostname, node);
- }
- }
-
- /**
- * Test helper: build or update a cluster from a list of node reports
- * @param reports the list of reports
- * @return true if this has been considered to have changed the cluster
- */
- @VisibleForTesting
- public boolean buildOrUpdate(List<NodeReport> reports) {
- boolean updated = false;
- for (NodeReport report : reports) {
- updated |= getOrCreate(report.getNodeId().getHost()).updateNode(report);
- }
- return updated;
- }
-
- /**
- * Scan the current node map for all nodes capable of hosting an instance
- * @param role role ID
- * @param label label which must match, or "" for no label checks
- * @return a possibly empty list of node instances matching the criteria.
- */
- public List<NodeInstance> findAllNodesForRole(int role, String label) {
- List<NodeInstance> nodes = new ArrayList<>(size());
- for (NodeInstance instance : values()) {
- if (instance.canHost(role, label)) {
- nodes.add(instance);
- }
- }
- Collections.sort(nodes, new NodeInstance.CompareNames());
- return nodes;
- }
-
- @Override
- public synchronized String toString() {
- final StringBuilder sb = new StringBuilder("NodeMap{");
- List<String> keys = new ArrayList<>(keySet());
- Collections.sort(keys);
- for (String key : keys) {
- sb.append(key).append(": ");
- sb.append(get(key).toFullString()).append("\n");
- }
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
deleted file mode 100644
index 4357ef8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ /dev/null
@@ -1,428 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.InvalidContainerRequestException;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Tracks an outstanding request. This is used to correlate an allocation response
- * with the node and role used in the request.
- * <p>
- * The node identifier may be null -which indicates that a request was made without
- * a specific target node
- * <p>
- * Equality and the hash code are based <i>only</i> on the role and hostname,
- * which are fixed in the constructor. This means that a simple
- * instance constructed with (role, hostname) can be used to look up
- * a complete request instance in the {@link OutstandingRequestTracker} map
- */
-public final class OutstandingRequest extends RoleHostnamePair {
- protected static final Logger log =
- LoggerFactory.getLogger(OutstandingRequest.class);
-
- /**
- * Node the request is for -may be null
- */
- public final NodeInstance node;
-
- /**
- * A list of all possible nodes to list in an AA request. For a non-AA
- * request where {@link #node} is set, element 0 of the list is the same
- * value.
- */
- public final List<NodeInstance> nodes = new ArrayList<>(1);
-
- /**
- * Optional label. This is cached as the request option (explicit-location + label) is forbidden,
- * yet the label needs to be retained for escalation.
- */
- public String label;
-
- /**
- * Requested time in millis.
- * <p>
- * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long)}
- */
- private AMRMClient.ContainerRequest issuedRequest;
-
- /**
- * Requested time in millis.
- * <p>
- * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long)}
- */
- private long requestedTimeMillis;
-
- /**
- * Time in millis after which escalation should be triggered..
- * <p>
- * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long)}
- */
- private long escalationTimeoutMillis;
-
- /**
- * Has the placement request been escalated?
- */
- private boolean escalated;
-
- /**
- * Flag to indicate that escalation is allowed
- */
- private boolean mayEscalate;
-
- /**
- * Priority of request; only valid after the request is built up
- */
- private int priority = -1;
-
- /**
- * Is this an Anti-affine request which should be cancelled on
- * a cluster resize?
- */
- private boolean antiAffine = false;
-
- /**
- * Create a request
- * @param roleId role
- * @param node node -can be null
- */
- public OutstandingRequest(int roleId,
- NodeInstance node) {
- super(roleId, node != null ? node.hostname : null);
- this.node = node;
- nodes.add(node);
- }
-
- /**
- * Create an outstanding request with the given role and hostname
- * Important: this is useful only for map lookups -the other constructor
- * with the NodeInstance parameter is needed to generate node-specific
- * container requests
- * @param roleId role
- * @param hostname hostname
- */
- public OutstandingRequest(int roleId, String hostname) {
- super(roleId, hostname);
- this.node = null;
- }
-
- /**
- * Create an Anti-affine reques, including all listed nodes (there must be one)
- * as targets.
- * @param roleId role
- * @param nodes list of nodes
- */
- public OutstandingRequest(int roleId, List<NodeInstance> nodes) {
- super(roleId, nodes.get(0).hostname);
- this.node = null;
- this.antiAffine = true;
- this.nodes.addAll(nodes);
- }
-
- /**
- * Is the request located in the cluster, that is: does it have a node.
- * @return true if a node instance was supplied in the constructor
- */
- public boolean isLocated() {
- return node != null;
- }
-
- public long getRequestedTimeMillis() {
- return requestedTimeMillis;
- }
-
- public long getEscalationTimeoutMillis() {
- return escalationTimeoutMillis;
- }
-
- public synchronized boolean isEscalated() {
- return escalated;
- }
-
- public boolean mayEscalate() {
- return mayEscalate;
- }
-
- public AMRMClient.ContainerRequest getIssuedRequest() {
- return issuedRequest;
- }
-
- public int getPriority() {
- return priority;
- }
-
- public boolean isAntiAffine() {
- return antiAffine;
- }
-
- public void setAntiAffine(boolean antiAffine) {
- this.antiAffine = antiAffine;
- }
-
- /**
- * Build a container request.
- * <p>
- * The value of {@link #node} is used to direct a lot of policy. If null,
- * placement is relaxed.
- * If not null, the choice of whether to use the suggested node
- * is based on the placement policy and failure history.
- * <p>
- * If the request has an address, it is set in the container request
- * (with a flag to enable relaxed priorities).
- * <p>
- * This operation sets the requested time flag, used for tracking timeouts
- * on outstanding requests
- * @param resource resource
- * @param role role
- * @param time time in millis to record as request time
- * @return the request to raise
- */
- public synchronized AMRMClient.ContainerRequest buildContainerRequest(
- Resource resource, RoleStatus role, long time) {
- Preconditions.checkArgument(resource != null, "null `resource` arg");
- Preconditions.checkArgument(role != null, "null `role` arg");
-
- // cache label for escalation
- label = role.getLabelExpression();
- requestedTimeMillis = time;
- escalationTimeoutMillis = time + role.getPlacementTimeoutSeconds() * 1000;
- String[] hosts;
- boolean relaxLocality;
- boolean strictPlacement = role.isStrictPlacement();
- NodeInstance target = this.node;
- String nodeLabels;
-
- if (isAntiAffine()) {
- int size = nodes.size();
- log.info("Creating anti-affine request across {} nodes; first node = {}",
- size, hostname);
- hosts = new String[size];
- StringBuilder builder = new StringBuilder(size * 16);
- int c = 0;
- for (NodeInstance nodeInstance : nodes) {
- hosts[c++] = nodeInstance.hostname;
- builder.append(nodeInstance.hostname).append(" ");
- }
- log.debug("Full host list: [ {}]", builder);
- escalated = false;
- mayEscalate = false;
- relaxLocality = false;
- nodeLabels = null;
- } else if (target != null) {
- // placed request. Hostname is used in request
- hosts = new String[1];
- hosts[0] = target.hostname;
- // and locality flag is set to false; Slider will decide when
- // to relax things
- relaxLocality = false;
-
- log.info("Submitting request for container on {}", hosts[0]);
- // enable escalation for all but strict placements.
- escalated = false;
- mayEscalate = !strictPlacement;
- nodeLabels = null;
- } else {
- // no hosts
- hosts = null;
- // relax locality is mandatory on an unconstrained placement
- relaxLocality = true;
- // declare that the the placement is implicitly escalated.
- escalated = true;
- // and forbid it happening
- mayEscalate = false;
- nodeLabels = label;
- }
- Priority pri = ContainerPriority.createPriority(roleId, !relaxLocality);
- priority = pri.getPriority();
- issuedRequest = new AMRMClient.ContainerRequest(resource,
- hosts,
- null,
- pri,
- relaxLocality,
- nodeLabels);
- validate();
- return issuedRequest;
- }
-
-
- /**
- * Build an escalated container request, updating {@link #issuedRequest} with
- * the new value.
- * @return the new container request, which has the same resource and label requirements
- * as the original one, and the same host, but: relaxed placement, and a changed priority
- * so as to place it into the relaxed list.
- */
- public synchronized AMRMClient.ContainerRequest escalate() {
- Preconditions.checkNotNull(issuedRequest, "cannot escalate if request not issued " + this);
- log.debug("Escalating {}", this.toString());
- escalated = true;
-
- // this is now the priority
- // it is tagged as unlocated because it needs to go into a different
- // set of outstanding requests from the strict placements
- Priority pri = ContainerPriority.createPriority(roleId, false);
- // update the field
- priority = pri.getPriority();
-
- String[] nodes;
- List<String> issuedRequestNodes = issuedRequest.getNodes();
- if (SliderUtils.isUnset(label) && issuedRequestNodes != null) {
- nodes = issuedRequestNodes.toArray(new String[issuedRequestNodes.size()]);
- } else {
- nodes = null;
- }
-
- issuedRequest = new AMRMClient.ContainerRequest(issuedRequest.getCapability(),
- nodes,
- null,
- pri,
- true,
- label);
- validate();
- return issuedRequest;
- }
-
- /**
- * Mark the request as completed (or canceled).
- * <p>
- * Current action: if a node is defined, its request count is decremented
- */
- public void completed() {
- if (node != null) {
- node.getOrCreate(roleId).requestCompleted();
- }
- }
-
- /**
- * Query to see if the request is available and ready to be escalated
- * @param time time to check against
- * @return true if escalation should begin
- */
- public synchronized boolean shouldEscalate(long time) {
- return mayEscalate
- && !escalated
- && issuedRequest != null
- && escalationTimeoutMillis < time;
- }
-
- /**
- * Query for the resource requirements matching; always false before a request is issued
- * @param resource
- * @return
- */
- public synchronized boolean resourceRequirementsMatch(Resource resource) {
- return issuedRequest != null && Resources.fitsIn(issuedRequest.getCapability(), resource);
- }
-
- @Override
- public String toString() {
- boolean requestHasLocation = ContainerPriority.hasLocation(getPriority());
- final StringBuilder sb = new StringBuilder("OutstandingRequest{");
- sb.append("roleId=").append(roleId);
- if (hostname != null) {
- sb.append(", hostname='").append(hostname).append('\'');
- }
- sb.append(", node=").append(node);
- sb.append(", hasLocation=").append(requestHasLocation);
- sb.append(", label=").append(label);
- sb.append(", requestedTimeMillis=").append(requestedTimeMillis);
- sb.append(", mayEscalate=").append(mayEscalate);
- sb.append(", escalated=").append(escalated);
- sb.append(", escalationTimeoutMillis=").append(escalationTimeoutMillis);
- sb.append(", issuedRequest=").append(
- issuedRequest != null ? SliderUtils.requestToString(issuedRequest) : "(null)");
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * Create a cancel operation
- * @return an operation that can be used to cancel the request
- */
- public CancelSingleRequest createCancelOperation() {
- Preconditions.checkState(issuedRequest != null, "No issued request to cancel");
- return new CancelSingleRequest(issuedRequest);
- }
-
- /**
- * Valid if a node label expression specified on container request is valid or
- * not. Mimics the logic in AMRMClientImpl, so can be used for preflight checking
- * and in mock tests
- *
- */
- public void validate() throws InvalidContainerRequestException {
- Preconditions.checkNotNull(issuedRequest, "request has not yet been built up");
- AMRMClient.ContainerRequest containerRequest = issuedRequest;
- String requestDetails = this.toString();
- validateContainerRequest(containerRequest, priority, requestDetails);
- }
-
- /**
- * Inner Validation logic for container request
- * @param containerRequest request
- * @param priority raw priority of role
- * @param requestDetails details for error messages
- */
- @VisibleForTesting
- public static void validateContainerRequest(AMRMClient.ContainerRequest containerRequest,
- int priority, String requestDetails) {
- String exp = containerRequest.getNodeLabelExpression();
- boolean hasRacks = containerRequest.getRacks() != null &&
- (!containerRequest.getRacks().isEmpty());
- boolean hasNodes = containerRequest.getNodes() != null &&
- (!containerRequest.getNodes().isEmpty());
-
- boolean hasLabel = SliderUtils.isSet(exp);
-
- // Don't support specifying >= 2 node labels in a node label expression now
- if (hasLabel && (exp.contains("&&") || exp.contains("||"))) {
- throw new InvalidContainerRequestException(
- "Cannot specify more than two node labels"
- + " in a single node label expression: " + requestDetails);
- }
-
- // Don't allow specify node label against ANY request listing hosts or racks
- if (hasLabel && ( hasRacks || hasNodes)) {
- throw new InvalidContainerRequestException(
- "Cannot specify node label with rack or node: " + requestDetails);
- }
- }
-
- /**
- * Create a new role/hostname pair for indexing.
- * @return a new index.
- */
- public RoleHostnamePair getIndex() {
- return new RoleHostnamePair(roleId, hostname);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
deleted file mode 100644
index dbdf8ca..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ /dev/null
@@ -1,482 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Tracks outstanding requests made with a specific placement option.
- * <p>
- * <ol>
- * <li>Used to decide when to return a node to 'can request containers here' list</li>
- * <li>Used to identify requests where placement has timed out, and so issue relaxed requests</li>
- * </ol>
- * <p>
- * If an allocation comes in that is not in the map: either the allocation
- * was unplaced, or the placed allocation could not be met on the specified
- * host, and the RM/scheduler fell back to another location.
- */
-
-public class OutstandingRequestTracker {
- protected static final Logger log =
- LoggerFactory.getLogger(OutstandingRequestTracker.class);
-
- /**
- * no requests; saves creating a new list if not needed
- */
- private final List<AbstractRMOperation> NO_REQUESTS = new ArrayList<>(0);
-
- private Map<RoleHostnamePair, OutstandingRequest> placedRequests = new HashMap<>();
-
- /**
- * List of open requests; no specific details on them.
- */
- private List<OutstandingRequest> openRequests = new ArrayList<>();
-
- /**
- * Create a new request for the specific role.
- * <p>
- * If a location is set, the request is added to {@link #placedRequests}.
- * If not, it is added to {@link #openRequests}
- * <p>
- * This does not update the node instance's role's request count
- * @param instance node instance to manager
- * @param role role index
- * @return a new request
- */
- public synchronized OutstandingRequest newRequest(NodeInstance instance, int role) {
- OutstandingRequest request = new OutstandingRequest(role, instance);
- if (request.isLocated()) {
- placedRequests.put(request.getIndex(), request);
- } else {
- openRequests.add(request);
- }
- return request;
- }
-
- /**
- * Create a new Anti-affine request for the specific role
- * <p>
- * It is added to {@link #openRequests}
- * <p>
- * This does not update the node instance's role's request count
- * @param role role index
- * @param nodes list of suitable nodes
- * @param label label to use
- * @return a new request
- */
- public synchronized OutstandingRequest newAARequest(int role,
- List<NodeInstance> nodes,
- String label) {
- Preconditions.checkArgument(!nodes.isEmpty());
- // safety check to verify the allocation will hold
- for (NodeInstance node : nodes) {
- Preconditions.checkState(node.canHost(role, label),
- "Cannot allocate role ID %d to node %s", role, node);
- }
- OutstandingRequest request = new OutstandingRequest(role, nodes);
- openRequests.add(request);
- return request;
- }
-
- /**
- * Look up any oustanding request to a (role, hostname).
- * @param role role index
- * @param hostname hostname
- * @return the request or null if there was no outstanding one in the {@link #placedRequests}
- */
- @VisibleForTesting
- public synchronized OutstandingRequest lookupPlacedRequest(int role, String hostname) {
- Preconditions.checkArgument(hostname != null, "null hostname");
- return placedRequests.get(new RoleHostnamePair(role, hostname));
- }
-
- /**
- * Remove a request
- * @param request matching request to find
- * @return the request or null for no match in the {@link #placedRequests}
- */
- @VisibleForTesting
- public synchronized OutstandingRequest removePlacedRequest(OutstandingRequest request) {
- return placedRequests.remove(request);
- }
-
- /**
- * Notification that a container has been allocated
- *
- * <ol>
- * <li>drop it from the {@link #placedRequests} structure.</li>
- * <li>generate the cancellation request</li>
- * <li>for AA placement, any actions needed</li>
- * </ol>
- *
- * @param role role index
- * @param hostname hostname
- * @return the allocation outcome
- */
- public synchronized ContainerAllocationResults onContainerAllocated(int role,
- String hostname,
- Container container) {
- final String containerDetails = SliderUtils.containerToString(container);
- log.debug("Processing allocation for role {} on {}", role,
- containerDetails);
- ContainerAllocationResults allocation = new ContainerAllocationResults();
- ContainerAllocationOutcome outcome;
- OutstandingRequest request = placedRequests.remove(new OutstandingRequest(role, hostname));
- if (request != null) {
- //satisfied request
- log.debug("Found oustanding placed request for container: {}", request);
- request.completed();
- // derive outcome from status of tracked request
- outcome = request.isEscalated()
- ? ContainerAllocationOutcome.Escalated
- : ContainerAllocationOutcome.Placed;
- } else {
- // not in the list; this is an open placement
- // scan through all containers in the open request list
- request = removeOpenRequest(container);
- if (request != null) {
- log.debug("Found open outstanding request for container: {}", request);
- request.completed();
- outcome = ContainerAllocationOutcome.Open;
- } else {
- log.warn("No oustanding request found for container {}, outstanding queue has {} entries ",
- containerDetails,
- openRequests.size());
- outcome = ContainerAllocationOutcome.Unallocated;
- }
- }
- if (request != null && request.getIssuedRequest() != null) {
- allocation.operations.add(request.createCancelOperation());
- } else {
- // there's a request, but no idea what to cancel.
- // rather than try to recover from it inelegantly, (and cause more confusion),
- // log the event, but otherwise continue
- log.warn("Unexpected allocation of container " + SliderUtils.containerToString(container));
- }
-
- allocation.origin = request;
- allocation.outcome = outcome;
- return allocation;
- }
-
- /**
- * Find and remove an open request. Determine it by scanning open requests
- * for one whose priority & resource requirements match that of the container
- * allocated.
- * @param container container allocated
- * @return a request which matches the allocation, or null for "no match"
- */
- private OutstandingRequest removeOpenRequest(Container container) {
- int pri = container.getPriority().getPriority();
- Resource resource = container.getResource();
- OutstandingRequest request = null;
- ListIterator<OutstandingRequest> openlist = openRequests.listIterator();
- while (openlist.hasNext() && request == null) {
- OutstandingRequest r = openlist.next();
- if (r.getPriority() == pri) {
- // matching resource
- if (r.resourceRequirementsMatch(resource)) {
- // match of priority and resources
- request = r;
- openlist.remove();
- } else {
- log.debug("Matched priorities but resources different");
- }
- }
- }
- return request;
- }
-
- /**
- * Determine which host was a role type most recently used on, so that
- * if a choice is made of which (potentially surplus) containers to use,
- * the most recent one is picked first. This operation <i>does not</i>
- * change the role history, though it queries it.
- */
- static class newerThan implements Comparator<Container> {
- private RoleHistory rh;
-
- public newerThan(RoleHistory rh) {
- this.rh = rh;
- }
-
- /**
- * Get the age of a node hosting container. If it is not known in the history,
- * return 0.
- * @param c container
- * @return age, null if there's no entry for it.
- */
- private long getAgeOf(Container c) {
- long age = 0;
- NodeInstance node = rh.getExistingNodeInstance(c);
- int role = ContainerPriority.extractRole(c);
- if (node != null) {
- NodeEntry nodeEntry = node.get(role);
- if (nodeEntry != null) {
- age = nodeEntry.getLastUsed();
- }
- }
- return age;
- }
-
- /**
- * Comparator: which host is more recent?
- * @param c1 container 1
- * @param c2 container 2
- * @return 1 if c2 older-than c1, 0 if equal; -1 if c1 older-than c2
- */
- @Override
- public int compare(Container c1, Container c2) {
- int role1 = ContainerPriority.extractRole(c1);
- int role2 = ContainerPriority.extractRole(c2);
- if (role1 < role2) return -1;
- if (role1 > role2) return 1;
-
- long age = getAgeOf(c1);
- long age2 = getAgeOf(c2);
-
- if (age > age2) {
- return -1;
- } else if (age < age2) {
- return 1;
- }
- // equal
- return 0;
- }
- }
-
- /**
- * Take a list of requests and split them into specific host requests and
- * generic assignments. This is to give requested hosts priority
- * in container assignments if more come back than expected
- * @param rh RoleHistory instance
- * @param inAllocated the list of allocated containers
- * @param outPlaceRequested initially empty list of requested locations
- * @param outUnplaced initially empty list of unrequested hosts
- */
- public synchronized void partitionRequests(RoleHistory rh,
- List<Container> inAllocated,
- List<Container> outPlaceRequested,
- List<Container> outUnplaced) {
- Collections.sort(inAllocated, new newerThan(rh));
- for (Container container : inAllocated) {
- int role = ContainerPriority.extractRole(container);
- String hostname = RoleHistoryUtils.hostnameOf(container);
- if (placedRequests.containsKey(new OutstandingRequest(role, hostname))) {
- outPlaceRequested.add(container);
- } else {
- outUnplaced.add(container);
- }
- }
- }
-
-
- /**
- * Reset list all outstanding requests for a role: return the hostnames
- * of any canceled requests
- *
- * @param role role to cancel
- * @return possibly empty list of hostnames
- */
- public synchronized List<NodeInstance> resetOutstandingRequests(int role) {
- List<NodeInstance> hosts = new ArrayList<>();
- Iterator<Map.Entry<RoleHostnamePair, OutstandingRequest>> iterator =
- placedRequests.entrySet().iterator();
- while (iterator.hasNext()) {
- Map.Entry<RoleHostnamePair, OutstandingRequest> next =
- iterator.next();
- OutstandingRequest request = next.getValue();
- if (request.roleId == role) {
- iterator.remove();
- request.completed();
- hosts.add(request.node);
- }
- }
- ListIterator<OutstandingRequest> openlist = openRequests.listIterator();
- while (openlist.hasNext()) {
- OutstandingRequest next = openlist.next();
- if (next.roleId == role) {
- openlist.remove();
- }
- }
- return hosts;
- }
-
- /**
- * Get a list of outstanding requests. The list is cloned, but the contents
- * are shared
- * @return a list of the current outstanding requests
- */
- public synchronized List<OutstandingRequest> listPlacedRequests() {
- return new ArrayList<>(placedRequests.values());
- }
-
- /**
- * Get a list of outstanding requests. The list is cloned, but the contents
- * are shared
- * @return a list of the current outstanding requests
- */
- public synchronized List<OutstandingRequest> listOpenRequests() {
- return new ArrayList<>(openRequests);
- }
-
- /**
- * Escalate operation as triggered by external timer.
- * @return a (usually empty) list of cancel/request operations.
- */
- @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
- public synchronized List<AbstractRMOperation> escalateOutstandingRequests(long now) {
- if (placedRequests.isEmpty()) {
- return NO_REQUESTS;
- }
-
- List<AbstractRMOperation> operations = new ArrayList<>();
- for (OutstandingRequest outstandingRequest : placedRequests.values()) {
- synchronized (outstandingRequest) {
- // sync escalation check with operation so that nothing can happen to state
- // of the request during the escalation
- if (outstandingRequest.shouldEscalate(now)) {
-
- // time to escalate
- CancelSingleRequest cancel = outstandingRequest.createCancelOperation();
- operations.add(cancel);
- AMRMClient.ContainerRequest escalated = outstandingRequest.escalate();
- operations.add(new ContainerRequestOperation(escalated));
- }
- }
-
- }
- return operations;
- }
-
- /**
- * Cancel all outstanding AA requests from the lists of requests.
- *
- * This does not remove them from the role status; they must be reset
- * by the caller.
- *
- */
- @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
- public synchronized List<AbstractRMOperation> cancelOutstandingAARequests() {
-
- log.debug("Looking for AA request to cancel");
- List<AbstractRMOperation> operations = new ArrayList<>();
-
- // first, all placed requests
- List<RoleHostnamePair> requestsToRemove = new ArrayList<>(placedRequests.size());
- for (Map.Entry<RoleHostnamePair, OutstandingRequest> entry : placedRequests.entrySet()) {
- OutstandingRequest outstandingRequest = entry.getValue();
- synchronized (outstandingRequest) {
- if (outstandingRequest.isAntiAffine()) {
- // time to escalate
- operations.add(outstandingRequest.createCancelOperation());
- requestsToRemove.add(entry.getKey());
- }
- }
- }
- for (RoleHostnamePair keys : requestsToRemove) {
- placedRequests.remove(keys);
- }
-
- // second, all open requests
- ListIterator<OutstandingRequest> orit = openRequests.listIterator();
- while (orit.hasNext()) {
- OutstandingRequest outstandingRequest = orit.next();
- synchronized (outstandingRequest) {
- if (outstandingRequest.isAntiAffine()) {
- // time to escalate
- operations.add(outstandingRequest.createCancelOperation());
- orit.remove();
- }
- }
- }
- log.info("Cancelling {} outstanding AA requests", operations.size());
-
- return operations;
- }
-
- /**
- * Extract a specific number of open requests for a role
- * @param roleId role Id
- * @param count count to extract
- * @return a list of requests which are no longer in the open request list
- */
- public synchronized List<OutstandingRequest> extractOpenRequestsForRole(int roleId, int count) {
- List<OutstandingRequest> results = new ArrayList<>();
- ListIterator<OutstandingRequest> openlist = openRequests.listIterator();
- while (openlist.hasNext() && count > 0) {
- OutstandingRequest openRequest = openlist.next();
- if (openRequest.roleId == roleId) {
- results.add(openRequest);
- openlist.remove();
- count--;
- }
- }
- return results;
- }
-
- /**
- * Extract a specific number of placed requests for a role
- * @param roleId role Id
- * @param count count to extract
- * @return a list of requests which are no longer in the placed request data structure
- */
- public synchronized List<OutstandingRequest> extractPlacedRequestsForRole(int roleId, int count) {
- List<OutstandingRequest> results = new ArrayList<>();
- Iterator<Map.Entry<RoleHostnamePair, OutstandingRequest>>
- iterator = placedRequests.entrySet().iterator();
- while (iterator.hasNext() && count > 0) {
- OutstandingRequest request = iterator.next().getValue();
- if (request.roleId == roleId) {
- results.add(request);
- count--;
- }
- }
- // now cull them from the map
- for (OutstandingRequest result : results) {
- placedRequests.remove(result);
- }
-
- return results;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
deleted file mode 100644
index 8fc08b7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.google.common.cache.LoadingCache;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.api.types.ApplicationLivenessInformation;
-import org.apache.slider.api.types.ComponentInformation;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.api.types.RoleStatistics;
-import org.apache.slider.core.exceptions.NoSuchNodeException;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
-import org.apache.slider.core.registry.docstore.PublishedExportsSet;
-import org.apache.slider.server.appmaster.web.rest.RestPaths;
-import org.apache.slider.server.services.utility.PatternValidator;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * Implementation of {@link StateAccessForProviders}, which means
- * state access for providers, web UI and IPC/REST views.
- */
-public class ProviderAppState implements StateAccessForProviders {
-
-
- private final Map<String, PublishedConfigSet> publishedConfigSets =
- new ConcurrentHashMap<>(5);
- private final PublishedExportsSet publishedExportsSets = new PublishedExportsSet();
- private static final PatternValidator validator = new PatternValidator(
- RestPaths.PUBLISHED_CONFIGURATION_SET_REGEXP);
- private String applicationName;
-
- private final AppState appState;
-
- public ProviderAppState(String applicationName, AppState appState) {
- this.appState = appState;
- this.applicationName = applicationName;
- }
-
- public void setApplicationName(String applicationName) {
- this.applicationName = applicationName;
- }
-
- @Override
- public String getApplicationName() {
- return applicationName;
- }
-
- @Override
- public PublishedConfigSet getPublishedSliderConfigurations() {
- return getOrCreatePublishedConfigSet(RestPaths.SLIDER_CONFIGSET);
- }
-
- @Override
- public PublishedExportsSet getPublishedExportsSet() {
- return publishedExportsSets;
- }
-
- @Override
- public PublishedConfigSet getPublishedConfigSet(String name) {
- return publishedConfigSets.get(name);
- }
-
- @Override
- public PublishedConfigSet getOrCreatePublishedConfigSet(String name) {
- PublishedConfigSet set = publishedConfigSets.get(name);
- if (set == null) {
- validator.validate(name);
- synchronized (publishedConfigSets) {
- // synchronized double check to ensure that there is never an overridden
- // config set created
- set = publishedConfigSets.get(name);
- if (set == null) {
- set = new PublishedConfigSet();
- publishedConfigSets.put(name, set);
- }
- }
- }
- return set;
- }
-
- @Override
- public List<String> listConfigSets() {
-
- synchronized (publishedConfigSets) {
- List<String> sets = new ArrayList<>(publishedConfigSets.keySet());
- return sets;
- }
- }
-
- @Override
- public Map<Integer, RoleStatus> getRoleStatusMap() {
- return appState.getRoleStatusMap();
- }
-
-
- @Override
- public Map<ContainerId, RoleInstance> getFailedContainers() {
- return appState.getFailedContainers();
- }
-
- @Override
- public Map<ContainerId, RoleInstance> getLiveContainers() {
- return appState.getLiveContainers();
- }
-
- @Override
- public Application getApplication() {
- return appState.getClusterStatus();
- }
-
- @Override
- public boolean isApplicationLive() {
- return appState.isApplicationLive();
- }
-
- @Override
- public RoleStatus lookupRoleStatus(int key) {
- return appState.lookupRoleStatus(key);
- }
-
- @Override
- public RoleStatus lookupRoleStatus(Container c) throws YarnRuntimeException {
- return appState.lookupRoleStatus(c);
- }
-
- @Override
- public RoleStatus lookupRoleStatus(String name) throws YarnRuntimeException {
- return appState.lookupRoleStatus(name);
- }
-
- @Override
- public List<RoleInstance> cloneOwnedContainerList() {
- return appState.cloneOwnedContainerList();
- }
-
- @Override
- public int getNumOwnedContainers() {
- return appState.getNumOwnedContainers();
- }
-
- @Override
- public RoleInstance getOwnedContainer(ContainerId id) {
- return appState.getOwnedContainer(id);
- }
-
- @Override
- public RoleInstance getOwnedContainer(String id) throws NoSuchNodeException {
- return appState.getOwnedInstanceByContainerID(id);
- }
-
- @Override
- public List<RoleInstance> cloneLiveContainerInfoList() {
- return appState.cloneLiveContainerInfoList();
- }
-
- @Override
- public RoleInstance getLiveInstanceByContainerID(String containerId) throws
- NoSuchNodeException {
- return appState.getLiveInstanceByContainerID(containerId);
- }
-
- @Override
- public List<RoleInstance> getLiveInstancesByContainerIDs(Collection<String> containerIDs) {
- return appState.getLiveInstancesByContainerIDs(containerIDs);
- }
-
- @Override
- public Application refreshClusterStatus() {
- return appState.refreshClusterStatus();
- }
-
- @Override
- public ApplicationLivenessInformation getApplicationLivenessInformation() {
- return appState.getApplicationLivenessInformation();
- }
-
- @Override
- public Map<String, ComponentInformation> getComponentInfoSnapshot() {
- return appState.getComponentInfoSnapshot();
- }
-
- @Override
- public Map<String, Map<String, ClusterNode>> getRoleClusterNodeMapping() {
- return appState.createRoleToClusterNodeMap();
- }
-
- @Override
- public List<RoleInstance> enumLiveInstancesInRole(String role) {
- List<RoleInstance> nodes = new ArrayList<>();
- Collection<RoleInstance> allRoleInstances = cloneLiveContainerInfoList();
- for (RoleInstance node : allRoleInstances) {
- if (role.isEmpty() || role.equals(node.role)) {
- nodes.add(node);
- }
- }
- return nodes;
- }
-
- @Override
- public List<RoleInstance> lookupRoleContainers(String component) {
- RoleStatus roleStatus = lookupRoleStatus(component);
- List<RoleInstance> ownedContainerList = cloneOwnedContainerList();
- List<RoleInstance> matching = new ArrayList<>(ownedContainerList.size());
- int roleId = roleStatus.getPriority();
- for (RoleInstance instance : ownedContainerList) {
- if (instance.roleId == roleId) {
- matching.add(instance);
- }
- }
- return matching;
- }
-
- @Override
- public ComponentInformation getComponentInformation(String component) {
- RoleStatus roleStatus = lookupRoleStatus(component);
- ComponentInformation info = roleStatus.serialize();
- List<RoleInstance> containers = lookupRoleContainers(component);
- info.containers = new ArrayList<>(containers.size());
- for (RoleInstance container : containers) {
- info.containers.add(container.id);
- }
- return info;
- }
-
- @Override
- public Map<String, NodeInformation> getNodeInformationSnapshot() {
- return appState.getRoleHistory()
- .getNodeInformationSnapshot(appState.buildNamingMap());
- }
-
- @Override
- public NodeInformation getNodeInformation(String hostname) {
- return appState.getRoleHistory()
- .getNodeInformation(hostname, appState.buildNamingMap());
- }
-
- @Override
- public RoleStatistics getRoleStatistics() {
- return appState.getRoleStatistics();
- }
-
- @Override
- public Map<String, String> getGlobalSubstitutionTokens() {
- return appState.globalTokens;
- }
-
- @Override
- public LoadingCache<ConfigFile, Object> getConfigFileCache() {
- return appState.configFileCache;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
deleted file mode 100644
index ba923bc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ /dev/null
@@ -1,2120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.ServiceApiConstants;
-import org.apache.slider.api.StatusKeys;
-import org.apache.slider.api.proto.Messages;
-import org.apache.slider.api.proto.Messages.ComponentCountProto;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.ApplicationState;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.api.types.ApplicationLivenessInformation;
-import org.apache.slider.api.types.ComponentInformation;
-import org.apache.slider.api.types.RoleStatistics;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.NoSuchNodeException;
-import org.apache.slider.core.exceptions.SliderInternalStateException;
-import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.core.zk.ZKIntegration;
-import org.apache.slider.providers.PlacementPolicy;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
-import org.apache.slider.server.appmaster.management.MetricsConstants;
-import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
-import static org.apache.hadoop.registry.client.api.RegistryConstants.DEFAULT_REGISTRY_ZK_QUORUM;
-import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_DOMAIN;
-import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM;
-import static org.apache.slider.api.ResourceKeys.*;
-import static org.apache.slider.api.ServiceApiConstants.*;
-import static org.apache.slider.api.StateValues.*;
-import static org.apache.slider.api.resource.ApplicationState.STARTED;
-
-/**
- * The model of all the ongoing state of a Slider AM.
- *
- * concurrency rules: any method which begins with <i>build</i>
- * is not synchronized and intended to be used during
- * initialization.
- */
-public class AppState {
- protected static final Logger log =
- LoggerFactory.getLogger(AppState.class);
-
- private final AbstractClusterServices recordFactory;
-
- private final MetricsAndMonitoring metricsAndMonitoring;
- /**
- * Flag set to indicate the application is live -this only happens
- * after the buildInstance operation
- */
- private boolean applicationLive = false;
-
- private Application app;
-
- // priority_id -> RoleStatus
- private final Map<Integer, RoleStatus> roleStatusMap =
- new ConcurrentSkipListMap<>();
-
- // component_name -> ProviderRole
- private final Map<String, ProviderRole> roles =
- new ConcurrentHashMap<>();
-
- private final ConcurrentSkipListMap<Integer, ProviderRole> rolePriorityMap =
- new ConcurrentSkipListMap<>();
-
- /**
- * Hash map of the containers we have. This includes things that have
- * been allocated but are not live; it is a superset of the live list
- */
- private final ConcurrentMap<ContainerId, RoleInstance> ownedContainers =
- new ConcurrentHashMap<>();
-
- /**
- * Hash map of the containers we have released, but we
- * are still awaiting acknowledgements on. Any failure of these
- * containers is treated as a successful outcome
- */
- private final ConcurrentMap<ContainerId, Container> containersBeingReleased =
- new ConcurrentHashMap<>();
-
- /**
- * Map of requested nodes. This records the command used to start it,
- * resources, etc. When container started callback is received,
- * the node is promoted from here to the containerMap
- */
- private final Map<ContainerId, RoleInstance> startingContainers =
- new ConcurrentHashMap<>();
-
- /**
- * List of completed nodes. This isn't kept in the CD as it gets too
- * big for the RPC responses. Indeed, we should think about how deep to get this
- */
- private final Map<ContainerId, RoleInstance> completedContainers
- = new ConcurrentHashMap<>();
-
- /**
- * Nodes that failed to start.
- * Again, kept out of the CD
- */
- private final Map<ContainerId, RoleInstance> failedContainers =
- new ConcurrentHashMap<>();
-
- /**
- * Nodes that came assigned to a role above that
- * which were asked for -this appears to happen
- */
- private final Set<ContainerId> surplusContainers = new HashSet<>();
-
- /**
- * Map of containerID to cluster nodes, for status reports.
- * Access to this should be synchronized on the clusterDescription
- */
- private final Map<ContainerId, RoleInstance> liveNodes =
- new ConcurrentHashMap<>();
- private final AtomicInteger completionOfNodeNotInLiveListEvent =
- new AtomicInteger();
- private final AtomicInteger completionOfUnknownContainerEvent =
- new AtomicInteger();
-
- /**
- * limits of container core numbers in this queue
- */
- private int containerMaxCores;
- private int containerMinCores;
-
- /**
- * limits of container memory in this queue
- */
- private int containerMaxMemory;
- private int containerMinMemory;
-
- private RoleHistory roleHistory;
- private long startTimeThreshold;
-
- private int failureThreshold = 10;
- private int nodeFailureThreshold = 3;
-
- private String logServerURL = "";
- public Map<String, String> globalTokens = new HashMap<>();
- /**
- * Selector of containers to release; application wide.
- */
- private ContainerReleaseSelector containerReleaseSelector;
- private Resource minResource;
- private Resource maxResource;
-
- private ServiceMetrics appMetrics;
-
- private ServiceTimelinePublisher serviceTimelinePublisher;
-
- // A cache for loading config files from remote such as hdfs
- public LoadingCache<ConfigFile, Object> configFileCache = null;
-
- /**
- * Create an instance
- * @param recordFactory factory for YARN records
- * @param metricsAndMonitoring metrics and monitoring services
- */
- public AppState(AbstractClusterServices recordFactory,
- MetricsAndMonitoring metricsAndMonitoring) {
- Preconditions.checkArgument(recordFactory != null, "null recordFactory");
- Preconditions.checkArgument(metricsAndMonitoring != null, "null metricsAndMonitoring");
- this.recordFactory = recordFactory;
- this.metricsAndMonitoring = metricsAndMonitoring;
- }
-
-
- public Map<Integer, RoleStatus> getRoleStatusMap() {
- return roleStatusMap;
- }
-
- protected Map<String, ProviderRole> getRoleMap() {
- return roles;
- }
-
- public Map<Integer, ProviderRole> getRolePriorityMap() {
- return rolePriorityMap;
- }
-
- private Map<ContainerId, RoleInstance> getStartingContainers() {
- return startingContainers;
- }
-
- private Map<ContainerId, RoleInstance> getCompletedContainers() {
- return completedContainers;
- }
-
- public Map<ContainerId, RoleInstance> getFailedContainers() {
- return failedContainers;
- }
-
- public Map<ContainerId, RoleInstance> getLiveContainers() {
- return liveNodes;
- }
-
- /**
- * Get the current view of the cluster status.
- * This is read-only
- * to the extent that changes here do not trigger updates in the
- * application state.
- * @return the cluster status
- */
- public synchronized Application getClusterStatus() {
- return app;
- }
-
- /**
- * Get the role history of the application
- * @return the role history
- */
- @VisibleForTesting
- public RoleHistory getRoleHistory() {
- return roleHistory;
- }
-
- @VisibleForTesting
- public void setRoleHistory(RoleHistory roleHistory) {
- this.roleHistory = roleHistory;
- }
-
- /**
- * Get the path used for history files
- * @return the directory used for history files
- */
- @VisibleForTesting
- public Path getHistoryPath() {
- return roleHistory.getHistoryPath();
- }
-
- /**
- * Set the container limits -the min and max values for
- * resource requests. All requests must be multiples of the min
- * values.
- * @param minMemory min memory MB
- * @param maxMemory maximum memory
- * @param minCores min v core count
- * @param maxCores maximum cores
- */
- public void setContainerLimits(int minMemory,int maxMemory, int minCores, int maxCores) {
- containerMinCores = minCores;
- containerMaxCores = maxCores;
- containerMinMemory = minMemory;
- containerMaxMemory = maxMemory;
- minResource = recordFactory.newResource(containerMinMemory, containerMinCores);
- maxResource = recordFactory.newResource(containerMaxMemory, containerMaxCores);
- }
-
- public boolean isApplicationLive() {
- return applicationLive;
- }
-
-
- public synchronized void buildInstance(AppStateBindingInfo binding)
- throws BadClusterStateException, BadConfigException, IOException {
- binding.validate();
- containerReleaseSelector = binding.releaseSelector;
-
- // set the cluster specification (once its dependency the client properties
- // is out the way
- this.app = binding.application;
- appMetrics = ServiceMetrics.register(app.getName(),
- "Metrics for service");
- appMetrics.tag("type", "Metrics type [component or service]", "service");
- appMetrics.tag("appId", "Application id for service", app.getId());
-
- org.apache.slider.api.resource.Configuration conf = app.getConfiguration();
- startTimeThreshold =
- conf.getPropertyLong(InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
- InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE);
- failureThreshold = conf.getPropertyInt(CONTAINER_FAILURE_THRESHOLD,
- DEFAULT_CONTAINER_FAILURE_THRESHOLD);
- nodeFailureThreshold = conf.getPropertyInt(NODE_FAILURE_THRESHOLD,
- DEFAULT_NODE_FAILURE_THRESHOLD);
- initGlobalTokensForSubstitute(binding);
-
- // build the initial component list
- Collection<Component> sortedComponents = ServiceApiUtil
- .sortByDependencies(app.getComponents());
- int priority = 1;
- for (Component component : sortedComponents) {
- priority = getNewPriority(priority);
- String name = component.getName();
- if (roles.containsKey(name)) {
- continue;
- }
- log.info("Adding component: " + name);
- createComponent(name, component, priority++);
- }
-
- //then pick up the requirements
-// buildRoleRequirementsFromResources();
-
- // set up the role history
- roleHistory = new RoleHistory(roleStatusMap.values(), recordFactory);
- roleHistory.onStart(binding.fs, binding.historyPath);
- // trigger first node update
- roleHistory.onNodesUpdated(binding.nodeReports);
- //rebuild any live containers
- rebuildModelFromRestart(binding.liveContainers);
-
- // any am config options to pick up
- logServerURL = binding.serviceConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL, "");
- //mark as live
- applicationLive = true;
- app.setState(STARTED);
- createConfigFileCache(binding.fs);
- }
-
- private void initGlobalTokensForSubstitute(AppStateBindingInfo binding)
- throws IOException {
- // ZK
- globalTokens.put(ServiceApiConstants.CLUSTER_ZK_QUORUM,
- binding.serviceConfig
- .getTrimmed(KEY_REGISTRY_ZK_QUORUM, DEFAULT_REGISTRY_ZK_QUORUM));
- String user = UserGroupInformation.getCurrentUser().getShortUserName();
- globalTokens
- .put(SERVICE_ZK_PATH, ZKIntegration.mkClusterPath(user, app.getName()));
-
- globalTokens.put(ServiceApiConstants.USER, user);
- String dnsDomain = binding.serviceConfig.getTrimmed(KEY_DNS_DOMAIN);
- if (dnsDomain != null && !dnsDomain.isEmpty()) {
- globalTokens.put(ServiceApiConstants.DOMAIN, dnsDomain);
- }
- // HDFS
- String clusterFs = binding.serviceConfig.getTrimmed(FS_DEFAULT_NAME_KEY);
- if (clusterFs != null && !clusterFs.isEmpty()) {
- globalTokens.put(ServiceApiConstants.CLUSTER_FS_URI, clusterFs);
- globalTokens.put(ServiceApiConstants.CLUSTER_FS_HOST,
- URI.create(clusterFs).getHost());
- }
- globalTokens.put(SERVICE_HDFS_DIR, binding.serviceHdfsDir);
- // service name
- globalTokens.put(SERVICE_NAME_LC, app.getName().toLowerCase());
- globalTokens.put(SERVICE_NAME, app.getName());
- }
-
- private void createConfigFileCache(final FileSystem fileSystem) {
- this.configFileCache =
- CacheBuilder.newBuilder().expireAfterAccess(10, TimeUnit.MINUTES)
- .build(new CacheLoader<ConfigFile, Object>() {
- @Override public Object load(ConfigFile key) throws Exception {
- switch (key.getType()) {
- case HADOOP_XML:
- try (FSDataInputStream input = fileSystem
- .open(new Path(key.getSrcFile()))) {
- org.apache.hadoop.conf.Configuration confRead =
- new org.apache.hadoop.conf.Configuration(false);
- confRead.addResource(input);
- Map<String, String> map = new HashMap<>(confRead.size());
- for (Map.Entry<String, String> entry : confRead) {
- map.put(entry.getKey(), entry.getValue());
- }
- return map;
- }
- case TEMPLATE:
- try (FSDataInputStream fileInput = fileSystem
- .open(new Path(key.getSrcFile()))) {
- return IOUtils.toString(fileInput);
- }
- default:
- return null;
- }
- }
- });
- }
-
- public ProviderRole createComponent(String name, Component component,
- int priority) throws BadConfigException {
- org.apache.slider.api.resource.Configuration conf =
- component.getConfiguration();
- long placementTimeout = conf.getPropertyLong(PLACEMENT_ESCALATE_DELAY,
- DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS);
- long placementPolicy = conf.getPropertyLong(COMPONENT_PLACEMENT_POLICY,
- PlacementPolicy.DEFAULT);
- int threshold = conf.getPropertyInt(NODE_FAILURE_THRESHOLD,
- nodeFailureThreshold);
- String label = conf.getProperty(YARN_LABEL_EXPRESSION,
- DEF_YARN_LABEL_EXPRESSION);
- ProviderRole newRole =
- new ProviderRole(name, priority, (int)placementPolicy, threshold,
- placementTimeout, label, component);
- buildRole(newRole, component);
- log.info("Created a new role " + newRole);
- return newRole;
- }
-
- @VisibleForTesting
- public synchronized void updateComponents(Map<String, Long>
- componentCounts) throws BadConfigException {
- for (Component component : app.getComponents()) {
- if (componentCounts.containsKey(component.getName())) {
- long count = componentCounts.get(component.getName());
- component.setNumberOfContainers(count);
- ProviderRole role = roles.get(component.getName());
- if (role != null && roleStatusMap.get(role.id) != null) {
- setDesiredContainers(roleStatusMap.get(role.id), (int) count);
- }
- }
- }
- }
-
- public synchronized void updateComponents(
- Messages.FlexComponentsRequestProto requestProto)
- throws BadConfigException {
- Map<String, Long> componentCounts = new HashMap<>();
- for (ComponentCountProto componentCount : requestProto
- .getComponentsList()) {
- componentCounts.put(componentCount.getName(), componentCount
- .getNumberOfContainers());
- }
- updateComponents(componentCounts);
- }
-
- /**
- * build the role requirements from the cluster specification
- * @return a list of any dynamically added provider roles
- */
-
-// private List<ProviderRole> buildRoleRequirementsFromResources()
-// throws BadConfigException {
-//
-// List<ProviderRole> newRoles = new ArrayList<>(0);
-//
-// // now update every role's desired count.
-// // if there are no instance values, that role count goes to zero
-// // Add all the existing roles
-// // component name -> number of containers
-// Map<String, Integer> groupCounts = new HashMap<>();
-//
-// for (RoleStatus roleStatus : getRoleStatusMap().values()) {
-// if (roleStatus.isExcludeFromFlexing()) {
-// // skip inflexible roles, e.g AM itself
-// continue;
-// }
-// long currentDesired = roleStatus.getDesired();
-// String role = roleStatus.getName();
-// String roleGroup = roleStatus.getGroup();
-// Component component = roleStatus.getProviderRole().component;
-// int desiredInstanceCount = component.getNumberOfContainers().intValue();
-//
-// int newDesired = desiredInstanceCount;
-// if (component.getUniqueComponentSupport()) {
-// Integer groupCount = 0;
-// if (groupCounts.containsKey(roleGroup)) {
-// groupCount = groupCounts.get(roleGroup);
-// }
-//
-// newDesired = desiredInstanceCount - groupCount;
-//
-// if (newDesired > 0) {
-// newDesired = 1;
-// groupCounts.put(roleGroup, groupCount + newDesired);
-// } else {
-// newDesired = 0;
-// }
-// }
-//
-// if (newDesired == 0) {
-// log.info("Role {} has 0 instances specified", role);
-// }
-// if (currentDesired != newDesired) {
-// log.info("Role {} flexed from {} to {}", role, currentDesired,
-// newDesired);
-// setDesiredContainers(roleStatus, newDesired);
-// }
-// }
-//
-// log.info("Counts per component: " + groupCounts);
-// // now the dynamic ones. Iterate through the the cluster spec and
-// // add any role status entries not in the role status
-//
-// List<RoleStatus> list = new ArrayList<>(getRoleStatusMap().values());
-// for (RoleStatus roleStatus : list) {
-// String name = roleStatus.getName();
-// Component component = roleStatus.getProviderRole().component;
-// if (roles.containsKey(name)) {
-// continue;
-// }
-// if (component.getUniqueComponentSupport()) {
-// // THIS NAME IS A GROUP
-// int desiredInstanceCount = component.getNumberOfContainers().intValue();
-// Integer groupCount = 0;
-// if (groupCounts.containsKey(name)) {
-// groupCount = groupCounts.get(name);
-// }
-// log.info("Component " + component.getName() + ", current count = "
-// + groupCount + ", desired count = " + desiredInstanceCount);
-// for (int i = groupCount + 1; i <= desiredInstanceCount; i++) {
-// int priority = roleStatus.getPriority();
-// // this is a new instance of an existing group
-// String newName = String.format("%s%d", name, i);
-// int newPriority = getNewPriority(priority + i - 1);
-// log.info("Adding new role {}", newName);
-// ProviderRole dynamicRole =
-// createComponent(newName, name, component, newPriority);
-// RoleStatus newRole = buildRole(dynamicRole);
-// incDesiredContainers(newRole);
-// log.info("New role {}", newRole);
-// if (roleHistory != null) {
-// roleHistory.addNewRole(newRole);
-// }
-// newRoles.add(dynamicRole);
-// }
-// } else {
-// // this is a new value
-// log.info("Adding new role {}", name);
-// ProviderRole dynamicRole =
-// createComponent(name, name, component, roleStatus.getPriority());
-// RoleStatus newRole = buildRole(dynamicRole);
-// incDesiredContainers(roleStatus,
-// component.getNumberOfContainers().intValue());
-// log.info("New role {}", newRole);
-// if (roleHistory != null) {
-// roleHistory.addNewRole(newRole);
-// }
-// newRoles.add(dynamicRole);
-// }
-// }
-// // and fill in all those roles with their requirements
-// buildRoleResourceRequirements();
-//
-// return newRoles;
-// }
-
- private int getNewPriority(int start) {
- if (!rolePriorityMap.containsKey(start)) {
- return start;
- }
- return rolePriorityMap.lastKey() + 1;
- }
-
- /**
- * Add knowledge of a role.
- * This is a build-time operation that is not synchronized, and
- * should be used while setting up the system state -before servicing
- * requests.
- * @param providerRole role to add
- * @return the role status built up
- * @throws BadConfigException if a role of that priority already exists
- */
- public RoleStatus buildRole(ProviderRole providerRole, Component component)
- throws BadConfigException {
- // build role status map
- int priority = providerRole.id;
- if (roleStatusMap.containsKey(priority)) {
- throw new BadConfigException("Duplicate component priority Key: %s and %s",
- providerRole, roleStatusMap.get(priority));
- }
- RoleStatus roleStatus = new RoleStatus(providerRole);
- roleStatus.setResourceRequirements(buildResourceRequirements(roleStatus));
- long prev = roleStatus.getDesired();
- setDesiredContainers(roleStatus, component.getNumberOfContainers().intValue());
- log.info("Set desired containers for component " + component.getName() +
- " from " + prev + " to " + roleStatus.getDesired());
- roleStatusMap.put(priority, roleStatus);
- String name = providerRole.name;
- roles.put(name, providerRole);
- rolePriorityMap.put(priority, providerRole);
- // register its entries
- metricsAndMonitoring.addMetricSet(MetricsConstants.PREFIX_SLIDER_ROLES + name, roleStatus);
- return roleStatus;
- }
-
- /**
- * Look up the status entry of a role or raise an exception
- * @param key role ID
- * @return the status entry
- * @throws RuntimeException if the role cannot be found
- */
- public RoleStatus lookupRoleStatus(int key) {
- RoleStatus rs = getRoleStatusMap().get(key);
- if (rs == null) {
- throw new RuntimeException("Cannot find role for role ID " + key);
- }
- return rs;
- }
-
- /**
- * Look up the status entry of a container or raise an exception
- *
- * @param c container
- * @return the status entry
- * @throws RuntimeException if the role cannot be found
- */
- public RoleStatus lookupRoleStatus(Container c) {
- return lookupRoleStatus(ContainerPriority.extractRole(c));
- }
-
-
- /**
- * Look up a role in the map
- * @param name role name
- * @return the instance
- * @throws YarnRuntimeException if not found
- */
- public RoleStatus lookupRoleStatus(String name) throws YarnRuntimeException {
- ProviderRole providerRole = roles.get(name);
- if (providerRole == null) {
- throw new YarnRuntimeException("Unknown role " + name);
- }
- return lookupRoleStatus(providerRole.id);
- }
-
-
- /**
- * Clone the list of active (==owned) containers
- * @return the list of role instances representing all owned containers
- */
- public synchronized List<RoleInstance> cloneOwnedContainerList() {
- Collection<RoleInstance> values = ownedContainers.values();
- return new ArrayList<>(values);
- }
-
- /**
- * Get the number of active (==owned) containers
- * @return
- */
- public int getNumOwnedContainers() {
- return ownedContainers.size();
- }
-
- /**
- * Look up an active container: any container that the AM has, even
- * if it is not currently running/live
- */
- public RoleInstance getOwnedContainer(ContainerId id) {
- return ownedContainers.get(id);
- }
-
- /**
- * Remove an owned container
- * @param id container ID
- * @return the instance removed
- */
- private RoleInstance removeOwnedContainer(ContainerId id) {
- return ownedContainers.remove(id);
- }
-
- /**
- * set/update an owned container
- * @param id container ID
- * @param instance
- * @return
- */
- private RoleInstance putOwnedContainer(ContainerId id,
- RoleInstance instance) {
- return ownedContainers.put(id, instance);
- }
-
- /**
- * Clone the live container list. This is synchronized.
- * @return a snapshot of the live node list
- */
- public synchronized List<RoleInstance> cloneLiveContainerInfoList() {
- List<RoleInstance> allRoleInstances;
- Collection<RoleInstance> values = getLiveContainers().values();
- allRoleInstances = new ArrayList<>(values);
- return allRoleInstances;
- }
-
- /**
- * Lookup live instance by string value of container ID
- * @param containerId container ID as a string
- * @return the role instance for that container
- * @throws NoSuchNodeException if it does not exist
- */
- public synchronized RoleInstance getLiveInstanceByContainerID(String containerId)
- throws NoSuchNodeException {
- Collection<RoleInstance> nodes = getLiveContainers().values();
- return findNodeInCollection(containerId, nodes);
- }
-
- /**
- * Lookup owned instance by string value of container ID
- * @param containerId container ID as a string
- * @return the role instance for that container
- * @throws NoSuchNodeException if it does not exist
- */
- public synchronized RoleInstance getOwnedInstanceByContainerID(String containerId)
- throws NoSuchNodeException {
- Collection<RoleInstance> nodes = ownedContainers.values();
- return findNodeInCollection(containerId, nodes);
- }
-
- /**
- * Iterate through a collection of role instances to find one with a
- * specific (string) container ID
- * @param containerId container ID as a string
- * @param nodes collection
- * @return the found node
- * @throws NoSuchNodeException if there was no match
- */
- private RoleInstance findNodeInCollection(String containerId,
- Collection<RoleInstance> nodes) throws NoSuchNodeException {
- RoleInstance found = null;
- for (RoleInstance node : nodes) {
- if (containerId.equals(node.id)) {
- found = node;
- break;
- }
- }
- if (found != null) {
- return found;
- } else {
- //at this point: no node
- throw new NoSuchNodeException("Unknown node: " + containerId);
- }
- }
-
- public synchronized List<RoleInstance> getLiveInstancesByContainerIDs(
- Collection<String> containerIDs) {
- //first, a hashmap of those containerIDs is built up
- Set<String> uuidSet = new HashSet<String>(containerIDs);
- List<RoleInstance> nodes = new ArrayList<RoleInstance>(uuidSet.size());
- Collection<RoleInstance> clusterNodes = getLiveContainers().values();
-
- for (RoleInstance node : clusterNodes) {
- if (uuidSet.contains(node.id)) {
- nodes.add(node);
- }
- }
- //at this point: a possibly empty list of nodes
- return nodes;
- }
-
- /**
- * Enum all nodes by role.
- * @param role role, or "" for all roles
- * @return a list of nodes, may be empty
- */
- public synchronized List<RoleInstance> enumLiveNodesInRole(String role) {
- List<RoleInstance> nodes = new ArrayList<RoleInstance>();
- Collection<RoleInstance> allRoleInstances = getLiveContainers().values();
- for (RoleInstance node : allRoleInstances) {
- if (role.isEmpty() || role.equals(node.role)) {
- nodes.add(node);
- }
- }
- return nodes;
- }
-
-
- /**
- * enum nodes by role ID, from either the owned or live node list
- * @param roleId role the container must be in
- * @param owned flag to indicate "use owned list" rather than the smaller
- * "live" list
- * @return a list of nodes, may be empty
- */
- public synchronized List<RoleInstance> enumNodesWithRoleId(int roleId,
- boolean owned) {
- List<RoleInstance> nodes = new ArrayList<RoleInstance>();
- Collection<RoleInstance> allRoleInstances;
- allRoleInstances = owned ? ownedContainers.values() : liveNodes.values();
- for (RoleInstance node : allRoleInstances) {
- if (node.roleId == roleId) {
- nodes.add(node);
- }
- }
- return nodes;
- }
-
- /**
- * Build an instance map.
- * @return the map of Role name to list of role instances
- */
- private synchronized Map<String, List<String>> createRoleToInstanceMap() {
- Map<String, List<String>> map = new HashMap<String, List<String>>();
- for (RoleInstance node : getLiveContainers().values()) {
- List<String> containers = map.get(node.role);
- if (containers == null) {
- containers = new ArrayList<String>();
- map.put(node.role, containers);
- }
- containers.add(node.id);
- }
- return map;
- }
-
- /**
- * Build a map of Component_name -> ContainerId -> ClusterNode
- *
- * @return the map of Role name to list of Cluster Nodes
- */
- public synchronized Map<String, Map<String, ClusterNode>> createRoleToClusterNodeMap() {
- Map<String, Map<String, ClusterNode>> map = new HashMap<>();
- for (RoleInstance node : getLiveContainers().values()) {
-
- Map<String, ClusterNode> containers = map.get(node.role);
- if (containers == null) {
- containers = new HashMap<String, ClusterNode>();
- map.put(node.role, containers);
- }
- ClusterNode clusterNode = node.toClusterNode();
- containers.put(clusterNode.name, clusterNode);
- }
- return map;
- }
-
- /**
- * Notification called just before the NM is asked to
- * start a container
- * @param container container to start
- * @param instance clusterNode structure
- */
- public void containerStartSubmitted(Container container,
- RoleInstance instance) {
- instance.state = STATE_SUBMITTED;
- instance.container = container;
- instance.createTime = now();
- getStartingContainers().put(container.getId(), instance);
- putOwnedContainer(container.getId(), instance);
- roleHistory.onContainerStartSubmitted(container, instance);
- }
-
- /**
- * Note that a container has been submitted for release; update internal state
- * and mark the associated ContainerInfo released field to indicate that
- * while it is still in the active list, it has been queued for release.
- *
- * @param container container
- * @throws SliderInternalStateException if there is no container of that ID
- * on the active list
- */
- public synchronized void containerReleaseSubmitted(Container container)
- throws SliderInternalStateException {
- ContainerId id = container.getId();
- //look up the container
- RoleInstance instance = getOwnedContainer(id);
- if (instance == null) {
- throw new SliderInternalStateException(
- "No active container with ID " + id);
- }
- //verify that it isn't already released
- if (containersBeingReleased.containsKey(id)) {
- throw new SliderInternalStateException(
- "Container %s already queued for release", id);
- }
- instance.released = true;
- containersBeingReleased.put(id, instance.container);
- roleHistory.onContainerReleaseSubmitted(container);
- }
-
- /**
- * Create a container request.
- * Update internal state, such as the role request count.
- * Anti-Affine: the {@link RoleStatus#outstandingAArequest} is set here.
- * This is where role history information will be used for placement decisions.
- * @param role role
- * @return the container request to submit or null if there is none
- */
- private AMRMClient.ContainerRequest createContainerRequest(RoleStatus role) {
- if (role.isAntiAffinePlacement()) {
- return createAAContainerRequest(role);
- } else {
- OutstandingRequest request = roleHistory.requestContainerForRole(role);
- if (request != null) {
- return request.getIssuedRequest();
- } else {
- return null;
- }
- }
- }
-
- /**
- * Create a container request.
- * Update internal state, such as the role request count.
- * Anti-Affine: the {@link RoleStatus#outstandingAArequest} is set here.
- * This is where role history information will be used for placement decisions.
- * @param role role
- * @return the container request to submit or null if there is none
- */
- private AMRMClient.ContainerRequest createAAContainerRequest(RoleStatus role) {
- OutstandingRequest request = roleHistory.requestContainerForAARole(role);
- if (request == null) {
- return null;
- }
- role.setOutstandingAArequest(request);
- return request.getIssuedRequest();
- }
-
- @VisibleForTesting
- public void incRequestedContainers(RoleStatus role) {
- log.info("Incrementing requested containers for {}", role.getName());
- role.getComponentMetrics().containersRequested.incr();
- appMetrics.containersRequested.incr();
- }
-
- private void decRequestedContainers(RoleStatus role) {
- role.getComponentMetrics().containersRequested.decr();
- appMetrics.containersRequested.decr();
- log.info("Decrementing requested containers for {} by {} to {}", role
- .getName(), 1, role.getComponentMetrics().containersRequested.value());
- }
-
- private int decRequestedContainersToFloor(RoleStatus role, int delta) {
- int actual = decMetricToFloor(role.getComponentMetrics()
- .containersRequested, delta);
- appMetrics.containersRequested.decr(actual);
- log.info("Decrementing requested containers for {} by {} to {}", role
- .getName(), actual, role.getComponentMetrics().containersRequested
- .value());
- return actual;
- }
-
- private int decAAPendingToFloor(RoleStatus role, int delta) {
- int actual = decMetricToFloor(role.getComponentMetrics()
- .pendingAAContainers, delta);
- appMetrics.pendingAAContainers.decr(actual);
- log.info("Decrementing AA pending containers for {} by {} to {}", role
- .getName(), actual, role.getComponentMetrics().pendingAAContainers
- .value());
- return actual;
- }
-
- private int decMetricToFloor(MutableGaugeInt metric, int delta) {
- int currentValue = metric.value();
- int decrAmount = delta;
- if (currentValue - delta < 0) {
- decrAmount = currentValue;
- }
- metric.decr(decrAmount);
- return decrAmount;
- }
-
- @VisibleForTesting
- public void incRunningContainers(RoleStatus role) {
- role.getComponentMetrics().containersRunning.incr();
- appMetrics.containersRunning.incr();
- }
-
- private void decRunningContainers(RoleStatus role) {
- role.getComponentMetrics().containersRunning.decr();
- appMetrics.containersRunning.decr();
- }
-
- private void setDesiredContainers(RoleStatus role, int n) {
- int delta = n - role.getComponentMetrics().containersDesired.value();
- role.getComponentMetrics().containersDesired.set(n);
- appMetrics.containersDesired.incr(delta);
- }
-
- private void incCompletedContainers(RoleStatus role) {
- role.getComponentMetrics().containersSucceeded.incr();
- appMetrics.containersSucceeded.incr();
- }
-
- @VisibleForTesting
- public void incFailedContainers(RoleStatus role, ContainerOutcome outcome) {
- switch (outcome) {
- case Preempted:
- appMetrics.containersPreempted.incr();
- role.getComponentMetrics().containersPreempted.incr();
- break;
- case Disk_failure:
- appMetrics.containersDiskFailure.incr();
- appMetrics.containersFailed.incr();
- role.getComponentMetrics().containersDiskFailure.incr();
- role.getComponentMetrics().containersFailed.incr();
- break;
-// case Failed:
-// appMetrics.failedSinceLastThreshold.incr();
-// appMetrics.containersFailed.incr();
-// role.getComponentMetrics().failedSinceLastThreshold.incr();
-// role.getComponentMetrics().containersFailed.incr();
-// break;
-// case Failed_limits_exceeded:
-// appMetrics.containersLimitsExceeded.incr();
-// appMetrics.failedSinceLastThreshold.incr();
-// appMetrics.containersFailed.incr();
-// role.getComponentMetrics().containersLimitsExceeded.incr();
-// role.getComponentMetrics().failedSinceLastThreshold.incr();
-// role.getComponentMetrics().containersFailed.incr();
-// break;
-// default:
-// appMetrics.failedSinceLastThreshold.incr();
-// appMetrics.containersFailed.incr();
-// role.getComponentMetrics().failedSinceLastThreshold.incr();
-// role.getComponentMetrics().containersFailed.incr();
-// break;
- }
- }
-
- /**
- * Build up the resource requirements for this role from the cluster
- * specification, including substituting max allowed values if the
- * specification asked for it (except when
- * {@link org.apache.slider.api.ResourceKeys#YARN_RESOURCE_NORMALIZATION_ENABLED}
- * is set to false).
- * @param role role
- * during normalization
- */
- public Resource buildResourceRequirements(RoleStatus role) {
- // Set up resource requirements from role values
- String name = role.getName();
- Component component = role.getProviderRole().component;
- int cores = DEF_YARN_CORES;
- if (component.getResource() != null && component.getResource().getCpus()
- != null) {
- cores = Math.min(containerMaxCores, component.getResource().getCpus());
- }
- if (cores <= 0) {
- cores = DEF_YARN_CORES;
- }
- long rawMem = DEF_YARN_MEMORY;
- if (component.getResource() != null && component.getResource().getMemory()
- != null) {
- if (YARN_RESOURCE_MAX.equals(component.getResource().getMemory())) {
- rawMem = containerMaxMemory;
- } else {
- rawMem = Long.parseLong(component.getResource().getMemory());
- }
- }
- boolean normalize = component.getConfiguration().getPropertyBool(
- YARN_RESOURCE_NORMALIZATION_ENABLED, true);
- if (!normalize) {
- log.info("Resource normalization: disabled");
- log.debug("Component {} has RAM={}, vCores={}", name, rawMem, cores);
- return Resources.createResource(rawMem, cores);
- }
- long mem = Math.min(containerMaxMemory, rawMem);
- if (mem <= 0) {
- mem = DEF_YARN_MEMORY;
- }
- Resource capability = Resource.newInstance(mem, cores);
- log.debug("Component {} has RAM={}, vCores={}", name, mem, cores);
- Resource normalized = recordFactory.normalize(capability, minResource,
- maxResource);
- if (!Resources.equals(normalized, capability)) {
- // resource requirements normalized to something other than asked for.
- // LOG @ WARN so users can see why this is happening.
- log.warn("Resource requirements of {} normalized" +
- " from {} to {}", name, capability, normalized);
- }
- return normalized;
- }
-
- /**
- * add a launched container to the node map for status responses
- * @param container id
- * @param node node details
- */
- private void addLaunchedContainer(Container container, RoleInstance node) {
- node.container = container;
- if (node.role == null) {
- throw new RuntimeException(
- "Unknown role for node " + node);
- }
- getLiveContainers().put(node.getContainerId(), node);
- //tell role history
- roleHistory.onContainerStarted(container);
- }
-
- /**
- * container start event
- * @param containerId container that is to be started
- * @return the role instance, or null if there was a problem
- */
- public synchronized RoleInstance onNodeManagerContainerStarted(ContainerId containerId) {
- try {
- return innerOnNodeManagerContainerStarted(containerId);
- } catch (YarnRuntimeException e) {
- log.error("NodeManager callback on started container {} failed",
- containerId,
- e);
- return null;
- }
- }
-
- /**
- * container start event handler -throwing an exception on problems
- * @param containerId container that is to be started
- * @return the role instance
- * @throws RuntimeException on problems
- */
- @VisibleForTesting
- public RoleInstance innerOnNodeManagerContainerStarted(ContainerId containerId) {
- RoleInstance instance = getOwnedContainer(containerId);
- if (instance == null) {
- //serious problem
- throw new YarnRuntimeException("Container not in active containers start "+
- containerId);
- }
- if (instance.role == null) {
- throw new YarnRuntimeException("Component instance has no instance name " +
- instance);
- }
- instance.startTime = now();
- RoleInstance starting = getStartingContainers().remove(containerId);
- if (null == starting) {
- throw new YarnRuntimeException(
- "Container "+ containerId +" is already started");
- }
- instance.state = STATE_LIVE;
- Container container = instance.container;
- addLaunchedContainer(container, instance);
- return instance;
- }
-
- /**
- * update the application state after a failure to start a container.
- * This is perhaps where blacklisting could be most useful: failure
- * to start a container is a sign of a more serious problem
- * than a later exit.
- *
- * -relayed from NMClientAsync.CallbackHandler
- * @param containerId failing container
- * @param thrown what was thrown
- */
- public synchronized void onNodeManagerContainerStartFailed(ContainerId containerId,
- Throwable thrown) {
- removeOwnedContainer(containerId);
- RoleInstance instance = getStartingContainers().remove(containerId);
- if (null != instance) {
- RoleStatus roleStatus = lookupRoleStatus(instance.roleId);
- String text;
- if (null != thrown) {
- text = SliderUtils.stringify(thrown);
- } else {
- text = "container start failure";
- }
- instance.diagnostics = text;
- roleStatus.noteFailed(text);
- getFailedContainers().put(containerId, instance);
- roleHistory.onNodeManagerContainerStartFailed(instance.container);
- incFailedContainers(roleStatus, ContainerOutcome.Failed);
- }
- }
-
- /**
- * Handle node update from the RM. This syncs up the node map with the RM's view
- * @param updatedNodes updated nodes
- */
- public synchronized NodeUpdatedOutcome onNodesUpdated(List<NodeReport> updatedNodes) {
- boolean changed = roleHistory.onNodesUpdated(updatedNodes);
- if (changed) {
- log.info("YARN cluster changed —cancelling current AA requests");
- List<AbstractRMOperation> operations = cancelOutstandingAARequests();
- log.debug("Created {} cancel requests", operations.size());
- return new NodeUpdatedOutcome(true, operations);
- }
- return new NodeUpdatedOutcome(false, new ArrayList<>(0));
- }
-
- /**
- * Return value of the {@link #onNodesUpdated(List)} call.
- */
- public static class NodeUpdatedOutcome {
- public final boolean clusterChanged;
- public final List<AbstractRMOperation> operations;
-
- public NodeUpdatedOutcome(boolean clusterChanged,
- List<AbstractRMOperation> operations) {
- this.clusterChanged = clusterChanged;
- this.operations = operations;
- }
- }
- /**
- * Is a role short lived by the threshold set for this application
- * @param instance instance
- * @return true if the instance is considered short lived
- */
- @VisibleForTesting
- public boolean isShortLived(RoleInstance instance) {
- long time = now();
- long started = instance.startTime;
- boolean shortlived;
- if (started > 0) {
- long duration = time - started;
- shortlived = duration < (startTimeThreshold * 1000);
- log.info("Duration {} and startTimeThreshold {}", duration, startTimeThreshold);
- } else {
- // never even saw a start event
- shortlived = true;
- }
- return shortlived;
- }
-
- /**
- * Current time in milliseconds. Made protected for
- * the option to override it in tests.
- * @return the current time.
- */
- protected long now() {
- return System.currentTimeMillis();
- }
-
- /**
- * This is a very small class to send a multiple result back from
- * the completion operation
- */
- public static class NodeCompletionResult {
- public boolean surplusNode = false;
- public RoleInstance roleInstance;
- // did the container fail for *any* reason?
- public boolean containerFailed = false;
- // detailed outcome on the container failure
- public ContainerOutcome outcome = ContainerOutcome.Completed;
- public int exitStatus = 0;
- public boolean unknownNode = false;
-
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("NodeCompletionResult{");
- sb.append("surplusNode=").append(surplusNode);
- sb.append(", roleInstance=").append(roleInstance);
- sb.append(", exitStatus=").append(exitStatus);
- sb.append(", containerFailed=").append(containerFailed);
- sb.append(", outcome=").append(outcome);
- sb.append(", unknownNode=").append(unknownNode);
- sb.append('}');
- return sb.toString();
- }
- }
-
- /**
- * handle completed node in the CD -move something from the live
- * server list to the completed server list.
- * @param status the node that has just completed
- * @return NodeCompletionResult
- */
- public synchronized NodeCompletionResult onCompletedContainer(
- ContainerStatus status) {
- ContainerId containerId = status.getContainerId();
- NodeCompletionResult result = new NodeCompletionResult();
- RoleInstance roleInstance;
-
- int exitStatus = status.getExitStatus();
- result.exitStatus = exitStatus;
- if (containersBeingReleased.containsKey(containerId)) {
- log.info("Container was queued for release : {}", containerId);
- Container container = containersBeingReleased.remove(containerId);
- RoleStatus roleStatus = lookupRoleStatus(container);
- decRunningContainers(roleStatus);
- incCompletedContainers(roleStatus);
- log.info("decrementing role count for role {} to {}; completed={}",
- roleStatus.getName(),
- roleStatus.getComponentMetrics().containersRunning.value(),
- roleStatus.getComponentMetrics().containersSucceeded.value());
- result.outcome = ContainerOutcome.Completed;
- roleHistory.onReleaseCompleted(container);
-
- } else if (surplusContainers.remove(containerId)) {
- //its a surplus one being purged
- result.surplusNode = true;
- } else {
- // a container has failed or been killed
- // use the exit code to determine the outcome
- result.containerFailed = true;
- result.outcome = ContainerOutcome.fromExitStatus(exitStatus);
-
- roleInstance = removeOwnedContainer(containerId);
- if (roleInstance != null) {
- RoleStatus roleStatus = lookupRoleStatus(roleInstance.roleId);
- incFailedContainers(roleStatus, result.outcome);
- failedContainers.put(containerId, roleInstance);
- } else {
- // the container may have been noted as failed already, so look
- // it up
- roleInstance = failedContainers.get(containerId);
- }
- if (roleInstance != null) {
- int roleId = roleInstance.roleId;
- String rolename = roleInstance.role;
- log.info("Failed container in role[{}] : {}", roleId,
- roleInstance.getCompInstanceName());
- try {
- RoleStatus roleStatus = lookupRoleStatus(roleInstance.roleId);
- decRunningContainers(roleStatus);
- roleStatus.getProviderRole().failedInstances.offer(roleInstance);
- boolean shortLived = isShortLived(roleInstance);
- String message;
- Container failedContainer = roleInstance.container;
-
- //build the failure message
- if (failedContainer != null) {
- String completedLogsUrl = getLogsURLForContainer(failedContainer);
- message = String.format("Failure %s on host %s (%d): %s",
- roleInstance.getContainerId(),
- failedContainer.getNodeId().getHost(),
- exitStatus,
- completedLogsUrl);
- } else {
- message = String.format("Failure %s (%d)", containerId, exitStatus);
- }
- roleStatus.noteFailed(message);
- long failed =
- roleStatus.getComponentMetrics().containersFailed.value();
- log.info("Current count of failed role[{}] {} = {}",
- roleId, rolename, failed);
- if (failedContainer != null) {
- roleHistory.onFailedContainer(failedContainer, shortLived, result.outcome);
- }
-
- } catch (YarnRuntimeException e1) {
- log.error("Failed container of unknown role {}", roleId);
- }
- } else {
- //this isn't a known container.
-
- log.error("Notified of completed container {} that is not in the list" +
- " of active or failed containers", containerId);
- completionOfUnknownContainerEvent.incrementAndGet();
- result.unknownNode = true;
- }
- }
-
- if (result.surplusNode) {
- //a surplus node
- return result;
- }
-
- //record the complete node's details; this pulls it from the livenode set
- //remove the node
- ContainerId id = status.getContainerId();
- log.info("Removing node ID {}", id);
- RoleInstance node = getLiveContainers().remove(id);
- if (node != null) {
- node.state = STATE_DESTROYED;
- node.exitCode = exitStatus;
- node.diagnostics = status.getDiagnostics();
- getCompletedContainers().put(id, node);
- result.roleInstance = node;
- } else {
- // not in the list
- log.warn("Received notification of completion of unknown node {}", id);
- completionOfNodeNotInLiveListEvent.incrementAndGet();
- }
-
- // and the active node list if present
- removeOwnedContainer(containerId);
-
- // finally, verify the node doesn't exist any more
- assert !containersBeingReleased.containsKey(
- containerId) : "container still in release queue";
- assert !getLiveContainers().containsKey(
- containerId) : " container still in live nodes";
- assert getOwnedContainer(containerId) ==
- null : "Container still in active container list";
-
- return result;
- }
-
- /**
- * Get the URL log for a container
- * @param c container
- * @return the URL or "" if it cannot be determined
- */
- protected String getLogsURLForContainer(Container c) {
- if (c==null) {
- return null;
- }
- String user = null;
- try {
- user = SliderUtils.getCurrentUser().getShortUserName();
- } catch (IOException ignored) {
- }
- String completedLogsUrl = "";
- String url = logServerURL;
- if (user != null && SliderUtils.isSet(url)) {
- completedLogsUrl = url
- + "/" + c.getNodeId() + "/" + c.getId() + "/ctx/" + user;
- }
- return completedLogsUrl;
- }
-
- /**
- * Return the percentage done that Slider is to have YARN display in its
- * Web UI
- * @return an number from 0 to 100
- */
- public synchronized float getApplicationProgressPercentage() {
- float percentage;
- long desired = 0;
- float actual = 0;
- for (RoleStatus role : getRoleStatusMap().values()) {
- desired += role.getDesired();
- actual += role.getRunning();
- }
- if (desired == 0) {
- percentage = 100;
- } else {
- percentage = actual / desired;
- }
- return percentage;
- }
-
-
- /**
- * Update the cluster description with the current application state
- */
-
- public synchronized Application refreshClusterStatus() {
-
- //TODO replace ClusterDescription with Application + related statistics
- //TODO build container stats
- app.setState(ApplicationState.STARTED);
- return app;
- }
-
- /**
- * get application liveness information
- * @return a snapshot of the current liveness information
- */
- public ApplicationLivenessInformation getApplicationLivenessInformation() {
- ApplicationLivenessInformation li = new ApplicationLivenessInformation();
- RoleStatistics stats = getRoleStatistics();
- int outstanding = (int)(stats.desired - stats.actual);
- li.requestsOutstanding = outstanding;
- li.allRequestsSatisfied = outstanding <= 0;
- return li;
- }
-
-
- /**
- * Get the aggregate statistics across all roles
- * @return role statistics
- */
- public RoleStatistics getRoleStatistics() {
- RoleStatistics stats = new RoleStatistics();
- for (RoleStatus role : getRoleStatusMap().values()) {
- stats.add(role.getStatistics());
- }
- return stats;
- }
-
- /**
- * Get a snapshot of component information.
- * <p>
- * This does <i>not</i> include any container list, which
- * is more expensive to create.
- * @return a map of current role status values.
- */
- public Map<String, ComponentInformation> getComponentInfoSnapshot() {
-
- Map<Integer, RoleStatus> statusMap = getRoleStatusMap();
- Map<String, ComponentInformation> results = new HashMap<>(
- statusMap.size());
-
- for (RoleStatus status : statusMap.values()) {
- String name = status.getName();
- ComponentInformation info = status.serialize();
- results.put(name, info);
- }
- return results;
- }
-
- public synchronized AbstractRMOperation updateBlacklist() {
- UpdateBlacklistOperation blacklistOperation =
- roleHistory.updateBlacklist(getRoleStatusMap().values());
- if (blacklistOperation != null) {
- log.info("Updating {}", blacklistOperation);
- }
- return blacklistOperation;
- }
-
- /**
- * Look at where the current node state is -and whether it should be changed
- */
- public synchronized List<AbstractRMOperation> reviewRequestAndReleaseNodes()
- throws SliderInternalStateException, TriggerClusterTeardownException {
- log.info("in reviewRequestAndReleaseNodes()");
- List<AbstractRMOperation> allOperations = new ArrayList<>();
- AbstractRMOperation blacklistOperation = updateBlacklist();
- if (blacklistOperation != null) {
- allOperations.add(blacklistOperation);
- }
- for (RoleStatus roleStatus : getRoleStatusMap().values()) {
- if (!roleStatus.isExcludeFromFlexing() &&
- areDependenciesReady(roleStatus)) {
- List<AbstractRMOperation> operations = reviewOneRole(roleStatus);
- allOperations.addAll(operations);
- }
- }
- return allOperations;
- }
-
- @VisibleForTesting
- public boolean areDependenciesReady(RoleStatus roleStatus) {
- List<String> dependencies = roleStatus.getProviderRole().component
- .getDependencies();
- if (SliderUtils.isEmpty(dependencies)) {
- return true;
- }
- for (String dependency : dependencies) {
- ProviderRole providerRole = roles.get(dependency);
- if (providerRole == null) {
- log.error("Couldn't find dependency {} for {} (should never happen)",
- dependency, roleStatus.getName());
- continue;
- }
- RoleStatus other = getRoleStatusMap().get(providerRole.id);
- if (other.getRunning() < other.getDesired()) {
- log.info("Dependency {} not satisfied for {}, only {} of {} instances" +
- " running", dependency, roleStatus.getName(), other.getRunning(),
- other.getDesired());
- return false;
- }
- if (providerRole.probe == null) {
- continue;
- }
- List<RoleInstance> dependencyInstances = enumLiveNodesInRole(
- providerRole.name);
- if (dependencyInstances.size() < other.getDesired()) {
- log.info("Dependency {} not satisfied for {}, only {} of {} instances" +
- " live", dependency, roleStatus.getName(),
- dependencyInstances.size(), other.getDesired());
- return false;
- }
- for (RoleInstance instance : dependencyInstances) {
- if (instance.state != STATE_READY) {
- return false;
- }
- }
- }
- return true;
- }
-
- /**
- * Check the "recent" failure threshold for a role
- * @param role role to examine
- * @throws TriggerClusterTeardownException if the role
- * has failed too many times
- */
- private void checkFailureThreshold(RoleStatus role)
- throws TriggerClusterTeardownException {
- long failures = role.getFailedRecently();
- int threshold = getFailureThresholdForRole(role);
- if (log.isDebugEnabled() && failures > 0) {
- log.debug("Failure count of component: {}: {}, threshold={}",
- role.getName(), failures, threshold);
- }
-
- if (threshold > 0 && failures > threshold) {
- throw new TriggerClusterTeardownException(
- SliderExitCodes.EXIT_DEPLOYMENT_FAILED, FinalApplicationStatus.FAILED,
- ErrorStrings.E_UNSTABLE_CLUSTER
- + " - failed with component %s failed 'recently' %d times;"
- + " threshold is %d - last failure: %s", role.getName(),
- role.getFailedRecently(), threshold, role.getFailureMessage());
- }
- }
-
- /**
- * Get the failure threshold for a specific role, falling back to
- * the global one if not
- * @param roleStatus role
- * @return the threshold for failures
- */
- private int getFailureThresholdForRole(RoleStatus roleStatus) {
- return (int) roleStatus.getProviderRole().component.getConfiguration()
- .getPropertyLong(CONTAINER_FAILURE_THRESHOLD,
- failureThreshold);
- }
-
-
- /**
- * Reset the "recent" failure counts of all roles
- */
- public void resetFailureCounts() {
- for (RoleStatus roleStatus : getRoleStatusMap().values()) {
- long failed = roleStatus.resetFailedRecently();
- log.info("Resetting failure count of {}; was {}", roleStatus.getName(),
- failed);
-
- }
- roleHistory.resetFailedRecently();
- }
-
- /**
- * Escalate operation as triggered by external timer.
- * @return a (usually empty) list of cancel/request operations.
- */
- public List<AbstractRMOperation> escalateOutstandingRequests() {
- return roleHistory.escalateOutstandingRequests();
- }
-
- /**
- * Cancel any outstanding AA Requests, building up the list of ops to
- * cancel, removing them from RoleHistory structures and the RoleStatus
- * entries.
- * @return a (usually empty) list of cancel/request operations.
- */
- public synchronized List<AbstractRMOperation> cancelOutstandingAARequests() {
- // get the list of cancel operations
- List<AbstractRMOperation> operations = roleHistory.cancelOutstandingAARequests();
- for (RoleStatus roleStatus : roleStatusMap.values()) {
- if (roleStatus.isAARequestOutstanding()) {
- log.info("Cancelling outstanding AA request for {}", roleStatus);
- roleStatus.cancelOutstandingAARequest();
- }
- }
- return operations;
- }
-
- public synchronized boolean monitorComponentInstances() {
- boolean hasChanged = false;
- for (RoleInstance instance : getLiveContainers().values()) {
- if (instance.providerRole.probe == null) {
- continue;
- }
- boolean ready = instance.providerRole.probe.ping(null).isSuccess();
- if (ready) {
- if (instance.state != STATE_READY) {
- instance.state = STATE_READY;
- hasChanged = true;
- log.info("State of {} changed to ready", instance.role);
- }
- } else {
- if (instance.state == STATE_READY) {
- instance.state = STATE_NOT_READY;
- hasChanged = true;
- log.info("State of {} changed from ready to not ready", instance
- .role);
- }
- }
- }
- return hasChanged;
- }
-
- /**
- * Look at the allocation status of one role, and trigger add/release
- * actions if the number of desired role instances doesn't equal
- * (actual + pending).
- * <p>
- * MUST be executed from within a synchronized method
- * <p>
- * @param role role
- * @return a list of operations
- * @throws SliderInternalStateException if the operation reveals that
- * the internal state of the application is inconsistent.
- */
- @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
- private List<AbstractRMOperation> reviewOneRole(RoleStatus role)
- throws SliderInternalStateException, TriggerClusterTeardownException {
- List<AbstractRMOperation> operations = new ArrayList<>();
- long delta;
- long expected;
- String name = role.getName();
- synchronized (role) {
- delta = role.getDelta();
- expected = role.getDesired();
- }
-
- log.info("Reviewing " + role.getName() + ": " + role.getComponentMetrics());
- checkFailureThreshold(role);
-
- if (expected < 0 ) {
- // negative value: fail
- throw new TriggerClusterTeardownException(
- SliderExitCodes.EXIT_DEPLOYMENT_FAILED,
- FinalApplicationStatus.FAILED,
- "Negative component count of %d desired for component %s",
- expected, role);
- }
-
- if (delta > 0) {
- // more workers needed than we have -ask for more
- log.info("{}: Asking for {} more nodes(s) for a total of {} ", name, delta, expected);
-
- if (role.isAntiAffinePlacement()) {
- long pending = delta;
- if (roleHistory.canPlaceAANodes()) {
- // build one only if there is none outstanding, the role history knows
- // enough about the cluster to ask, and there is somewhere to place
- // the node
- if (!role.isAARequestOutstanding()) {
- // no outstanding AA; try to place things
- AMRMClient.ContainerRequest request = createAAContainerRequest(role);
- if (request != null) {
- pending--;
- log.info("Starting an anti-affine request sequence for {} nodes; pending={}",
- delta, pending);
- addContainerRequest(operations, request, role);
- } else {
- log.info("No location for anti-affine request");
- }
- }
- } else {
- log.warn("Awaiting node map before generating anti-affinity requests");
- }
- log.info("Setting pending to {}", pending);
- //TODO
- role.setAAPending(pending);
- } else {
-
- for (int i = 0; i < delta; i++) {
- //get the role history to select a suitable node, if available
- addContainerRequest(operations, createContainerRequest(role), role);
- }
- }
- } else if (delta < 0) {
- log.info("{}: Asking for {} fewer node(s) for a total of {}", name,
- -delta,
- expected);
- // reduce the number expected (i.e. subtract the delta)
- long excess = -delta;
-
- // how many requests are outstanding? for AA roles, this includes pending
- long outstandingRequests = role.getRequested() + role.getAAPending();
- if (outstandingRequests > 0) {
- // outstanding requests.
- int toCancel = (int)Math.min(outstandingRequests, excess);
-
- int pendingCancelled = 0;
- if (role.getAAPending() > 0) {
- pendingCancelled = decAAPendingToFloor(role, toCancel);
- }
- int remainingToCancel = toCancel - pendingCancelled;
-
- // Delegate to Role History
- List<AbstractRMOperation> cancellations = roleHistory
- .cancelRequestsForRole(role, remainingToCancel);
- log.info("Found {} outstanding requests to cancel", cancellations.size());
- operations.addAll(cancellations);
- if (remainingToCancel != cancellations.size()) {
- log.error("Tracking of outstanding requests is not in sync with the summary statistics:" +
- " expected to be able to cancel {} requests, but got {}",
- remainingToCancel, cancellations.size());
- }
-
- int requestCancelled = decRequestedContainersToFloor(role,
- remainingToCancel);
- excess -= pendingCancelled;
- excess -= requestCancelled;
- assert excess >= 0 : "Attempted to cancel too many requests";
- log.info("Submitted {} cancellations, leaving {} to release",
- pendingCancelled + requestCancelled, excess);
- if (excess == 0) {
- log.info("After cancelling requests, application is now at desired size");
- }
- }
-
- // after the cancellation there may be no excess
- if (excess > 0) {
-
- // there's an excess, so more to cancel
- // get the nodes to release
- int roleId = role.getKey();
-
- // enum all active nodes that aren't being released
- List<RoleInstance> containersToRelease = enumNodesWithRoleId(roleId, true);
- if (containersToRelease.isEmpty()) {
- log.info("No containers for component {}", roleId);
- }
-
- // filter out all release-in-progress nodes
- ListIterator<RoleInstance> li = containersToRelease.listIterator();
- while (li.hasNext()) {
- RoleInstance next = li.next();
- if (next.released) {
- li.remove();
- }
- }
-
- // warn if the desired state can't be reached
- int numberAvailableForRelease = containersToRelease.size();
- if (numberAvailableForRelease < excess) {
- log.warn("Not enough containers to release, have {} and need {} more",
- numberAvailableForRelease,
- excess - numberAvailableForRelease);
- }
-
- // ask the release selector to sort the targets
- containersToRelease = containerReleaseSelector.sortCandidates(
- roleId,
- containersToRelease);
-
- // crop to the excess
- List<RoleInstance> finalCandidates = (excess < numberAvailableForRelease)
- ? containersToRelease.subList(0, (int)excess)
- : containersToRelease;
-
- // then build up a release operation, logging each container as released
- for (RoleInstance possible : finalCandidates) {
- log.info("Targeting for release: {}", possible);
- containerReleaseSubmitted(possible.container);
- role.getProviderRole().failedInstances.offer(possible);
- operations.add(new ContainerReleaseOperation(possible.getContainerId()));
- }
- }
-
- } else {
- // actual + requested == desired
- // there's a special case here: clear all pending AA requests
- if (role.getAAPending() > 0) {
- log.debug("Clearing outstanding pending AA requests");
- role.setAAPending(0);
- }
- }
-
- // there's now a list of operations to execute
- log.debug("operations scheduled: {}; updated role: {}", operations.size(), role);
- return operations;
- }
-
- /**
- * Add a container request if the request is non-null
- * @param operations operations to add the entry to
- * @param containerAsk what to ask for
- * @return true if a request was added
- */
- private boolean addContainerRequest(List<AbstractRMOperation> operations,
- AMRMClient.ContainerRequest containerAsk, RoleStatus role) {
- if (containerAsk != null) {
- log.info("Container ask is {} and label = {}", containerAsk,
- containerAsk.getNodeLabelExpression());
- int askMemory = containerAsk.getCapability().getMemory();
- if (askMemory > this.containerMaxMemory) {
- log.warn("Memory requested: {} > max of {}", askMemory, containerMaxMemory);
- }
- operations.add(new ContainerRequestOperation(containerAsk));
- incRequestedContainers(role);
- return true;
- } else {
- return false;
- }
- }
-
- /**
- * Releases a container based on container id
- * @param containerId
- * @return
- * @throws SliderInternalStateException
- */
- public List<AbstractRMOperation> releaseContainer(ContainerId containerId)
- throws SliderInternalStateException {
- List<AbstractRMOperation> operations = new ArrayList<AbstractRMOperation>();
- List<RoleInstance> activeRoleInstances = cloneOwnedContainerList();
- for (RoleInstance role : activeRoleInstances) {
- if (role.container.getId().equals(containerId)) {
- containerReleaseSubmitted(role.container);
- operations.add(new ContainerReleaseOperation(role.getContainerId()));
- }
- }
-
- return operations;
- }
-
- /**
- * Release all containers.
- * @return a list of operations to execute
- */
- public synchronized List<AbstractRMOperation> releaseAllContainers() {
-
- Collection<RoleInstance> targets = cloneOwnedContainerList();
- log.info("Releasing {} containers", targets.size());
- List<AbstractRMOperation> operations =
- new ArrayList<>(targets.size());
- for (RoleInstance instance : targets) {
- if (instance.roleId == SliderKeys.ROLE_AM_PRIORITY_INDEX) {
- // don't worry about the AM
- continue;
- }
- Container possible = instance.container;
- ContainerId id = possible.getId();
- if (!instance.released) {
- String url = getLogsURLForContainer(possible);
- log.info("Releasing container. Log: " + url);
- try {
- containerReleaseSubmitted(possible);
- // update during finish call
- if (serviceTimelinePublisher != null) {
- serviceTimelinePublisher.componentInstanceFinished(instance);
- }
- } catch (SliderInternalStateException e) {
- log.warn("when releasing container {} :", possible, e);
- }
- operations.add(new ContainerReleaseOperation(id));
- }
- }
- return operations;
- }
-
- /**
- * Event handler for allocated containers: builds up the lists
- * of assignment actions (what to run where), and possibly
- * a list of operations to perform
- * @param allocatedContainers the containers allocated
- * @param assignments the assignments of roles to containers
- * @param operations any allocation or release operations
- */
- public synchronized void onContainersAllocated(
- List<Container> allocatedContainers,
- List<ContainerAssignment> assignments,
- List<AbstractRMOperation> operations) {
- assignments.clear();
- operations.clear();
- List<Container> ordered = roleHistory.prepareAllocationList(allocatedContainers);
- log.info("onContainersAllocated(): Total containers allocated = {}", ordered.size());
- for (Container container : ordered) {
- final NodeId nodeId = container.getNodeId();
- String containerHostInfo = nodeId.getHost() + ":" + nodeId.getPort();
- //get the role
- final ContainerId cid = container.getId();
- final RoleStatus role = lookupRoleStatus(container);
-
- //inc allocated count -this may need to be dropped in a moment,
- // but us needed to update the logic below
- MutableGaugeInt containersRunning = role.getComponentMetrics().containersRunning;
- incRunningContainers(role);
- final long allocated = containersRunning.value();
- final long desired = role.getDesired();
-
- final String roleName = role.getName();
- final ContainerAllocationResults allocation =
- roleHistory.onContainerAllocated(container, desired, allocated);
- final ContainerAllocationOutcome outcome = allocation.outcome;
-
- // add all requests to the operations list
- operations.addAll(allocation.operations);
-
- //look for condition where we get more back than we asked
- if (allocated > desired) {
- log.info("Discarding surplus {} container {} on {}", roleName, cid, containerHostInfo);
- operations.add(new ContainerReleaseOperation(cid));
- //register as a surplus node
- surplusContainers.add(cid);
- role.getComponentMetrics().surplusContainers.incr();
- containersRunning.decr();
- } else {
- decRequestedContainers(role);
- log.info("Assigning role {} to container" + " {}," + " on {}:{},",
- roleName, cid, nodeId.getHost(), nodeId.getPort());
-
- assignments.add(new ContainerAssignment(container, role, outcome));
- //add to the history
- roleHistory.onContainerAssigned(container);
- // now for AA requests, add some more
- if (role.isAntiAffinePlacement()) {
- role.completeOutstandingAARequest();
- // check invariants. The new node must become unavailable.
- NodeInstance node = roleHistory.getOrCreateNodeInstance(container);
- if (node.canHost(role.getKey(), role.getLabelExpression())) {
- log.error("Assigned node still declares as available {}", node.toFullString() );
- }
- if (role.getAAPending() > 0) {
- // still an outstanding AA request: need to issue a new one.
- log.info("Asking for next container for AA role {}", roleName);
- if (!addContainerRequest(operations, createAAContainerRequest(role),
- role)) {
- log.info("No capacity in cluster for new requests");
- } else {
- role.decAAPending();
- }
- log.debug("Current AA role status {}", role);
- } else {
- log.info("AA request sequence completed for role {}", role);
- }
- }
-
- }
- }
- }
-
- /**
- * Event handler for the list of active containers on restart.
- * Sets the info key {@link StatusKeys#INFO_CONTAINERS_AM_RESTART}
- * to the size of the list passed down (and does not set it if none were)
- * @param liveContainers the containers allocated
- * @return true if a rebuild took place (even if size 0)
- * @throws RuntimeException on problems
- */
- private boolean rebuildModelFromRestart(List<Container> liveContainers)
- throws BadClusterStateException {
- if (liveContainers == null) {
- return false;
- }
- for (Container container : liveContainers) {
- addRestartedContainer(container);
- }
- app.setNumberOfRunningContainers((long)liveContainers.size());
- return true;
- }
-
- /**
- * Add a restarted container by walking it through the create/submit/start
- * lifecycle, so building up the internal structures
- * @param container container that was running before the AM restarted
- * @throws RuntimeException on problems
- */
- private void addRestartedContainer(Container container)
- throws BadClusterStateException {
- String containerHostInfo = container.getNodeId().getHost()
- + ":" +
- container.getNodeId().getPort();
- // get the container ID
- ContainerId cid = container.getId();
-
- // get the role
- int roleId = ContainerPriority.extractRole(container);
- RoleStatus role = lookupRoleStatus(roleId);
- // increment its count
- incRunningContainers(role);
- String roleName = role.getName();
-
- log.info("Rebuilding container {} in role {} on {},",
- cid,
- roleName,
- containerHostInfo);
-
- //update app state internal structures and maps
-
- //TODO recover the component instance name from zk registry ?
- RoleInstance instance = new RoleInstance(container);
- instance.command = roleName;
- instance.role = roleName;
- instance.roleId = roleId;
- instance.environment = new String[0];
- instance.container = container;
- instance.createTime = now();
- instance.state = STATE_LIVE;
- instance.appVersion = SliderKeys.APP_VERSION_UNKNOWN;
- putOwnedContainer(cid, instance);
- //role history gets told
- roleHistory.onContainerAssigned(container);
- // pretend the container has just had its start actions submitted
- containerStartSubmitted(container, instance);
- // now pretend it has just started
- innerOnNodeManagerContainerStarted(cid);
- }
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder("AppState{");
- sb.append("applicationLive=").append(applicationLive);
- sb.append(", live nodes=").append(liveNodes.size());
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * Build map of role ID-> name
- * @return
- */
- public Map<Integer, String> buildNamingMap() {
- Map<Integer, RoleStatus> statusMap = getRoleStatusMap();
- Map<Integer, String> naming = new HashMap<>(statusMap.size());
- for (Map.Entry<Integer, RoleStatus> entry : statusMap.entrySet()) {
- naming.put(entry.getKey(), entry.getValue().getName());
- }
- return naming;
- }
-
- public void setServiceTimelinePublisher(ServiceTimelinePublisher serviceTimelinePublisher) {
- this.serviceTimelinePublisher = serviceTimelinePublisher;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
deleted file mode 100644
index 439a256..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.providers.ProviderRole;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Binding information for application states; designed to be extensible
- * so that tests don't have to be massivley reworked when new arguments
- * are added.
- */
-public class AppStateBindingInfo {
- public Configuration serviceConfig = new Configuration();
- public Application application = null;
- public List<ProviderRole> roles = new ArrayList<>();
- public FileSystem fs;
- public Path historyPath;
- public List<Container> liveContainers = new ArrayList<>(0);
- public ContainerReleaseSelector releaseSelector = new SimpleReleaseSelector();
- public String serviceHdfsDir = "";
- /** node reports off the RM. */
- public List<NodeReport> nodeReports = new ArrayList<>(0);
-
- public void validate() throws IllegalArgumentException {
- Preconditions.checkArgument(serviceConfig != null, "null appmasterConfig");
- Preconditions.checkArgument(releaseSelector != null, "null releaseSelector");
- Preconditions.checkArgument(roles != null, "null providerRoles");
- Preconditions.checkArgument(fs != null, "null fs");
- Preconditions.checkArgument(historyPath != null, "null historyDir");
- Preconditions.checkArgument(nodeReports != null, "null nodeReports");
- Preconditions.checkArgument(application != null, "null application");
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
deleted file mode 100644
index 5b3a93c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-/**
- * Outcome of the assignment
- */
-public enum ContainerAllocationOutcome {
- /**
- * There wasn't a request for this
- */
- Unallocated,
-
- /**
- * Open placement
- */
- Open,
-
- /**
- * Allocated explicitly where requested
- */
- Placed,
-
- /**
- * This was an escalated placement
- */
- Escalated
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationResults.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationResults.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationResults.java
deleted file mode 100644
index e80639e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationResults.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * This is just a tuple of the outcome of a container allocation
- */
-public class ContainerAllocationResults {
-
- /**
- * What was the outcome of this allocation: placed, escalated, ...
- */
- public ContainerAllocationOutcome outcome;
-
- /**
- * The outstanding request which originated this.
- * This will be null if the outcome is {@link ContainerAllocationOutcome#Unallocated}
- * as it wasn't expected.
- */
- public OutstandingRequest origin;
-
- /**
- * A possibly empty list of requests to add to the follow-up actions
- */
- public List<AbstractRMOperation> operations = new ArrayList<>(0);
-
- public ContainerAllocationResults() {
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[73/86] [abbrv] hadoop git commit: YARN-7113. Clean up packaging and
dependencies for yarn-native-services. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
index 6e6f4dd..7e53d18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
@@ -24,35 +24,17 @@ import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.GlobFilter;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.service.client.params.Arguments;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.zookeeper.server.util.KerberosUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,32 +45,19 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
-import java.io.Serializable;
-import java.net.InetSocketAddress;
import java.net.ServerSocket;
-import java.net.Socket;
import java.net.URL;
import java.net.URLDecoder;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Pattern;
import java.util.zip.GZIPOutputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
/**
* These are slider-specific Util methods
@@ -97,41 +66,6 @@ public final class SliderUtils {
private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
- /**
- * Atomic bool to track whether or not process security has already been
- * turned on (prevents re-entrancy)
- */
- private static final AtomicBoolean processSecurityAlreadyInitialized =
- new AtomicBoolean(false);
- public static final String JAVA_SECURITY_KRB5_REALM =
- "java.security.krb5.realm";
- public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
-
- /**
- * Winutils
- */
- public static final String WINUTILS = "WINUTILS.EXE";
- /**
- * name of openssl program
- */
- public static final String OPENSSL = "openssl";
-
- /**
- * name of python program
- */
- public static final String PYTHON = "python";
-
- /**
- * type of docker standalone service
- */
- public static final String DOCKER = "docker";
- /**
- * type of docker on yarn service
- */
- public static final String DOCKER_YARN = "yarn_docker";
-
- public static final int NODE_LIST_LIMIT = 10;
-
private SliderUtils() {
}
@@ -153,96 +87,6 @@ public final class SliderUtils {
}
/**
- * Probe for a collection existing and not being empty
- * @param l collection
- * @return true if the reference is valid and it contains entries
- */
-
- public static boolean isNotEmpty(Collection l) {
- return l != null && !l.isEmpty();
- }
-
- /**
- * Probe for a map existing and not being empty
- * @param m map
- * @return true if the reference is valid and it contains map entries
- */
- public static boolean isNotEmpty(Map m) {
- return m != null && !m.isEmpty();
- }
-
- /*
- * Validates whether num is an integer
- * @param num
- * @param msg the message to be shown in exception
- */
- @SuppressWarnings("ResultOfMethodCallIgnored")
- private static void validateNumber(String num, String msg) throws
- BadConfigException {
- try {
- Integer.parseInt(num);
- } catch (NumberFormatException nfe) {
- throw new BadConfigException(msg + num);
- }
- }
-
- /*
- * Translates the trailing JVM heapsize unit: g, G, m, M
- * This assumes designated unit of 'm'
- * @param heapsize
- * @return heapsize in MB
- */
- public static String translateTrailingHeapUnit(String heapsize) throws
- BadConfigException {
- String errMsg = "Bad heapsize: ";
- if (heapsize.endsWith("m") || heapsize.endsWith("M")) {
- String num = heapsize.substring(0, heapsize.length() - 1);
- validateNumber(num, errMsg);
- return num;
- }
- if (heapsize.endsWith("g") || heapsize.endsWith("G")) {
- String num = heapsize.substring(0, heapsize.length() - 1) + "000";
- validateNumber(num, errMsg);
- return num;
- }
- // check if specified heap size is a number
- validateNumber(heapsize, errMsg);
- return heapsize;
- }
-
- /**
- * recursive directory delete
- * @param dir dir to delete
- * @throws IOException on any problem
- */
- public static void deleteDirectoryTree(File dir) throws IOException {
- if (dir.exists()) {
- if (dir.isDirectory()) {
- log.info("Cleaning up {}", dir);
- //delete the children
- File[] files = dir.listFiles();
- if (files == null) {
- throw new IOException("listfiles() failed for " + dir);
- }
- for (File file : files) {
- log.info("deleting {}", file);
- if (!file.delete()) {
- log.warn("Unable to delete " + file);
- }
- }
- if (!dir.delete()) {
- log.warn("Unable to delete " + dir);
- }
- } else {
- throw new IOException("Not a directory " + dir);
- }
- } else {
- //not found, do nothing
- log.debug("No output dir yet");
- }
- }
-
- /**
* Find a containing JAR
* @param clazz class to find
* @return the file
@@ -298,127 +142,6 @@ public final class SliderUtils {
return null;
}
- public static void checkPort(String hostname, int port, int connectTimeout)
- throws IOException {
- InetSocketAddress addr = new InetSocketAddress(hostname, port);
- checkPort(hostname, addr, connectTimeout);
- }
-
- @SuppressWarnings("SocketOpenedButNotSafelyClosed")
- public static void checkPort(String name,
- InetSocketAddress address,
- int connectTimeout)
- throws IOException {
- try(Socket socket = new Socket()) {
- socket.connect(address, connectTimeout);
- } catch (Exception e) {
- throw new IOException("Failed to connect to " + name
- + " at " + address
- + " after " + connectTimeout + "milliseconds"
- + ": " + e,
- e);
- }
- }
-
- public static void checkURL(String name, String url, int timeout) throws
- IOException {
- InetSocketAddress address = NetUtils.createSocketAddr(url);
- checkPort(name, address, timeout);
- }
-
- /**
- * A required file
- * @param role role of the file (for errors)
- * @param filename the filename
- * @throws ExitUtil.ExitException if the file is missing
- * @return the file
- */
- public static File requiredFile(String filename, String role) throws
- IOException {
- if (filename.isEmpty()) {
- throw new ExitUtil.ExitException(-1, role + " file not defined");
- }
- File file = new File(filename);
- if (!file.exists()) {
- throw new ExitUtil.ExitException(-1,
- role + " file not found: " +
- file.getCanonicalPath());
- }
- return file;
- }
-
- private static final PatternValidator clusternamePattern
- = new PatternValidator("[a-z][a-z0-9_-]*");
-
- /**
- * Normalize a cluster name then verify that it is valid
- * @param name proposed cluster name
- * @return true iff it is valid
- */
- public static boolean isClusternameValid(String name) {
- return name != null && clusternamePattern.matches(name);
- }
-
- /**
- * Copy a directory to a new FS -both paths must be qualified. If
- * a directory needs to be created, supplied permissions can override
- * the default values. Existing directories are not touched
- * @param conf conf file
- * @param srcDirPath src dir
- * @param destDirPath dest dir
- * @param permission permission for the dest directory; null means "default"
- * @return # of files copies
- */
- @SuppressWarnings("deprecation")
- public static int copyDirectory(Configuration conf,
- Path srcDirPath,
- Path destDirPath,
- FsPermission permission) throws
- IOException,
- BadClusterStateException {
- FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf);
- FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf);
- //list all paths in the src.
- if (!srcFS.exists(srcDirPath)) {
- throw new FileNotFoundException("Source dir not found " + srcDirPath);
- }
- if (!srcFS.isDirectory(srcDirPath)) {
- throw new FileNotFoundException(
- "Source dir not a directory " + srcDirPath);
- }
- GlobFilter dotFilter = new GlobFilter("[!.]*");
- FileStatus[] entries = srcFS.listStatus(srcDirPath, dotFilter);
- int srcFileCount = entries.length;
- if (srcFileCount == 0) {
- return 0;
- }
- if (permission == null) {
- permission = FsPermission.getDirDefault();
- }
- if (!destFS.exists(destDirPath)) {
- new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath,
- permission);
- }
- Path[] sourcePaths = new Path[srcFileCount];
- for (int i = 0; i < srcFileCount; i++) {
- FileStatus e = entries[i];
- Path srcFile = e.getPath();
- if (srcFS.isDirectory(srcFile)) {
- String msg = "Configuration dir " + srcDirPath
- + " contains a directory " + srcFile;
- log.warn(msg);
- throw new IOException(msg);
- }
- log.debug("copying src conf file {}", srcFile);
- sourcePaths[i] = srcFile;
- }
- log.debug("Copying {} files from {} to dest {}", srcFileCount,
- srcDirPath,
- destDirPath);
- FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf);
- return srcFileCount;
- }
-
/**
* Copy a file to a new FS -both paths must be qualified.
* @param conf conf file
@@ -497,19 +220,6 @@ public final class SliderUtils {
/**
* Join an array of strings with a separator that appears after every
- * instance in the list -including at the end
- * @param collection strings
- * @param separator separator string
- * @return the joined entries
- */
- public static String join(String[] collection, String separator) {
- return join(collection, separator, true);
-
-
- }
-
- /**
- * Join an array of strings with a separator that appears after every
* instance in the list -optionally at the end
* @param collection strings
* @param separator separator string
@@ -522,30 +232,6 @@ public final class SliderUtils {
}
/**
- * Join an array of strings with a separator that appears after every
- * instance in the list -except at the end
- * @param collection strings
- * @param separator separator string
- * @return the list
- */
- public static String joinWithInnerSeparator(String separator,
- Object... collection) {
- StringBuilder b = new StringBuilder();
- boolean first = true;
-
- for (Object o : collection) {
- if (first) {
- first = false;
- } else {
- b.append(separator);
- }
- b.append(o.toString());
- b.append(separator);
- }
- return b.toString();
- }
-
- /**
* Resolve a mandatory environment variable
* @param key env var
* @return the resolved value
@@ -560,170 +246,6 @@ public final class SliderUtils {
return v;
}
- public static String appReportToString(ApplicationReport r,
- String separator) {
- StringBuilder builder = new StringBuilder(512);
- builder.append("service ")
- .append(
- r.getName())
- .append("/")
- .append(r.getApplicationType())
- .append(separator);
- Set<String> tags = r.getApplicationTags();
- if (!tags.isEmpty()) {
- for (String tag : tags) {
- builder.append(tag).append(separator);
- }
- }
- DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
- dateFormat.setTimeZone(TimeZone.getDefault());
- builder.append("state: ").append(r.getYarnApplicationState());
- String trackingUrl = r.getTrackingUrl();
- if (isSet(trackingUrl)) {
- builder.append(separator).append("URL: ").append(trackingUrl);
- }
- builder.append(separator)
- .append("Started: ")
- .append(dateFormat.format(new Date(r.getStartTime())));
- long finishTime = r.getFinishTime();
- if (finishTime > 0) {
- builder.append(separator)
- .append("Finished: ")
- .append(dateFormat.format(new Date(finishTime)));
- }
- String rpcHost = r.getHost();
- if (!isSet(rpcHost)) {
- builder.append(separator)
- .append("RPC :")
- .append(rpcHost)
- .append(':')
- .append(r.getRpcPort());
- }
- String diagnostics = r.getDiagnostics();
- if (!isSet(diagnostics)) {
- builder.append(separator).append("Diagnostics :").append(diagnostics);
- }
- return builder.toString();
- }
-
- /**
- * Filter a string value given a single filter
- *
- * @param value
- * the string value to check
- * @param filter
- * a single string filter
- * @return return true if value should be trapped, false if it should be let
- * through
- */
- public static boolean filter(String value, String filter) {
- return !(StringUtils.isEmpty(filter) || filter.equals(value));
- }
-
- /**
- * Filter a string value given a set of filters
- *
- * @param value
- * the string value to check
- * @param filters
- * a set of string filters
- * @return return true if value should be trapped, false if it should be let
- * through
- */
- public static boolean filter(String value, Set<String> filters) {
- return !(filters.isEmpty() || filters.contains(value));
- }
-
- /**
- * Sorts the given list of service reports, most recently started
- * or finished instance first.
- *
- * @param instances list of instances
- */
- public static void sortApplicationsByMostRecent(List<ApplicationReport> instances) {
- Collections.sort(instances, new MostRecentlyStartedOrFinishedFirst());
- }
-
- /**
- * Sorts the given list of service reports
- * Finished instances are ordered by finished time and running/accepted instances are
- * ordered by start time
- * Finally Instance are order by finished instances coming after running instances
- *
- * @param instances list of instances
- */
- public static void sortApplicationReport(List<ApplicationReport> instances) {
- if (instances.size() <= 1) {
- return;
- }
- List<ApplicationReport> nonLiveInstance =
- new ArrayList<>(instances.size());
- List<ApplicationReport> liveInstance =
- new ArrayList<>(instances.size());
-
- for (ApplicationReport report : instances) {
- if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
- ||
- report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
- liveInstance.add(report);
- } else {
- nonLiveInstance.add(report);
- }
- }
-
- if (liveInstance.size() > 1) {
- Collections.sort(liveInstance, new MostRecentlyStartedAppFirst());
- }
- if (nonLiveInstance.size() > 1) {
- Collections.sort(nonLiveInstance, new MostRecentAppFinishFirst());
- }
- instances.clear();
- instances.addAll(liveInstance);
- instances.addAll(nonLiveInstance);
- }
-
- /**
- * Merge in one map to another -all entries in the second map are
- * merged into the first -overwriting any duplicate keys.
- * @param first first map -the updated one.
- * @param second the map that is merged in
- * @return the first map
- */
- public static Map<String, String> mergeMap(Map<String, String> first,
- Map<String, String> second) {
- first.putAll(second);
- return first;
- }
-
- /**
- * Merge a set of entries into a map. This will take the entryset of
- * a map, or a Hadoop collection itself
- * @param dest destination
- * @param entries entries
- * @return dest -with the entries merged in
- */
- public static Map<String, String> mergeEntries(Map<String, String> dest,
- Iterable<Map.Entry<String, String>> entries) {
- for (Map.Entry<String, String> entry : entries) {
- dest.put(entry.getKey(), entry.getValue());
- }
- return dest;
- }
-
- /**
- * Generic map merge logic
- * @param first first map
- * @param second second map
- * @param <T1> key type
- * @param <T2> value type
- * @return 'first' merged with the second
- */
- public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
- Map<T1, T2> second) {
- first.putAll(second);
- return first;
- }
-
/**
* Generic map merge logic
* @param first first map
@@ -763,125 +285,6 @@ public final class SliderUtils {
}
/**
- * Parse an int value, replacing it with defval if undefined;
- * @param errorKey key to use in exceptions
- * @param defVal default value to use if the key is not in the map
- * @param min min value or -1 for do not check
- * @param max max value or -1 for do not check
- * @return the int value the integer value
- * @throws BadConfigException if the value could not be parsed
- */
- public static int parseAndValidate(String errorKey,
- String valS,
- int defVal,
- int min, int max) throws
- BadConfigException {
- if (valS == null) {
- valS = Integer.toString(defVal);
- }
- String trim = valS.trim();
- int val;
- try {
- val = Integer.decode(trim);
- } catch (NumberFormatException e) {
- throw new BadConfigException("Failed to parse value of "
- + errorKey + ": \"" + trim + "\"");
- }
- if (min >= 0 && val < min) {
- throw new BadConfigException("Value of "
- + errorKey + ": " + val + ""
- + "is less than the minimum of " + min);
- }
- if (max >= 0 && val > max) {
- throw new BadConfigException("Value of "
- + errorKey + ": " + val + ""
- + "is more than the maximum of " + max);
- }
- return val;
- }
-
- public static InetSocketAddress getRmAddress(Configuration conf) {
- return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_PORT);
- }
-
- public static InetSocketAddress getRmSchedulerAddress(Configuration conf) {
- return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
- }
-
- /**
- * probe to see if the RM scheduler is defined
- * @param conf config
- * @return true if the RM scheduler address is set to
- * something other than 0.0.0.0
- */
- public static boolean isRmSchedulerAddressDefined(Configuration conf) {
- InetSocketAddress address = getRmSchedulerAddress(conf);
- return isAddressDefined(address);
- }
-
- /**
- * probe to see if the address
- * @param address network address
- * @return true if the scheduler address is set to
- * something other than 0.0.0.0
- */
- public static boolean isAddressDefined(InetSocketAddress address) {
- if (address == null || address.getHostString() == null) {
- return false;
- }
- return !(address.getHostString().equals("0.0.0.0"));
- }
-
- public static void setRmAddress(Configuration conf, String rmAddr) {
- conf.set(YarnConfiguration.RM_ADDRESS, rmAddr);
- }
-
- public static void setRmSchedulerAddress(Configuration conf, String rmAddr) {
- conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);
- }
-
- public static boolean hasAppFinished(ApplicationReport report) {
- return report == null ||
- report.getYarnApplicationState().ordinal() >=
- YarnApplicationState.FINISHED.ordinal();
- }
-
- public static String containerToString(Container container) {
- if (container == null) {
- return "null container";
- }
- return String.format(Locale.ENGLISH,
- "ContainerID=%s nodeID=%s http=%s priority=%s resource=%s",
- container.getId(),
- container.getNodeId(),
- container.getNodeHttpAddress(),
- container.getPriority(),
- container.getResource());
- }
-
- /**
- * convert an AM report to a string for diagnostics
- * @param report the report
- * @return the string value
- */
- public static String reportToString(ApplicationReport report) {
- if (report == null) {
- return "Null service report";
- }
-
- return "App " + report.getName() + "/" + report.getApplicationType() +
- "# " +
- report.getApplicationId() + " user " + report.getUser() +
- " is in state " + report.getYarnApplicationState() +
- " RPC: " + report.getHost() + ":" + report.getRpcPort() +
- " URL: " + report.getOriginalTrackingUrl();
- }
-
- /**
* Convert a YARN URL into a string value of a normal URL
* @param url URL
* @return string representatin
@@ -949,151 +352,6 @@ public final class SliderUtils {
return env;
}
- /**
- * Apply a set of command line options to a cluster role map
- * @param clusterRoleMap cluster role map to merge onto
- * @param commandOptions command opts
- */
- public static void applyCommandLineRoleOptsToRoleMap(
- Map<String, Map<String, String>> clusterRoleMap,
- Map<String, Map<String, String>> commandOptions) {
- for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
- String key = entry.getKey();
- Map<String, String> optionMap = entry.getValue();
- Map<String, String> existingMap = clusterRoleMap.get(key);
- if (existingMap == null) {
- existingMap = new HashMap<String, String>();
- }
- log.debug("Overwriting role options with command line values {}",
- stringifyMap(optionMap));
- mergeMap(existingMap, optionMap);
- //set or overwrite the role
- clusterRoleMap.put(key, existingMap);
- }
- }
-
- /**
- * Verify that a Kerberos principal has been set -if not fail
- * with an error message that actually tells you what is missing
- * @param conf configuration to look at
- * @param principal key of principal
- * @throws BadConfigException if the key is not set
- */
- public static void verifyPrincipalSet(Configuration conf,
- String principal) throws
- BadConfigException {
- String principalName = conf.get(principal);
- if (principalName == null) {
- throw new BadConfigException("Unset Kerberos principal : %s",
- principal);
- }
- log.debug("Kerberos princial {}={}", principal, principalName);
- }
-
- /**
- * Flag to indicate whether the cluster is in secure mode
- * @param conf configuration to look at
- * @return true if the slider client/service should be in secure mode
- */
- public static boolean isHadoopClusterSecure(Configuration conf) {
- return SecurityUtil.getAuthenticationMethod(conf) !=
- UserGroupInformation.AuthenticationMethod.SIMPLE;
- }
-
- /**
- * Init security if the cluster configuration declares the cluster is secure
- * @param conf configuration to look at
- * @return true if the cluster is secure
- * @throws IOException cluster is secure
- * @throws SliderException the configuration/process is invalid
- */
- public static boolean maybeInitSecurity(Configuration conf) throws
- IOException,
- SliderException {
- boolean clusterSecure = isHadoopClusterSecure(conf);
- if (clusterSecure) {
- log.debug("Enabling security");
- initProcessSecurity(conf);
- }
- return clusterSecure;
- }
-
- /**
- * Turn on security. This is setup to only run once.
- * @param conf configuration to build up security
- * @return true if security was initialized in this call
- * @throws IOException IO/Net problems
- * @throws BadConfigException the configuration and system state are inconsistent
- */
- public static boolean initProcessSecurity(Configuration conf) throws
- IOException,
- SliderException {
-
- if (processSecurityAlreadyInitialized.compareAndSet(true, true)) {
- //security is already inited
- return false;
- }
-
- log.info("JVM initialized into secure mode with kerberos realm {}",
- SliderUtils.getKerberosRealm());
- //this gets UGI to reset its previous world view (i.e simple auth)
- //security
- log.debug("java.security.krb5.realm={}",
- System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
- log.debug("java.security.krb5.kdc={}",
- System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
- log.debug("hadoop.security.authentication={}",
- conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION));
- log.debug("hadoop.security.authorization={}",
- conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION));
- UserGroupInformation.setConfiguration(conf);
- UserGroupInformation authUser = UserGroupInformation.getCurrentUser();
- log.debug("Authenticating as {}", authUser);
- log.debug("Login user is {}", UserGroupInformation.getLoginUser());
- if (!UserGroupInformation.isSecurityEnabled()) {
- throw new SliderException(LauncherExitCodes.EXIT_UNAUTHORIZED,
- "Although secure mode is enabled," +
- "the service has already set up its user as an insecure entity %s",
- authUser);
- }
- if (authUser.getAuthenticationMethod() ==
- UserGroupInformation.AuthenticationMethod.SIMPLE) {
- throw new BadConfigException("Auth User is not Kerberized %s" +
- " -security has already been set up with the wrong authentication method. "
- + "This can occur if a file system has already been created prior to the loading of "
- + "the security configuration.",
- authUser);
-
- }
-
- SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
- SliderUtils.verifyPrincipalSet(conf, "dfs.namenode.kerberos.principal");
- return true;
- }
-
- /**
- * Force an early login: This catches any auth problems early rather than
- * in RPC operations
- * @throws IOException if the login fails
- */
- public static void forceLogin() throws IOException {
- if (UserGroupInformation.isSecurityEnabled()) {
- if (UserGroupInformation.isLoginKeytabBased()) {
- UserGroupInformation.getLoginUser().reloginFromKeytab();
- } else {
- UserGroupInformation.getLoginUser().reloginFromTicketCache();
- }
- }
- }
-
- public static String getLibDir() {
- String[] libDirs = getLibDirs();
- if (libDirs == null || libDirs.length == 0) {
- return null;
- }
- return libDirs[0];
- }
-
public static String[] getLibDirs() {
String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
if (isUnset(libDirStr)) {
@@ -1183,39 +441,6 @@ public final class SliderUtils {
sliderFileSystem.submitTarGzipAndUpdate(providerResources);
}
- public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
- Map<String, Map<String, String>> dest = new HashMap<>();
- for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) {
- dest.put(entry.getKey(), stringMapClone(entry.getValue()));
- }
- return dest;
- }
-
- public static Map<String, String> stringMapClone(Map<String, String> src) {
- Map<String, String> dest = new HashMap<>();
- return mergeEntries(dest, src.entrySet());
- }
-
- /**
- * List a directory in the local filesystem
- * @param dir directory
- * @return a listing, one to a line
- */
- public static String listDir(File dir) {
- if (dir == null) {
- return "";
- }
- String[] confDirEntries = dir.list();
- if (confDirEntries == null) {
- return "";
- }
- StringBuilder builder = new StringBuilder();
- for (String entry : confDirEntries) {
- builder.append(entry).append("\n");
- }
- return builder.toString();
- }
-
/**
* Create a file:// path from a local file
* @param file file to point the path
@@ -1225,16 +450,6 @@ public final class SliderUtils {
return new Path(file.toURI());
}
- public static String getKerberosRealm() {
- try {
- return KerberosUtil.getDefaultRealm();
- } catch (Exception e) {
- log.debug("introspection into JVM internals failed", e);
- return "(unknown)";
-
- }
- }
-
/**
* Build up the classpath for execution
* -behaves very differently on a mini test cluster vs a production
@@ -1279,171 +494,6 @@ public final class SliderUtils {
}
/**
- * Verify that a path refers to a directory. If not
- * logs the parent dir then throws an exception
- * @param dir the directory
- * @param errorlog log for output on an error
- * @throws FileNotFoundException if it is not a directory
- */
- public static void verifyIsDir(File dir, Logger errorlog) throws
- FileNotFoundException {
- if (!dir.exists()) {
- errorlog.warn("contents of {}: {}", dir,
- listDir(dir.getParentFile()));
- throw new FileNotFoundException(dir.toString());
- }
- if (!dir.isDirectory()) {
- errorlog.info("contents of {}: {}", dir,
- listDir(dir.getParentFile()));
- throw new FileNotFoundException(
- "Not a directory: " + dir);
- }
- }
-
- /**
- * Verify that a file exists
- * @param file file
- * @param errorlog log for output on an error
- * @throws FileNotFoundException
- */
- public static void verifyFileExists(File file, Logger errorlog) throws
- FileNotFoundException {
- if (!file.exists()) {
- errorlog.warn("contents of {}: {}", file,
- listDir(file.getParentFile()));
- throw new FileNotFoundException(file.toString());
- }
- if (!file.isFile()) {
- throw new FileNotFoundException("Not a file: " + file.toString());
- }
- }
-
- /**
- * verify that a config option is set
- * @param configuration config
- * @param key key
- * @return the value, in case it needs to be verified too
- * @throws BadConfigException if the key is missing
- */
- public static String verifyOptionSet(Configuration configuration, String key,
- boolean allowEmpty) throws BadConfigException {
- String val = configuration.get(key);
- if (val == null) {
- throw new BadConfigException(
- "Required configuration option \"%s\" not defined ", key);
- }
- if (!allowEmpty && val.isEmpty()) {
- throw new BadConfigException(
- "Configuration option \"%s\" must not be empty", key);
- }
- return val;
- }
-
- /**
- * Verify that a keytab property is defined and refers to a non-empty file
- *
- * @param siteConf configuration
- * @param prop property to look for
- * @return the file referenced
- * @throws BadConfigException on a failure
- */
- public static File verifyKeytabExists(Configuration siteConf,
- String prop) throws
- BadConfigException {
- String keytab = siteConf.get(prop);
- if (keytab == null) {
- throw new BadConfigException("Missing keytab property %s",
- prop);
-
- }
- File keytabFile = new File(keytab);
- if (!keytabFile.exists()) {
- throw new BadConfigException("Missing keytab file %s defined in %s",
- keytabFile,
- prop);
- }
- if (keytabFile.length() == 0 || !keytabFile.isFile()) {
- throw new BadConfigException("Invalid keytab file %s defined in %s",
- keytabFile,
- prop);
- }
- return keytabFile;
- }
-
- /**
- * Add a subpath to an existing URL. This extends
- * the path, inserting a / between all entries
- * if needed.
- * @param base base path/URL
- * @param path subpath
- * @return base+"/"+subpath
- */
- public static String appendToURL(String base, String path) {
- StringBuilder fullpath = new StringBuilder(base);
- if (!base.endsWith("/")) {
- fullpath.append("/");
- }
- if (path.startsWith("/")) {
- fullpath.append(path.substring(1));
- } else {
- fullpath.append(path);
- }
- return fullpath.toString();
- }
-
- /**
- * Truncate the given string to a maximum length provided
- * with a pad (...) added to the end if expected size if more than 10.
- * @param toTruncate string to truncate; may be null
- * @param maxSize maximum size
- * @return the truncated/padded string.
- */
- public static String truncate(String toTruncate, int maxSize) {
- if (toTruncate == null || maxSize < 1
- || toTruncate.length() <= maxSize) {
- return toTruncate;
- }
-
- String pad = "...";
- if (maxSize < 10) {
- pad = "";
- }
- return toTruncate.substring(0, maxSize - pad.length()).concat(pad);
- }
-
- /**
- * Given a source folder create zipped file
- *
- * @param srcFolder
- * @param zipFile
- *
- * @throws IOException
- */
- public static void zipFolder(File srcFolder, File zipFile) throws IOException {
- log.info("Zipping folder {} to {}", srcFolder.getAbsolutePath(), zipFile.getAbsolutePath());
- List<String> files = new ArrayList<>();
- generateFileList(files, srcFolder, srcFolder, true);
-
- byte[] buffer = new byte[1024];
-
- try (FileOutputStream fos = new FileOutputStream(zipFile)) {
- try (ZipOutputStream zos = new ZipOutputStream(fos)) {
-
- for (String file : files) {
- ZipEntry ze = new ZipEntry(file);
- zos.putNextEntry(ze);
- try (FileInputStream in = new FileInputStream(srcFolder + File.separator + file)) {
- int len;
- while ((len = in.read(buffer)) > 0) {
- zos.write(buffer, 0, len);
- }
- }
- }
- }
- }
- }
-
- /**
* Given a source folder create a tar.gz file
*
* @param libDirs
@@ -1479,11 +529,6 @@ public final class SliderUtils {
}
private static void generateFileList(List<String> fileList, File node,
- File rootFolder, Boolean relative) {
- generateFileList(fileList, node, rootFolder, relative, null);
- }
-
- private static void generateFileList(List<String> fileList, File node,
File rootFolder, Boolean relative, FilenameFilter filter) {
if (node.isFile()) {
String fileFullPath = node.toString();
@@ -1507,134 +552,6 @@ public final class SliderUtils {
}
}
- /**
- * Check for any needed libraries being present. On Unix none are needed;
- * on windows they must be present
- * @return true if all is well
- */
- public static String checkForRequiredNativeLibraries() {
-
- if (!Shell.WINDOWS) {
- return "";
- }
- StringBuilder errorText = new StringBuilder("");
- if (!NativeIO.isAvailable()) {
- errorText.append("No native IO library. ");
- }
- try {
- String path = Shell.getQualifiedBinPath(WINUTILS);
- log.debug("winutils is at {}", path);
- } catch (IOException e) {
- errorText.append("No " + WINUTILS);
- log.warn("No winutils: {}", e, e);
- }
- try {
- File target = new File("target");
- FileUtil.canRead(target);
- } catch (UnsatisfiedLinkError e) {
- log.warn("Failing to link to native IO methods: {}", e, e);
- errorText.append("No native IO methods");
- }
- return errorText.toString();
- }
-
- /**
- * Strictly verify that windows utils is present.
- * Checks go as far as opening the file and looking for
- * the headers.
- * @throws IOException on any problem reading the file
- * @throws FileNotFoundException if the file is not considered valid
- */
- public static void maybeVerifyWinUtilsValid() throws
- IOException,
- SliderException {
- String errorText = SliderUtils.checkForRequiredNativeLibraries();
- if (!errorText.isEmpty()) {
- throw new BadClusterStateException(errorText);
- }
- }
-
- /**
- * Write bytes to a file
- * @param outfile output file
- * @param data data to write
- * @throws IOException on any IO problem
- */
- public static void write(File outfile, byte[] data)
- throws IOException {
- File parentDir = outfile.getCanonicalFile().getParentFile();
- if (parentDir == null) {
- throw new IOException(outfile.getPath() + " has no parent dir");
- }
- if (!parentDir.exists()) {
- if(!parentDir.mkdirs()) {
- throw new IOException("Failed to create parent directory " + parentDir);
- }
- }
- SliderUtils.verifyIsDir(parentDir, log);
- try(FileOutputStream out = new FileOutputStream(outfile)) {
- out.write(data);
- }
- }
-
- /**
- * Compare the times of two applications: most recent app comes first
- * Specifically: the one whose start time value is greater.
- */
- private static class MostRecentlyStartedAppFirst
- implements Comparator<ApplicationReport>, Serializable {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- long x = r1.getStartTime();
- long y = r2.getStartTime();
- return compareTwoLongsReverse(x, y);
- }
- }
-
- /**
- * Compare the times of two applications: most recent app comes first.
- * "Recent"== the app whose start time <i>or finish time</i> is the greatest.
- */
- private static class MostRecentlyStartedOrFinishedFirst
- implements Comparator<ApplicationReport>, Serializable {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- long started1 = r1.getStartTime();
- long started2 = r2.getStartTime();
- long finished1 = r1.getFinishTime();
- long finished2 = r2.getFinishTime();
- long lastEvent1 = Math.max(started1, finished1);
- long lastEvent2 = Math.max(started2, finished2);
- return compareTwoLongsReverse(lastEvent1, lastEvent2);
- }
- }
-
- /**
- * Compare the times of two applications: most recently finished app comes first
- * Specifically: the one whose finish time value is greater.
- */
- private static class MostRecentAppFinishFirst
- implements Comparator<ApplicationReport>, Serializable {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- long x = r1.getFinishTime();
- long y = r2.getFinishTime();
- return compareTwoLongsReverse(x, y);
- }
- }
-
- /**
- * Compare two long values for sorting. As the return value for
- * comparators must be int, the simple value of <code>x-y</code>
- * is inapplicable
- * @param x x value
- * @param y y value
- * @return +ve if x is less than y, -ve if y is greater than x; 0 for equality
- */
- public static int compareTwoLongsReverse(long x, long y) {
- return (x < y) ? 1 : ((x == y) ? 0 : -1);
- }
-
public static String createNameTag(String name) {
return "Name: " + name;
}
@@ -1646,9 +563,4 @@ public final class SliderUtils {
public static String createDescriptionTag(String description) {
return "Description: " + description;
}
-
- public static final String DAYS = ".days";
- public static final String HOURS = ".hours";
- public static final String MINUTES = ".minutes";
- public static final String SECONDS = ".seconds";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index 0f4f598..a2edbc8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -18,12 +18,22 @@
package org.apache.hadoop.yarn.service;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Resource;
import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.codehaus.jackson.map.PropertyNamingStrategy;
+import java.io.IOException;
+
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
public class ServiceTestUtils {
public static final JsonSerDeser<Service> JSON_SER_DESER =
@@ -56,4 +66,22 @@ public class ServiceTestUtils {
resource.setCpus(1);
return comp1;
}
+
+ public static SliderFileSystem initMockFs() throws IOException {
+ return initMockFs(null);
+ }
+
+ public static SliderFileSystem initMockFs(Service ext) throws IOException {
+ SliderFileSystem sfs = mock(SliderFileSystem.class);
+ FileSystem mockFs = mock(FileSystem.class);
+ JsonSerDeser<Service> jsonSerDeser = mock(JsonSerDeser.class);
+ when(sfs.getFileSystem()).thenReturn(mockFs);
+ when(sfs.buildClusterDirPath(anyObject())).thenReturn(
+ new Path("cluster_dir_path"));
+ if (ext != null) {
+ when(jsonSerDeser.load(anyObject(), anyObject())).thenReturn(ext);
+ }
+ ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
+ return sfs;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
index be36335..959e4d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.service;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.service.api.records.Service;
@@ -27,7 +25,6 @@ import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.junit.Assert;
@@ -44,10 +41,8 @@ import java.util.List;
import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME;
import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*;
-import static org.easymock.EasyMock.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
/**
* Test for ServiceApiUtil helper methods.
@@ -78,7 +73,7 @@ public class TestServiceApiUtil {
assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR
.length());
- SliderFileSystem sfs = initMock(null);
+ SliderFileSystem sfs = ServiceTestUtils.initMockFs();
Service app = new Service();
@@ -230,7 +225,7 @@ public class TestServiceApiUtil {
@Test
public void testArtifacts() throws IOException {
- SliderFileSystem sfs = initMock(null);
+ SliderFileSystem sfs = ServiceTestUtils.initMockFs();
Service app = new Service();
app.setName("name");
@@ -309,27 +304,10 @@ public class TestServiceApiUtil {
return app;
}
- private static SliderFileSystem initMock(Service ext) throws IOException {
- SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
- FileSystem mockFs = createNiceMock(FileSystem.class);
- JsonSerDeser<Service> jsonSerDeser = createNiceMock(JsonSerDeser
- .class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- if (ext != null) {
- expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
- .anyTimes();
- }
- replay(sfs, mockFs, jsonSerDeser);
- ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
- return sfs;
- }
-
@Test
public void testExternalApplication() throws IOException {
Service ext = createValidApplication("comp1");
- SliderFileSystem sfs = initMock(ext);
+ SliderFileSystem sfs = ServiceTestUtils.initMockFs(ext);
Service app = createValidApplication(null);
@@ -350,7 +328,7 @@ public class TestServiceApiUtil {
@Test
public void testDuplicateComponents() throws IOException {
- SliderFileSystem sfs = initMock(null);
+ SliderFileSystem sfs = ServiceTestUtils.initMockFs();
String compName = "comp1";
Service app = createValidApplication(compName);
@@ -368,7 +346,7 @@ public class TestServiceApiUtil {
@Test
public void testExternalDuplicateComponent() throws IOException {
Service ext = createValidApplication("comp1");
- SliderFileSystem sfs = initMock(ext);
+ SliderFileSystem sfs = ServiceTestUtils.initMockFs(ext);
Service app = createValidApplication("comp1");
Artifact artifact = new Artifact();
@@ -387,7 +365,7 @@ public class TestServiceApiUtil {
@Test
public void testExternalComponent() throws IOException {
Service ext = createValidApplication("comp1");
- SliderFileSystem sfs = initMock(ext);
+ SliderFileSystem sfs = ServiceTestUtils.initMockFs(ext);
Service app = createValidApplication("comp2");
Artifact artifact = new Artifact();
@@ -454,7 +432,7 @@ public class TestServiceApiUtil {
e)), ex.getMessage());
}
- SliderFileSystem sfs = initMock(null);
+ SliderFileSystem sfs = ServiceTestUtils.initMockFs();
Service service = createValidApplication(null);
service.setComponents(Arrays.asList(c, d, e));
try {
@@ -470,7 +448,7 @@ public class TestServiceApiUtil {
@Test
public void testInvalidComponent() throws IOException {
- SliderFileSystem sfs = initMock(null);
+ SliderFileSystem sfs = ServiceTestUtils.initMockFs();
testComponent(sfs);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 63aa9c6..30f2aeb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.service;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.curator.test.TestingCluster;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -51,6 +49,8 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -78,8 +78,8 @@ import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_B
*/
public class TestYarnNativeServices extends ServiceTestUtils{
- private static final Log LOG =
- LogFactory.getLog(TestYarnNativeServices.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestYarnNativeServices.class);
private MiniYARNCluster yarnCluster = null;
private MiniDFSCluster hdfsCluster = null;
@@ -416,7 +416,7 @@ public class TestYarnNativeServices extends ServiceTestUtils{
LOG.info("Num Components " + retrievedApp.getComponents().size());
for (Component component : retrievedApp.getComponents()) {
LOG.info("looking for " + component.getName());
- LOG.info(component);
+ LOG.info(component.toString());
if (component.getContainers() != null) {
if (component.getContainers().size() == exampleApp
.getComponent(component.getName()).getNumberOfContainers()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
index ecc529d..c53ee2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
@@ -45,7 +45,6 @@ import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONN
import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS;
import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-import static org.mockito.Matchers.any;
import static org.mockito.Mockito.*;
public class TestServiceCLI {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
index 04ec526..8739382 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
@@ -18,13 +18,11 @@
package org.apache.hadoop.yarn.service.conf;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.ServiceTestUtils;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.hadoop.yarn.service.api.records.Configuration;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.junit.Assert;
@@ -40,7 +38,6 @@ import java.util.Map;
import java.util.Set;
import static org.apache.hadoop.yarn.service.conf.ExampleAppJson.*;
-import static org.easymock.EasyMock.*;
/**
* Test global configuration resolution.
@@ -78,12 +75,7 @@ public class TestAppJsonResolve extends Assert {
assertEquals("1000", worker.getProperty("timeout"));
// here is the resolution
- SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
- FileSystem mockFs = createNiceMock(FileSystem.class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- replay(sfs, mockFs);
+ SliderFileSystem sfs = ServiceTestUtils.initMockFs();
ServiceApiUtil.validateAndResolveService(orig, sfs, new
YarnConfiguration());
@@ -162,27 +154,13 @@ public class TestAppJsonResolve extends Assert {
assertEquals(0, other.getProperties().size());
// load the external service
- SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
- FileSystem mockFs = createNiceMock(FileSystem.class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- replay(sfs, mockFs);
+ SliderFileSystem sfs = ServiceTestUtils.initMockFs();
Service ext = ExampleAppJson.loadResource(APP_JSON);
ServiceApiUtil.validateAndResolveService(ext, sfs, new
YarnConfiguration());
- reset(sfs, mockFs);
// perform the resolution on original service
- JsonSerDeser<Service> jsonSerDeser = createNiceMock(JsonSerDeser
- .class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
- .anyTimes();
- replay(sfs, mockFs, jsonSerDeser);
- ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
+ sfs = ServiceTestUtils.initMockFs(ext);
ServiceApiUtil.validateAndResolveService(orig, sfs, new
YarnConfiguration());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
index 83e9502..a813da3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
@@ -18,9 +18,8 @@
package org.apache.hadoop.yarn.service.conf;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.ServiceTestUtils;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
@@ -33,7 +32,6 @@ import java.util.Arrays;
import java.util.Collection;
import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
-import static org.easymock.EasyMock.*;
/**
* Test loading example resources.
@@ -62,12 +60,7 @@ public class TestLoadExampleAppJson extends Assert {
try {
Service service = JSON_SER_DESER.fromResource(resource);
- SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
- FileSystem mockFs = createNiceMock(FileSystem.class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- replay(sfs, mockFs);
+ SliderFileSystem sfs = ServiceTestUtils.initMockFs();
ServiceApiUtil.validateAndResolveService(service, sfs,
new YarnConfiguration());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
index 5b24a1d..79406e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.service.providers;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
@@ -29,7 +28,9 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import static org.easymock.EasyMock.*;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
/**
* Test the AbstractClientProvider shared methods.
@@ -55,9 +56,8 @@ public class TestAbstractClientProvider {
@Test
public void testConfigFiles() throws IOException {
ClientProvider clientProvider = new ClientProvider();
- FileSystem mockFs = createNiceMock(FileSystem.class);
- expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes();
- replay(mockFs);
+ FileSystem mockFs = mock(FileSystem.class);
+ when(mockFs.exists(anyObject())).thenReturn(true);
ConfigFile configFile = new ConfigFile();
List<ConfigFile> configFiles = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index 3cbbaa7..1b3c5f0 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -82,6 +82,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-router</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-services-core</artifactId>
+ </dependency>
</dependencies>
<build>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[81/86] [abbrv] hadoop git commit: YARN-7165. Miscellaneous fixes in
yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
new file mode 100644
index 0000000..9022268
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -0,0 +1,592 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# YARN Service API
+
+Bringing a new service on YARN today is not a simple experience. The APIs of existing
+frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs)
+or writing a complex spec (for declarative frameworks).
+
+This simplified REST API can be used to create and manage the lifecycle of YARN services.
+In most cases, the application owner will not be forced to make any changes to their applications.
+This is primarily true if the application is packaged with containerization technologies like Docker.
+
+This document describes the API specifications (aka. YarnFile) for deploying/managing
+containerized services on YARN. The same JSON spec can be used for both REST API
+and CLI to manage the services.
+
+
+### Version information
+Version: 1.0.0
+
+### License information
+License: Apache 2.0
+License URL: http://www.apache.org/licenses/LICENSE-2.0.html
+
+### URI scheme
+Host: host.mycompany.com
+
+Port: 9191(default)
+
+Schemes: HTTP
+
+### Consumes
+
+* application/json
+
+
+### Produces
+
+* application/json
+
+
+## Paths
+### Create a service
+```
+POST /ws/v1/services
+```
+
+#### Description
+
+Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state.
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|BodyParameter|Service|Service request object|true|Service||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|202|The request to create a service is accepted|No Content|
+|400|Invalid service definition provided in the request body|No Content|
+|500|Failed to create a service|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### (TBD) List of services running in the cluster.
+```
+GET /ws/v1/services
+```
+
+#### Description
+
+Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name.
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|An array of services|Service array|
+|default|Unexpected error|ServiceStatus|
+
+
+### Get current version of the API server.
+```
+GET /ws/v1/services/version
+```
+
+#### Description
+
+Get current version of the API server.
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|Successful request|No Content|
+
+
+### Update a service or upgrade the binary version of the components of a running service
+```
+PUT /ws/v1/services/{service_name}
+```
+
+#### Description
+
+Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service. The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts (TBD).
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+|BodyParameter|Service|The updated service definition. It can contain the updated lifetime of a service or the desired state (STOPPED/STARTED) of a service to initiate a start/stop operation against the specified service|true|Service||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|204|Update or upgrade was successful|No Content|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### Destroy a service
+```
+DELETE /ws/v1/services/{service_name}
+```
+
+#### Description
+
+Destroy a service and release all resources. This API might have to return JSON data providing location of logs (TBD), etc.
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|204|Destroy was successful|No Content|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### Get details of a service.
+```
+GET /ws/v1/services/{service_name}
+```
+
+#### Description
+
+Return the details (including containers) of a running service
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|a service object|object|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### Flex a component's number of instances.
+```
+PUT /ws/v1/services/{service_name}/components/{component_name}
+```
+
+#### Description
+
+Set a component's desired number of instanes
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+|PathParameter|component_name|Component name|true|string||
+|BodyParameter|Component|The definition of a component which contains the updated number of instances.|true|Component||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|Flex was successful|No Content|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+## Definitions
+### Artifact
+
+Artifact of a service component. If not specified, component will just run the bare launch command and no artifact will be localized.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|id|Artifact id. Examples are package location uri for tarball based services, image name for docker, name of service, etc.|true|string||
+|type|Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For SERVICE type, the service specified will be read and its components will be added into this service. The original component with artifact type SERVICE will be removed (any properties specified in the original component will be ignored).|false|enum (DOCKER, TARBALL, SERVICE)|DOCKER|
+|uri|Artifact location to support multiple artifact stores (optional).|false|string||
+
+
+### Component
+
+One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|name|Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.|true|string||
+|dependencies|An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of a service should be represented as a DAG.|false|string array||
+|readiness_check|Readiness check for this component.|false|ReadinessCheck||
+|artifact|Artifact of the component (optional). If not specified, the service level global artifact takes effect.|false|Artifact||
+|launch_command|The custom launch command of this component (optional for DOCKER component, required otherwise). When specified at the component level, it overrides the value specified at the global level (if any).|false|string||
+|resource|Resource of this component (optional). If not specified, the service level global resource takes effect.|false|Resource||
+|number_of_containers|Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.|false|integer (int64)||
+|run_privileged_container|Run all containers of this component in privileged mode (YARN-4262).|false|boolean||
+|placement_policy|Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.|false|PlacementPolicy||
+|configuration|Config properties for this component.|false|Configuration||
+|quicklinks|A list of quicklink keys defined at the service level, and to be resolved by this component.|false|string array||
+
+
+### ConfigFile
+
+A config file that needs to be created and made available as a volume in a service component container.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|type|Config file in the standard format like xml, properties, json, yaml, template.|false|enum (XML, PROPERTIES, JSON, YAML, TEMPLATE, ENV, HADOOP_XML)||
+|dest_file|The path that this configuration file should be created as. If it is an absolute path, it will be mounted into the DOCKER container. Absolute paths are only allowed for DOCKER containers. If it is a relative path, only the file name should be provided, and the file will be created in the container local working directory under a folder named conf.|false|string||
+|src_file|This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.|false|string||
+|props|A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.|false|object||
+
+
+### Configuration
+
+Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|properties|A blob of key-value pairs of common service properties.|false|object||
+|env|A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.|false|object||
+|files|Array of list of files that needs to be created and made available as volumes in the service component containers.|false|ConfigFile array||
+
+
+### Container
+
+An instance of a running service container.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|id|Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002.|false|string||
+|launch_time|The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.|false|string (date)||
+|ip|IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.|false|string||
+|hostname|Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.|false|string||
+|bare_host|The bare node or host in which the container is running, e.g. cn008.example.com.|false|string||
+|state|State of the container of a service.|false|ContainerState||
+|component_name|Name of the component that this container instance belongs to.|false|string||
+|resource|Resource used for this container.|false|Resource||
+|artifact|Artifact used for this container.|false|Artifact||
+|privileged_container|Container running in privileged mode or not.|false|boolean||
+
+
+### ContainerState
+
+The current state of the container of a service.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|state|enum of the state of the container|false|enum (INIT, STARTED, READY)||
+
+
+### PlacementPolicy
+
+Placement policy of an instance of a service. This feature is in the works in YARN-6592.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|label|Assigns a service to a named partition of the cluster where the service desires to run (optional). If not specified all services are submitted to a default label of the service owner. One or more labels can be setup for each service owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.|false|string||
+
+
+### ReadinessCheck
+
+A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|type|E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).|true|enum (HTTP, PORT)||
+|props|A blob of key value pairs that will be used to configure the check.|false|object||
+|artifact|Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET|false|Artifact||
+
+
+### Resource
+
+Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|profile|Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.|false|string||
+|cpus|Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).|false|integer (int32)||
+|memory|Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.|false|string||
+
+
+### Service
+
+a service resource has the following attributes.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|name|A unique service name. If Registry DNS is enabled, the max length is 63 characters.|true|string||
+|id|A unique service id.|false|string||
+|artifact|Artifact of single-component service.|false|Artifact||
+|resource|Resource of single-component service or the global default for multi-component services. Mandatory if it is a single-component service and if cpus and memory are not specified at the Service level.|false|Resource||
+|launch_command|The custom launch command of a service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section.|false|string||
+|launch_time|The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.|false|string (date)||
+|number_of_containers|Number of containers for each component in the service. Each component can further override this service-level global default.|false|integer (int64)||
+|number_of_running_containers|In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.|false|integer (int64)||
+|lifetime|Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.|false|integer (int64)||
+|placement_policy|(TBD) Advanced scheduling and placement policies. If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.|false|PlacementPolicy||
+|components|Components of a service.|false|Component array||
+|configuration|Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.|false|Configuration||
+|containers|Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.|false|Container array||
+|state|State of the service. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started service.|false|ServiceState||
+|quicklinks|A blob of key-value pairs of quicklinks to be exported for a service.|false|object||
+|queue|The YARN queue that this service should be submitted to.|false|string||
+
+
+### ServiceState
+
+The current state of a service.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|state|enum of the state of the service|false|enum (ACCEPTED, STARTED, READY, STOPPED, FAILED)||
+
+
+### ServiceStatus
+
+The current status of a submitted service, returned as a response to the GET API.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|diagnostics|Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state.|false|string||
+|state|Service state.|false|ServiceState||
+|code|An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.|false|integer (int32)||
+
+
+
+## Examples
+
+### Create a simple single-component service with most attribute values as defaults
+POST URL - http://localhost:9191/ws/v1/services
+
+##### POST Request JSON
+```json
+{
+ "name": "hello-world",
+ "components" :
+ [
+ {
+ "name": "hello",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "nginx:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "./start_nginx.sh",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ }
+ }
+ ]
+}
+```
+
+##### GET Response JSON
+GET URL - http://localhost:9191/ws/v1/services/hello-world
+
+Note, lifetime value of -1 means unlimited lifetime.
+
+```json
+{
+ "name": "hello-world",
+ "id": "application_1503963985568_0002",
+ "lifetime": -1,
+ "components": [
+ {
+ "name": "hello",
+ "dependencies": [],
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ },
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": [],
+ "containers": [
+ {
+ "id": "container_e03_1503963985568_0002_01_000001",
+ "ip": "10.22.8.143",
+ "hostname": "myhost.local",
+ "state": "READY",
+ "launch_time": 1504051512412,
+ "bare_host": "10.22.8.143",
+ "component_name": "hello-0"
+ },
+ {
+ "id": "container_e03_1503963985568_0002_01_000002",
+ "ip": "10.22.8.143",
+ "hostname": "myhost.local",
+ "state": "READY",
+ "launch_time": 1504051536450,
+ "bare_host": "10.22.8.143",
+ "component_name": "hello-1"
+ }
+ ],
+ "launch_command": "./start_nginx.sh",
+ "number_of_containers": 1,
+ "run_privileged_container": false
+ }
+ ],
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": {}
+}
+
+```
+### Update to modify the lifetime of a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+
+Note, irrespective of what the current lifetime value is, this update request will set the lifetime of the service to be 3600 seconds (1 hour) from the time the request is submitted. Hence, if a a service has remaining lifetime of 5 mins (say) and would like to extend it to an hour OR if an application has remaining lifetime of 5 hours (say) and would like to reduce it down to an hour, then for both scenarios you need to submit the same request below.
+
+```json
+{
+ "lifetime": 3600
+}
+```
+### Stop a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+```json
+{
+ "state": "STOPPED"
+}
+```
+
+### Start a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+```json
+{
+ "state": "STARTED"
+}
+```
+
+### Update to flex up/down the no of containers (instances) of a component of a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello
+
+##### PUT Request JSON
+```json
+{
+ "name": "hello",
+ "number_of_containers": 3
+}
+```
+
+### Destroy a service
+DELETE URL - http://localhost:9191/ws/v1/services/hello-world
+
+***
+
+### Create a complicated service - HBase
+POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
+
+##### POST Request JSON
+
+```json
+{
+ "name": "hbase-app-1",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "hbasemaster",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "hbase:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/hdp/current/hbase-master/bin/hbase master start",
+ "resource": {
+ "cpus": 1,
+ "memory": "2048"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_DIR": "<LOG_DIR>"
+ },
+ "files": [
+ {
+ "type": "XML",
+ "dest_file": "/etc/hadoop/conf/core-site.xml",
+ "props": {
+ "fs.defaultFS": "${CLUSTER_FS_URI}"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "/etc/hbase/conf/hbase-site.xml",
+ "props": {
+ "hbase.cluster.distributed": "true",
+ "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
+ "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
+ "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
+ "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
+ "hbase.master.info.port": "16010"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "regionserver",
+ "number_of_containers": 3,
+ "unique_component_support": "true",
+ "artifact": {
+ "id": "hbase:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/hdp/current/hbase-regionserver/bin/hbase regionserver start",
+ "resource": {
+ "cpus": 1,
+ "memory": "2048"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_DIR": "<LOG_DIR>"
+ },
+ "files": [
+ {
+ "type": "XML",
+ "dest_file": "/etc/hadoop/conf/core-site.xml",
+ "props": {
+ "fs.defaultFS": "${CLUSTER_FS_URI}"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "/etc/hbase/conf/hbase-site.xml",
+ "props": {
+ "hbase.cluster.distributed": "true",
+ "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
+ "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
+ "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
+ "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
+ "hbase.master.info.port": "16010",
+ "hbase.regionserver.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}"
+ }
+ }
+ ]
+ }
+ }
+ ],
+ "quicklinks": {
+ "HBase Master Status UI": "http://hbasemaster0.${SERVICE_NAME}.${USER}.${DOMAIN}:16010/master-status",
+ "Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/"
+ }
+}
+```
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[79/86] [abbrv] hadoop git commit: YARN-7161. Fix special chars in
DNS documentation. Contributed by Gour Saha
Posted by ji...@apache.org.
YARN-7161. Fix special chars in DNS documentation. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8af82c8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8af82c8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8af82c8
Branch: refs/heads/yarn-native-services
Commit: e8af82c8516c778672f8808b1ce2359e908bb81d
Parents: 7164b07
Author: Jian He <ji...@apache.org>
Authored: Thu Sep 7 18:17:02 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 25 16:37:25 2017 -0700
----------------------------------------------------------------------
.../native-services/NativeServicesDiscovery.md | 87 ++++++++++----------
1 file changed, 45 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8af82c8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
index fa54a09..a927118 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
@@ -13,39 +13,42 @@
-->
# YARN DNS Server
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
## Introduction
The YARN DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions:
-1. **Exposing existing service discovery information via DNS** - Information provided in
+1. **Exposing existing service-discovery information via DNS** - Information provided in
the current YARN service registry’s records will be converted into DNS entries, thus
allowing users to discover information about YARN applications using standard DNS
client mechanisms (for e.g. a DNS SRV Record specifying the hostname and port
number for services).
-2. **Enabling Container to IP mappings** - Enables discovery of the IPs of containers via
+2. **Enabling Container to IP mappings** - Enables discovery of the IPs of containers via
standard DNS lookups. Given the availability of the records via DNS, container
-namebased communication will be facilitated (e.g. ‘curl
+name-based communication will be facilitated (e.g. ‘curl
http://myContainer.myDomain.com/endpoint’).
## Service Properties
The existing YARN Service Registry is leveraged as the source of information for the DNS Service.
-The following core functions are supported by the DNS Server:
+The following core functions are supported by the DNS-Server:
-###Functional properties
+### Functional properties
-1. Supports creation of DNS records for endpoints of the deployed YARN applications
+1. Supports creation of DNS records for end-points of the deployed YARN applications
2. Record names remain unchanged during restart of containers and/or applications
3. Supports reverse lookups (name based on IP).
4. Supports security using the standards defined by The Domain Name System Security
Extensions (DNSSEC)
5. Highly available
-6. Scalable - The service provides the responsiveness (e.g. low latency) required to
+6. Scalable - The service provides the responsiveness (e.g. low-latency) required to
respond to DNS queries (timeouts yield attempts to invoke other configured name
servers).
-###Deployment properties
+### Deployment properties
1. Supports integration with existing DNS assets (e.g. a corporate DNS server) by acting as
a DNS server for a Hadoop cluster zone/domain. The server is not intended to act as a
@@ -55,59 +58,59 @@ DNS standards. The default port for DNS protocols is in a restricted, administra
range (53), so the port is configurable for deployments in which the service may
not be managed via an administrative account.
-##DNS Record Name Structure
+## DNS Record Name Structure
The DNS names of generated records are composed from the following elements (labels). Note that these elements must be compatible with DNS conventions (see “Preferred Name Syntax” in RFC 1035):
-* **domain** - the name of the cluster DNS domain. This name is provided as a
+* **domain** - the name of the cluster DNS domain. This name is provided as a
configuration property. In addition, it is this name that is configured at a parent DNS
server as the zone name for the defined yDNS zone (the zone for which the parent DNS
server will forward requests to yDNS). E.g. yarncluster.com
-* **username** - the name of the application deployer. This name is the simple shortname (for
+* **username** - the name of the application deployer. This name is the simple short-name (for
e.g. the primary component of the Kerberos principal) associated with the user launching
-the application. As the username is one of the elements of DNS names, it is expected
-that this also confirms DNS name conventions (RFC 1035 linked above) so special translation is performed for names with special characters like hyphens and spaces.
-* **application name** - the name of the deployed YARN application. This name is inferred
-from the YARN registry path to the application's node. Application name, rather thn application id, was chosen as a way of making it easy for users to refer to human-readable DNS
-names. This obviously mandates certain uniqueness properties on application names.
-* **container id** - the YARN assigned ID to a container (e.g.
+the application. As the username is one of the elements of DNS names, it is expected
+that this also confirms DNS name conventions (RFC 1035 linked above), so special translation is performed for names with special characters like hyphens and spaces.
+* **application name** - the name of the deployed YARN application. This name is inferred
+from the YARN registry path to the application's node. Application name, rather thn application id, was chosen as a way of making it easy for users to refer to human-readable DNS
+names. This obviously mandates certain uniqueness properties on application names.
+* **container id** - the YARN assigned ID to a container (e.g.
container_e3741_1454001598828_01_000004)
-* **component name** - the name assigned to the deployed component (for e.g. a master
+* **component name** - the name assigned to the deployed component (for e.g. a master
component). A component is a distributed element of an application or service that is
launched in a YARN container (e.g. an HBase master). One can imagine multiple
-components within an application. A component name is not yet a first class concept in
+components within an application. A component name is not yet a first class concept in
YARN, but is a very useful one that we are introducing here for the sake of yDNS
-entries. Many frameworks like MapReduce, Slider already have component names
-(though, as mentioned, they are not yet supported in YARN in a first class fashion).
-* **api** - the api designation for the exposed endpoint
+entries. Many frameworks like MapReduce, Slider already have component names
+(though, as mentioned, they are not yet supported in YARN in a first class fashion).
+* **api** - the api designation for the exposed endpoint
-###Notes about DNS Names
+### Notes about DNS Names
* In most instances, the DNS names can be easily distinguished by the number of
elements/labels that compose the name. The cluster’s domain name is always the last
element. After that element is parsed out, reading from right to left, the first element
-maps to the application user and so on. Wherever it is not easily distinguishable, naming conventions are used to disambiguate the name using a prefix such as
-“container” or suffix such as “api”. For example, an endpoint published as a
-management endpoint will be referenced with the name *management-api.griduser.yarncluster.com*.
-* Unique application name (per user) is not currently supported/guaranteed by YARN, but
+maps to the application user and so on. Wherever it is not easily distinguishable, naming conventions are used to disambiguate the name using a prefix such as
+“container” or suffix such as “api”. For example, an endpoint published as a
+management endpoint will be referenced with the name *management-api.griduser.yarncluster.com*.
+* Unique application name (per user) is not currently supported/guaranteed by YARN, but
it is supported by frameworks such as Apache Slider. The yDNS service currently
leverages the last element of the ZK path entry for the application as an
-application name. These application names have to be unique for a given user.
+application name. These application names have to be unique for a given user.
-##DNS Server Functionality
+## DNS Server Functionality
The primary functions of the DNS service are illustrated in the following diagram:
-###DNS record creation
+### DNS record creation
The following figure illustrates at slightly greater detail the DNS record creation and registration sequence (NOTE: service record updates would follow a similar sequence of steps,
distinguished only by the different event type):
-###DNS record removal
+### DNS record removal
Similarly, record removal follows a similar sequence
@@ -115,7 +118,7 @@ Similarly, record removal follows a similar sequence
(NOTE: The DNS Zone requires a record as an argument for the deletion method, thus
requiring similar parsing logic to identify the specific records that should be removed).
-###DNS Service initialization
+### DNS Service initialization
* The DNS service initializes both UDP and TCP listeners on a configured port. As
noted above, the default port of 53 is in a restricted range that is only accessible to an
account with administrative privileges.
@@ -124,18 +127,18 @@ standard DNS requests from users or other DNS servers (for example, DNS servers
YARN DNS service configured as a forwarder).
## Configuration
-The YARN DNS server reads its configuration properties from the yarnsite.xml file. The following are the DNS associated configuration properties:
+The YARN DNS server reads its configuration properties from the yarn-site.xml file. The following are the DNS associated configuration properties:
| Name | Description |
| ------------ | ------------- |
| hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. |
-| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. |
-| hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. |
-| hadoop.registry.dns.bind-port | The port number for the DNS listener. The default port is 53. However, since that port falls in a administratoronly range, typical deployments may need to specify an alternate port. |
+| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. |
+| hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. |
+| hadoop.registry.dns.bind-port | The port number for the DNS listener. The default port is 53. However, since that port falls in a administrator-only range, typical deployments may need to specify an alternate port. |
| hadoop.registry.dns.dnssec.enabled | Indicates whether the DNSSEC support is enabled. Default is false. |
-| hadoop.registry.dns.public-key | The base64 representation of the server’s public key. Leveraged for creating the DNSKEY Record provided for DNSSEC client requests. |
-| hadoop.registry.dns.private-key-file | The path to the standard DNSSEC private key file. Must only be readable by the DNS launching identity. See [dnssec-keygen](https://ftp.isc.org/isc/bind/cur/9.9/doc/arm/man.dnssec-keygen.html) documentation. |
-| hadoop.registry.dns-ttl | The default TTL value to associate with DNS records. The default value is set to 1 (a value of 0 has undefined behavior). A typical value should be approximate to the time it takes YARN to restart a failed container. |
-| hadoop.registry.dns.zone-subnet | An indicator of the IP range associated with the cluster containers. The setting is utilized for the generation of the reverse zone name. |
-| hadoop.registry.dns.zone-mask | The network mask associated with the zone IP range. If specified, it is utilized to ascertain the IP range possible and come up with an appropriate reverse zone name. |
-| hadoop.registry.dns.zones-dir | A directory containing zone configuration files to read during zone initialization. This directory can contain zone master files named *zone-name.zone*. See [here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file documentation.|
+| hadoop.registry.dns.public-key | The base64 representation of the server’s public key. Leveraged for creating the DNSKEY Record provided for DNSSEC client requests. |
+| hadoop.registry.dns.private-key-file | The path to the standard DNSSEC private key file. Must only be readable by the DNS launching identity. See [dnssec-keygen](https://ftp.isc.org/isc/bind/cur/9.9/doc/arm/man.dnssec-keygen.html) documentation. |
+| hadoop.registry.dns-ttl | The default TTL value to associate with DNS records. The default value is set to 1 (a value of 0 has undefined behavior). A typical value should be approximate to the time it takes YARN to restart a failed container. |
+| hadoop.registry.dns.zone-subnet | An indicator of the IP range associated with the cluster containers. The setting is utilized for the generation of the reverse zone name. |
+| hadoop.registry.dns.zone-mask | The network mask associated with the zone IP range. If specified, it is utilized to ascertain the IP range possible and come up with an appropriate reverse zone name. |
+| hadoop.registry.dns.zones-dir | A directory containing zone configuration files to read during zone initialization. This directory can contain zone master files named *zone-name.zone*. See [here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file documentation.|
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
deleted file mode 100644
index 0cac430..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web;
-
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.health.HealthCheckRegistry;
-import com.codahale.metrics.servlets.HealthCheckServlet;
-import com.codahale.metrics.servlets.MetricsServlet;
-import com.codahale.metrics.servlets.PingServlet;
-import com.codahale.metrics.servlets.ThreadDumpServlet;
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.container.filter.GZIPContentEncodingFilter;
-import com.sun.jersey.api.core.ResourceConfig;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.spi.container.servlet.ServletContainer;
-import org.apache.hadoop.yarn.webapp.Dispatcher;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.WebApp;
-import org.apache.slider.server.appmaster.web.rest.AMWadlGeneratorConfig;
-import org.apache.slider.server.appmaster.web.rest.AMWebServices;
-import org.apache.slider.server.appmaster.web.rest.SliderJacksonJaxbJsonProvider;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
-
-/**
- *
- */
-public class SliderAMWebApp extends WebApp {
- public static final String BASE_PATH = "slideram";
- public static final String CONTAINER_STATS = "/stats";
- public static final String CLUSTER_SPEC = "/spec";
-
- private final WebAppApi webAppApi;
-
- public SliderAMWebApp(WebAppApi webAppApi) {
- Preconditions.checkArgument(webAppApi != null, "webAppApi null");
- this.webAppApi = webAppApi;
- }
-
- @Override
- public void setup() {
- Logger.getLogger("com.sun.jersey").setLevel(Level.FINEST);
- // Make one of these to ensure that the jax-b annotations
- // are properly picked up.
- bind(SliderJacksonJaxbJsonProvider.class);
-
- // Get exceptions printed to the screen
- bind(GenericExceptionHandler.class);
- // bind the REST interface
- bind(AMWebServices.class);
-
- //bind(AMAgentWebServices.class);
- route("/", SliderAMController.class);
- route(CONTAINER_STATS, SliderAMController.class, "containerStats");
- route(CLUSTER_SPEC, SliderAMController.class, "specification");
- }
-
- @Override
- public void configureServlets() {
- setup();
-
- serve("/", "/__stop").with(Dispatcher.class);
-
- for (String path : this.getServePathSpecs()) {
- serve(path).with(Dispatcher.class);
- }
-
- serve(SYSTEM_HEALTHCHECK)
- .with(new HealthCheckServlet(new HealthCheckRegistry()));
- serve(SYSTEM_METRICS).with(new MetricsServlet(new MetricRegistry()));
- serve(SYSTEM_PING).with(new PingServlet());
- serve(SYSTEM_THREADS).with(new ThreadDumpServlet());
-
- String regex = "(?!/ws)";
- serveRegex(regex).with(SliderDefaultWrapperServlet.class);
-
- Map<String, String> params = new HashMap<>();
- params.put(ResourceConfig.FEATURE_IMPLICIT_VIEWABLES, "true");
- params.put(ServletContainer.FEATURE_FILTER_FORWARD_ON_404, "true");
- params.put(ResourceConfig.FEATURE_XMLROOTELEMENT_PROCESSING, "true");
- params.put(ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS, GZIPContentEncodingFilter.class.getName());
- params.put(ResourceConfig.PROPERTY_CONTAINER_RESPONSE_FILTERS, GZIPContentEncodingFilter.class.getName());
- //params.put("com.sun.jersey.spi.container.ContainerRequestFilters", "com.sun.jersey.api.container.filter.LoggingFilter");
- //params.put("com.sun.jersey.spi.container.ContainerResponseFilters", "com.sun.jersey.api.container.filter.LoggingFilter");
- //params.put("com.sun.jersey.config.feature.Trace", "true");
- params.put("com.sun.jersey.config.property.WadlGeneratorConfig",
- AMWadlGeneratorConfig.CLASSNAME);
- filter("/*").through(GuiceContainer.class, params);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderDefaultWrapperServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderDefaultWrapperServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderDefaultWrapperServlet.java
deleted file mode 100644
index 12c41ac..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderDefaultWrapperServlet.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web;
-
-import com.google.inject.Singleton;
-import org.apache.hadoop.yarn.webapp.DefaultWrapperServlet;
-
-import javax.servlet.RequestDispatcher;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletRequestWrapper;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-
-/**
- *
- */
-@Singleton
-public class SliderDefaultWrapperServlet extends DefaultWrapperServlet {
- @Override
- protected void doPost(HttpServletRequest req, HttpServletResponse resp)
- throws ServletException, IOException {
- RequestDispatcher rd = getServletContext().getNamedDispatcher("default");
-
- HttpServletRequest wrapped = new HttpServletRequestWrapper(req) {
- public String getServletPath() {
- return "";
- }
- };
-
- rd.forward(wrapped, resp);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
deleted file mode 100644
index 02f3f0c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web;
-
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.slider.server.appmaster.actions.QueueAccess;
-import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-
-/**
- * Interface to pass information from the Slider AppMaster to the WebApp
- */
-public interface WebAppApi {
-
- /**
- * The {@link AppState} for the current cluster
- */
- StateAccessForProviders getAppState();
-
- /**
- * Registry operations accessor
- * @return registry access
- */
- RegistryOperations getRegistryOperations();
-
- /**
- * Metrics and monitoring service
- * @return the (singleton) instance
- */
- MetricsAndMonitoring getMetricsAndMonitoring();
-
- /**
- * Get the queue accessor
- * @return the immediate and scheduled queues
- */
- QueueAccess getQueues();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
deleted file mode 100644
index f88f501..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web;
-
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.slider.server.appmaster.actions.QueueAccess;
-import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- *
- */
-public class WebAppApiImpl implements WebAppApi {
- private static final Logger log = LoggerFactory.getLogger(WebAppApiImpl.class);
-
- protected final StateAccessForProviders appState;
- private final RegistryOperations registryOperations;
- private final MetricsAndMonitoring metricsAndMonitoring;
- private final QueueAccess queues;
-
- public WebAppApiImpl(StateAccessForProviders appState,
- RegistryOperations registryOperations,
- MetricsAndMonitoring metricsAndMonitoring, QueueAccess queues) {
- checkNotNull(appState);
- this.queues = queues;
-
- this.registryOperations = registryOperations;
- this.appState = appState;
- this.metricsAndMonitoring = metricsAndMonitoring;
- }
-
- @Override
- public StateAccessForProviders getAppState() {
- return appState;
- }
-
- @Override
- public RegistryOperations getRegistryOperations() {
- return registryOperations;
- }
-
- @Override
- public MetricsAndMonitoring getMetricsAndMonitoring() {
- return metricsAndMonitoring;
- }
-
- @Override
- public QueueAccess getQueues() {
- return queues;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/AppLayout.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/AppLayout.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/AppLayout.java
deleted file mode 100644
index d9a2cda..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/AppLayout.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.layout;
-
-import org.apache.hadoop.yarn.webapp.SubView;
-import org.apache.slider.server.appmaster.web.view.IndexBlock;
-
-/**
- *
- */
-public class AppLayout extends WebUILayout {
-
- @Override
- protected Class<? extends SubView> content() {
- return IndexBlock.class;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/ClusterSpecificationView.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/ClusterSpecificationView.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/ClusterSpecificationView.java
deleted file mode 100644
index b54ca71..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/ClusterSpecificationView.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.layout;
-
-import org.apache.hadoop.yarn.webapp.SubView;
-import org.apache.slider.server.appmaster.web.view.ClusterSpecificationBlock;
-
-/**
- *
- */
-public class ClusterSpecificationView extends WebUILayout {
-
- @Override
- protected Class<? extends SubView> content() {
- return ClusterSpecificationBlock.class;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/ContainerStatsView.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/ContainerStatsView.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/ContainerStatsView.java
deleted file mode 100644
index 39ba0ad..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/ContainerStatsView.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.layout;
-
-import org.apache.hadoop.yarn.webapp.SubView;
-import org.apache.slider.server.appmaster.web.view.ContainerStatsBlock;
-
-
-
-/**
- *
- */
-public class ContainerStatsView extends WebUILayout {
-
- @Override
- protected Class<? extends SubView> content() {
- return ContainerStatsBlock.class;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/WebUILayout.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/WebUILayout.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/WebUILayout.java
deleted file mode 100644
index 1681f59..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/WebUILayout.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.layout;
-
-import org.apache.hadoop.yarn.webapp.SubView;
-import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout;
-import org.apache.slider.server.appmaster.web.view.NavBlock;
-
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
-
-/**
- *
- */
-public class WebUILayout extends TwoColumnLayout {
-
- @Override
- protected void preHead(Page.HTML<_> html) {
- set(ACCORDION_ID, "nav");
- set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
- }
-
- @Override
- protected Class<? extends SubView> nav() {
- return NavBlock.class;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGenerator.java
deleted file mode 100644
index 05aaa5b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGenerator.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest;
-
-import com.sun.jersey.server.wadl.ApplicationDescription;
-import com.sun.jersey.server.wadl.WadlGenerator;
-import com.sun.jersey.server.wadl.WadlGeneratorImpl;
-import com.sun.research.ws.wadl.Application;
-import com.sun.research.ws.wadl.Resource;
-import com.sun.research.ws.wadl.Resources;
-
-import java.util.Iterator;
-import java.util.List;
-
-/**
- *
- */
-public class AMWadlGenerator extends WadlGeneratorImpl {
- @Override
- /**
- * This method is called once the WADL application has been assembled, so it
- * affords an opportunity to edit the resources presented by the WADL. In
- * this case, we're removing the internal "/agents" resources.
- */
- public void attachTypes(ApplicationDescription egd) {
- super.attachTypes(egd);
-
- Application application = egd.getApplication();
- List<Resources> resources = application.getResources();
-
- for (Resources appResources : resources) {
- List<Resource> resourceList = appResources.getResource();
- for (Resource appResource : resourceList) {
- String path = appResource.getPath();
- if (RestPaths.SLIDER_CONTEXT_ROOT.equals(path)) {
- List<Object> sliderResources = appResource.getMethodOrResource();
- Iterator<Object> itor = sliderResources.iterator();
- while (itor.hasNext()) {
- Object sliderRes = itor.next();
- if (sliderRes instanceof Resource) {
- Resource res = (Resource) sliderRes;
- if (RestPaths.SLIDER_SUBPATH_AGENTS.equals(res.getPath())) {
- // assuming I'll get a list modification issue if I remove at this
- // point
- itor.remove();
- }
- }
- }
- }
- }
- }
- }
-
- @Override
- public void setWadlGeneratorDelegate(WadlGenerator delegate) {
- // do nothing
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGeneratorConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGeneratorConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGeneratorConfig.java
deleted file mode 100644
index ea9f22b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGeneratorConfig.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest;
-
-import com.sun.jersey.api.wadl.config.WadlGeneratorConfig;
-import com.sun.jersey.api.wadl.config.WadlGeneratorDescription;
-
-import java.util.List;
-
-/**
- * App master's WADL generation support
- */
-public class AMWadlGeneratorConfig extends WadlGeneratorConfig {
-
- public static final String CLASSNAME = "org.apache.slider.server.appmaster.web.rest.AMWadlGeneratorConfig";
- @Override
- public List<WadlGeneratorDescription> configure() {
- return generator(AMWadlGenerator.class).descriptions();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
deleted file mode 100644
index 44259d3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest;
-
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.rest.application.actions.RestActionStop;
-import org.apache.slider.server.appmaster.web.rest.application.actions.StopResponse;
-import org.apache.slider.server.appmaster.web.rest.publisher.PublisherResource;
-import org.apache.slider.server.appmaster.web.rest.registry.RegistryResource;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.UriInfo;
-
-import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.ACTION_STOP;
-
-/**
- * The available REST services exposed by a slider AM.
- */
-@Singleton
-@Path(RestPaths.SLIDER_CONTEXT_ROOT)
-public class AMWebServices {
-
- /** AM/WebApp info object */
- private WebAppApi slider;
- private final PublisherResource publisherResource;
- private final RegistryResource registryResource;
-
- @Inject
- public AMWebServices(WebAppApi slider) {
- this.slider = slider;
- publisherResource = new PublisherResource(slider);
- registryResource = new RegistryResource(slider);
- }
- //TODO add an endpoint for exposing configs
-
- @Path(RestPaths.SLIDER_SUBPATH_PUBLISHER)
- public PublisherResource getPublisherResource() {
- return publisherResource;
- }
-
- @Path(RestPaths.SLIDER_SUBPATH_REGISTRY)
- public RegistryResource getRegistryResource() {
- return registryResource;
- }
-
-
- @GET
- @Path(RestPaths.SLIDER_SUBPATH_APPLICATION)
- @Produces({APPLICATION_JSON})
- public Application getApplicationResource() {
- return slider.getAppState().getApplication();
- }
-
- @POST
- @Path(ACTION_STOP)
- @Produces({APPLICATION_JSON})
- public StopResponse actionStop(@Context HttpServletRequest request,
- @Context UriInfo uriInfo,
- String body) {
- return new RestActionStop(slider).stop(request, uriInfo, body);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
deleted file mode 100644
index 7ff83b6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest;
-
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.registry.client.exceptions.AuthenticationFailedException;
-import org.apache.hadoop.registry.client.exceptions.NoPathPermissionsException;
-import org.apache.hadoop.yarn.webapp.ForbiddenException;
-import org.apache.hadoop.yarn.webapp.NotFoundException;
-import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Response;
-import java.io.FileNotFoundException;
-import java.net.URI;
-import java.net.URL;
-
-/**
- * Abstract resource base class for REST resources
- * that use the slider WebAppApi
- */
-public abstract class AbstractSliderResource {
- private static final Logger log =
- LoggerFactory.getLogger(AbstractSliderResource.class);
- protected final WebAppApi slider;
- protected final MetricsAndMonitoring metricsAndMonitoring;
-
- protected AbstractSliderResource(WebAppApi slider) {
- this.slider = slider;
- metricsAndMonitoring = slider.getMetricsAndMonitoring();
- }
-
- /**
- * Generate a redirect to the WASL
- * @param request to base the URL on
- * @return a 302 response
- */
- protected Response redirectToAppWadl(HttpServletRequest request) {
- try {
- URI location = new URL(request.getScheme(),
- request.getServerName(),
- request.getServerPort(),
- RestPaths.APPLICATION_WADL).toURI();
- return Response.temporaryRedirect(location).build();
- } catch (Exception e) {
- log.error("Error during redirect to WADL", e);
- throw new WebApplicationException(Response.serverError().build());
- }
- }
-
- /**
- * Convert any exception caught into a web application
- * exception for rethrowing
- * @param path path of request
- * @param ex exception
- * @return an exception to throw
- */
- public WebApplicationException buildException(String path,
- Exception ex) {
- try {
- throw ex;
- } catch (WebApplicationException e) {
- // rethrow direct
- throw e;
- } catch (FileNotFoundException e) {
- return new NotFoundException("Not found: " + path);
- } catch (PathNotFoundException e) {
- return new NotFoundException("Not found: " + path);
- } catch (AuthenticationFailedException e) {
- return new ForbiddenException(path);
- } catch (NoPathPermissionsException e) {
- return new ForbiddenException(path);
- } catch (Exception e) {
- log.error("Error during generation of response: {}", e, e);
- return new WebApplicationException(e);
- }
- }
-
- /**
- * Mark an GET operation on a path
- * @param verb HTTP Verb
- * @param path path relative to slider API
- */
- protected void mark(String verb, String path) {
- metricsAndMonitoring.markMeterAndCounter(verb + "-" + path);
- }
-
- /**
- * Mark an GET operation on a path
- * @param verb HTTP Verb
- * @param path path relative to slider API
- */
- protected void mark(String verb, String path, String subpath) {
- metricsAndMonitoring.markMeterAndCounter(verb + "-" + path + subpath);
- }
-
- /**
- * Mark a GET operation on a path
- * @param path path relative to slider API
- */
- protected void markGet(String path) {
- mark("GET", path);
- }
-
- /**
- * Mark a GET operation on a path
- * @param path path relative to slider API
- */
- protected void markGet(String path, String subpath) {
- mark("GET", path, subpath);
- }
-
- /**
- * Mark a GET operation on a path
- * @param path path relative to slider API
- */
- protected void markPost(String path, String subpath) {
- mark("POST", path, subpath);
- }
-
- /**
- * Mark a GET operation on a path
- * @param path path relative to slider API
- */
- protected void markPut(String path, String subpath) {
- mark("PUT", path, subpath);
- }
-
- /**
- * Mark a GET operation on a path
- * @param path path relative to slider API
- */
- protected void markDelete(String path, String subpath) {
- mark("DELETE", path, subpath);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java
deleted file mode 100644
index b4a92ba..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest;
-
-import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
-import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter;
-import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpPrincipal;
-import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpServletRequestWrapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.FilterChain;
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.Cookie;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-
-/**
- * This is a filter which is used to forward insecure operations
- * There's some metrics to track all operations too
- */
-public class InsecureAmFilter extends AmIpFilter {
- public static final String WS_CONTEXT_ROOT = "slider.rest.context.root";
- protected static final Logger log =
- LoggerFactory.getLogger(InsecureAmFilter.class);
-
- private String wsContextRoot;
-
-
- @Override
- public void init(FilterConfig conf) throws ServletException {
- super.init(conf);
- wsContextRoot = conf.getInitParameter(WS_CONTEXT_ROOT);
- if (wsContextRoot == null) {
- throw new ServletException("No value set for " + WS_CONTEXT_ROOT);
- }
- }
-
- private void rejectNonHttpRequests(ServletRequest req) throws
- ServletException {
- if (!(req instanceof HttpServletRequest)) {
- throw new ServletException("This filter only works for HTTP/HTTPS");
- }
- }
-
- @Override
- public void doFilter(ServletRequest req,
- ServletResponse resp,
- FilterChain chain) throws IOException, ServletException {
- rejectNonHttpRequests(req);
- HttpServletRequest httpReq = (HttpServletRequest) req;
-
-
- String requestURI = httpReq.getRequestURI();
- if (requestURI == null || !requestURI.startsWith(wsContextRoot)) {
- // hand off to the AM filter if it is not the context root
- super.doFilter(req, resp, chain);
- return;
- }
-
- String user = null;
-
- if (httpReq.getCookies() != null) {
- for (Cookie c : httpReq.getCookies()) {
- if (WebAppProxyServlet.PROXY_USER_COOKIE_NAME.equals(c.getName())) {
- user = c.getValue();
- break;
- }
- }
- }
-
- if (user == null) {
- log.debug("Could not find " + WebAppProxyServlet.PROXY_USER_COOKIE_NAME
- + " cookie, so user will not be set");
- chain.doFilter(req, resp);
- } else {
- final AmIpPrincipal principal = new AmIpPrincipal(user);
- ServletRequest requestWrapper = new AmIpServletRequestWrapper(httpReq,
- principal);
- chain.doFilter(requestWrapper, resp);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilterInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilterInitializer.java
deleted file mode 100644
index 42a5bdd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilterInitializer.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.FilterContainer;
-import org.apache.hadoop.http.FilterInitializer;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class InsecureAmFilterInitializer extends FilterInitializer {
- private static final String FILTER_NAME = "AM_PROXY_FILTER";
- private static final String FILTER_CLASS =
- InsecureAmFilter.class.getCanonicalName();
- private static final String HTTPS_PREFIX = "https://";
- private static final String HTTP_PREFIX = "http://";
-
- static final String PROXY_HOSTS = "PROXY_HOSTS";
- static final String PROXY_HOSTS_DELIMITER = ",";
- static final String PROXY_URI_BASES = "PROXY_URI_BASES";
- static final String PROXY_URI_BASES_DELIMITER = ",";
-
- private Configuration configuration;
-
- public static final String NAME =
- "org.apache.slider.server.appmaster.web.rest.InsecureAmFilterInitializer";
-
- @Override
- public void initFilter(FilterContainer container, Configuration conf) {
- configuration = conf;
- Map<String, String> params = new HashMap<String, String>();
- String proxy = WebAppUtils.getProxyHostAndPort(conf);
- String[] parts = proxy.split(":");
- params.put(InsecureAmFilter.PROXY_HOST, parts[0]);
- // todo: eventually call WebAppUtils.getHttpSchemePrefix
- params.put(InsecureAmFilter.PROXY_URI_BASE, getHttpSchemePrefix()
- + proxy +
- getApplicationWebProxyBase());
- params.put(InsecureAmFilter.WS_CONTEXT_ROOT, RestPaths.WS_CONTEXT_ROOT);
- container.addFilter(FILTER_NAME, FILTER_CLASS, params);
- }
-
- private void classicAmFilterInitializerInit(FilterContainer container,
- Configuration conf) {
- Map<String, String> params = new HashMap<String, String>();
- List<String> proxies = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf);
- StringBuilder sb = new StringBuilder();
- for (String proxy : proxies) {
- sb.append(proxy.split(":")[0]).append(PROXY_HOSTS_DELIMITER);
- }
- sb.setLength(sb.length() - 1);
- params.put(PROXY_HOSTS, sb.toString());
-
- String prefix = WebAppUtils.getHttpSchemePrefix(conf);
- String proxyBase = getApplicationWebProxyBase();
- sb = new StringBuilder();
- for (String proxy : proxies) {
- sb.append(prefix).append(proxy).append(proxyBase)
- .append(PROXY_HOSTS_DELIMITER);
- }
- sb.setLength(sb.length() - 1);
- params.put(PROXY_URI_BASES, sb.toString());
-
- }
-
- @VisibleForTesting
- protected String getApplicationWebProxyBase() {
- return System.getenv(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV);
- }
-
- private String getHttpSchemePrefix() {
- return HttpConfig.Policy.HTTPS_ONLY ==
- HttpConfig.Policy.fromString(configuration
- .get(
- YarnConfiguration.YARN_HTTP_POLICY_KEY,
- YarnConfiguration.YARN_HTTP_POLICY_DEFAULT))
- ? HTTPS_PREFIX : HTTP_PREFIX;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
deleted file mode 100644
index 581f5b4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest;
-
-/**
- * Paths in the REST App
- */
-public class RestPaths {
-
- public static final String WS_CONTEXT = "ws";
- public static final String AGENT_WS_CONTEXT = "ws";
-
- /**
- * Root path for the web services context: {@value}
- */
- public static final String WS_CONTEXT_ROOT = "/" + WS_CONTEXT;
-
- /**
- * agent content root: {@value}
- */
- public static final String SLIDER_CONTEXT_ROOT = WS_CONTEXT_ROOT + "/v1";
- public static final String RELATIVE_API = WS_CONTEXT + "/v1";
- public static final String MANAGEMENT = "mgmt";
- public static final String SLIDER_SUBPATH_MANAGEMENT = "/" + MANAGEMENT;
- public static final String SLIDER_SUBPATH_AGENTS = "/agents";
- public static final String SLIDER_SUBPATH_PUBLISHER = "/publisher";
-
-
- /**
- * Publisher: {@value}
- */
- public static final String SLIDER_PATH_PUBLISHER = SLIDER_CONTEXT_ROOT
- + SLIDER_SUBPATH_PUBLISHER;
-
- public static final String RELATIVE_PATH_PUBLISHER = RELATIVE_API
- + SLIDER_SUBPATH_PUBLISHER;
-
- /**
- * Registry subpath: {@value}
- */
- public static final String SLIDER_SUBPATH_REGISTRY = "/registry";
-
- /**
- * Registry: {@value}
- */
- public static final String SLIDER_PATH_REGISTRY = SLIDER_CONTEXT_ROOT
- + SLIDER_SUBPATH_REGISTRY;
- public static final String RELATIVE_PATH_REGISTRY = RELATIVE_API
- + SLIDER_SUBPATH_REGISTRY;
-
-
- /**
- * The regular expressions used to define valid configuration names/url path
- * fragments: {@value}
- */
- public static final String PUBLISHED_CONFIGURATION_REGEXP
- = "[a-z0-9][a-z0-9_\\+-]*";
-
- public static final String PUBLISHED_CONFIGURATION_SET_REGEXP
- = "[a-z0-9][a-z0-9_.\\+-]*";
-
- public static final String SLIDER_CONFIGSET = "slider";
- public static final String SLIDER_EXPORTS = "exports";
-
- public static final String SLIDER_CLASSPATH = "classpath";
-
- /**
- * Codahale Metrics - base path: {@value}
- */
-
- public static final String SYSTEM = "/system";
-
-
-
- /**
- * Codahale Metrics - health: {@value}
- */
- public static final String SYSTEM_HEALTHCHECK = SYSTEM + "/health";
- /**
- * Codahale Metrics - metrics: {@value}
- */
- public static final String SYSTEM_METRICS = SYSTEM + "/metrics";
- /**
- * Codahale Metrics - metrics as JSON: {@value}
- */
- public static final String SYSTEM_METRICS_JSON = SYSTEM_METRICS + "?format=json";
- /**
- * Codahale Metrics - ping: {@value}
- */
- public static final String SYSTEM_PING = SYSTEM + "/ping";
- /**
- * Codahale Metrics - thread dump: {@value}
- */
- public static final String SYSTEM_THREADS = SYSTEM + "/threads";
-
- /**
- * application subpath
- */
- public static final String SLIDER_SUBPATH_APPLICATION = "/application";
-
- /**
- * management path: {@value}
- */
- public static final String SLIDER_PATH_APPLICATION =
- SLIDER_CONTEXT_ROOT + SLIDER_SUBPATH_APPLICATION;
-
-
- public static final String APPLICATION_WADL = "/application.wadl";
- public static final String LIVE = "/live";
- public static final String LIVE_RESOURCES = "/live/resources";
- public static final String LIVE_CONTAINERS = "/live/containers";
- public static final String LIVE_COMPONENTS = "/live/components";
- public static final String LIVE_NODES = "/live/nodes";
- public static final String LIVE_LIVENESS = "/live/liveness";
- public static final String LIVE_STATISTICS = "/live/statistics";
- public static final String MODEL = "/model";
- public static final String MODEL_DESIRED = MODEL +"/desired";
- public static final String MODEL_DESIRED_APPCONF = MODEL_DESIRED +"/appconf";
- public static final String MODEL_DESIRED_RESOURCES = MODEL_DESIRED +"/resources";
- public static final String MODEL_RESOLVED = "/model/resolved";
- public static final String MODEL_RESOLVED_APPCONF = MODEL_RESOLVED +"/appconf";
- public static final String MODEL_RESOLVED_RESOURCES = MODEL_RESOLVED +"/resources";
- public static final String MODEL_INTERNAL = "/model/internal";
-
- public static final String ACTION = "/action";
- public static final String ACTION_PING = ACTION + "/ping";
- public static final String ACTION_STOP = ACTION + "/stop";
-
- /**
- * Path to a role
- * @param name role name
- * @return a path to it
- */
- public String pathToRole(String name) {
-
- // ws/v1/slider/application/live/components/$name
- return SLIDER_PATH_APPLICATION + LIVE_COMPONENTS + "/" + name;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/SliderJacksonJaxbJsonProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/SliderJacksonJaxbJsonProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/SliderJacksonJaxbJsonProvider.java
deleted file mode 100644
index 86d68a8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/SliderJacksonJaxbJsonProvider.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest;
-
-import com.google.inject.Singleton;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.codehaus.jackson.jaxrs.JacksonJaxbJsonProvider;
-import org.codehaus.jackson.map.AnnotationIntrospector;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.introspect.JacksonAnnotationIntrospector;
-import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
-
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.ext.Provider;
-
-/**
- * Implementation of JAX-RS abstractions based on {@link
- * JacksonJaxbJsonProvider} needed to deserialize JSON content to, or serialize
- * it from, POJO objects.
- */
-@Singleton
-@Provider
-@Unstable
-@Private
-public class SliderJacksonJaxbJsonProvider extends JacksonJaxbJsonProvider {
-
- public SliderJacksonJaxbJsonProvider() {
- }
-
- @Override
- public ObjectMapper locateMapper(Class<?> type, MediaType mediaType) {
- ObjectMapper mapper = super.locateMapper(type, mediaType);
- AnnotationIntrospector introspector = new AnnotationIntrospector.Pair(
- new JaxbAnnotationIntrospector(),
- new JacksonAnnotationIntrospector()
- );
- mapper.setAnnotationIntrospector(introspector);
- //mapper.setSerializationInclusion(Inclusion.NON_NULL);
- return mapper;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java
deleted file mode 100644
index d23fcee..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.application;
-
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
-
-public class ApplicationResouceContentCacheFactory {
- public static final int LIFESPAN = 500;
-
- /**
- * Build the content cache
- * @param state state view
- */
- public static ContentCache createContentCache(
- StateAccessForProviders state) {
- ContentCache cache = new ContentCache();
- return cache;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionPing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionPing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionPing.java
deleted file mode 100644
index 96b4f75..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionPing.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.application.actions;
-
-import org.apache.slider.api.types.PingInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.core.UriInfo;
-import java.util.Locale;
-
-public class RestActionPing {
- private static final Logger log =
- LoggerFactory.getLogger(RestActionPing.class);
-
- public RestActionPing() {
- }
-
- public PingInformation ping(HttpServletRequest request, UriInfo uriInfo, String body) {
- String verb = request.getMethod();
- log.info("Ping {}", verb);
- PingInformation pingInformation = new PingInformation();
- pingInformation.time = System.currentTimeMillis();
- pingInformation.verb = verb;
- pingInformation.body = body;
- String text =
- String.format(Locale.ENGLISH,
- "Ping verb %s received at %tc",
- verb, pingInformation.time);
- pingInformation.text = text;
- return pingInformation;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionStop.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionStop.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionStop.java
deleted file mode 100644
index 544f589..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionStop.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.application.actions;
-
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.slider.core.main.LauncherExitCodes;
-import org.apache.slider.server.appmaster.actions.ActionStopSlider;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.core.UriInfo;
-import java.util.Locale;
-import java.util.concurrent.TimeUnit;
-
-public class RestActionStop {
- private static final Logger log =
- LoggerFactory.getLogger(RestActionStop.class);
-
- private final WebAppApi slider;
-
- public RestActionStop(WebAppApi slider) {
- this.slider = slider;
- }
-
- public StopResponse stop(HttpServletRequest request, UriInfo uriInfo, String body) {
- String verb = request.getMethod();
- log.info("Ping {}", verb);
- StopResponse response = new StopResponse();
- response.verb = verb;
- long time = System.currentTimeMillis();
- String text =
- String.format(Locale.ENGLISH,
- "Stopping action %s received at %tc",
- verb, time);
- response.text = text;
- log.info(text);
- ActionStopSlider stopSlider =
- new ActionStopSlider(text,
- 1000,
- TimeUnit.MILLISECONDS,
- LauncherExitCodes.EXIT_SUCCESS,
- FinalApplicationStatus.SUCCEEDED,
- text);
- log.info("SliderAppMasterApi.stopCluster: {}", stopSlider);
- slider.getQueues().schedule(stopSlider);
-
- return response;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/StopResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/StopResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/StopResponse.java
deleted file mode 100644
index d591f57..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/StopResponse.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.application.actions;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class StopResponse {
- public String verb;
- public String text;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/package-info.java
deleted file mode 100644
index 34eb82c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-/**
- * This package contains resources related to the application
- * REST API
- */
-package org.apache.slider.server.appmaster.web.rest.application;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java
deleted file mode 100644
index 22fd0fe..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.application.resources;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A cached resource is one that can be stored and served up, with a refresh
- * only taking place when the expiry happens.
- *
- * The refresh check/refresh is synchronized.
- * @param <T> type to return
- */
-public class CachedContent<T> {
- private static final Logger log =
- LoggerFactory.getLogger(CachedContent.class);
- private T cachedValue;
- private long expires;
- private final long lifespan;
- private final ResourceRefresher<T> refresh;
- private int refreshCounter;
-
- public CachedContent(long lifespan,
- ResourceRefresher<T> refresh) {
- this.lifespan = lifespan;
- this.refresh = refresh;
- }
-
- /**
- * Get the value, triggering a refresh if needed
- * @return the cached or latest value.
- */
- public T get() throws Exception {
- maybeRefresh();
- return getCachedValue();
- }
-
- /**
- * Get the cached value without any expiry check
- * @return the last value set. May be null.
- */
- public synchronized T getCachedValue() {
- return cachedValue;
- }
-
- public synchronized int getRefreshCounter() {
- return refreshCounter;
- }
-
- /**
- * Get the lifespan in millis of the cached value
- * @return the lifespan
- */
- public long getLifespan() {
- return lifespan;
- }
-
- /**
- * Maybe refresh the content
- * @return true if a refresh took place.
- */
- public synchronized boolean maybeRefresh() throws Exception {
- long now = now();
- if (cachedValue == null || now >= expires) {
- log.debug("Refreshing at time {}", now);
- forceRefresh();
- log.debug("Refreshed value now {}", cachedValue);
- return true;
- }
- return false;
- }
-
- protected long now() {
- return Time.monotonicNow();
- }
-
- /**
- * Force a refresh and reset the expiry counter
- * @return the new value
- */
- protected synchronized T forceRefresh() throws Exception {
- refreshCounter ++;
- T updated = refresh.refresh();
- Preconditions.checkNotNull(updated);
- cachedValue = updated;
- expires = now() + lifespan;
- return cachedValue;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("CachedContent{");
- sb.append(" expires=").append(expires);
- sb.append(", lifespan=").append(lifespan);
- sb.append(", refresh=").append(refresh);
- sb.append(", refreshCounter=").append(refreshCounter);
- sb.append(", cached=").append(cachedValue);
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
deleted file mode 100644
index 8f026a1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.application.resources;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * Cache of content
- */
-public class ContentCache extends ConcurrentHashMap<String, CachedContent> {
-
- public ContentCache(int initialCapacity) {
- super(initialCapacity);
- }
-
- public ContentCache() {
- }
-
-
- public Object lookup(String key) throws Exception {
- CachedContent content = get(key);
- if (content == null) {
- throw new FileNotFoundException("no content for path " + key);
- }
- return content.get();
- }
-
-
- /**
- * Lookup a cached item. If an exception is raised on the refresh...
- * <ol>
- * <li>IOExceptions are thrown directly</li>
- * <li>Other exceptions are wrapped with an IOExceptions</li>
- * </ol>
- * @param key
- * @return
- * @throws IOException
- */
- public Object lookupWithIOE(String key) throws IOException {
- try {
- return lookup(key);
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException("Looking up " + key + ": " + e, e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceRefresher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceRefresher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceRefresher.java
deleted file mode 100644
index 35f0367..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceRefresher.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.application.resources;
-
-/**
- * Interface which must be implemented to act as a source for cached content.
- * @param <T> type to return
- */
-public interface ResourceRefresher<T> {
- /**
- * Build an up to date version of the data
- * @return a new instance of the (JSON serializable) data
- */
- T refresh() throws Exception;
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[77/86] [abbrv] hadoop git commit: YARN-7126. Create introductory
site documentation for YARN native services. Contributed by Gour Saha
Posted by ji...@apache.org.
YARN-7126. Create introductory site documentation for YARN native services. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54d2e6be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54d2e6be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54d2e6be
Branch: refs/heads/yarn-native-services
Commit: 54d2e6be4a3ce5bc8eb4bfc77fd277835a11cea0
Parents: 5fec5b7
Author: Jian He <ji...@apache.org>
Authored: Fri Sep 1 16:19:31 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 25 16:37:24 2017 -0700
----------------------------------------------------------------------
LICENSE.txt | 1 +
.../native-services/NativeServicesIntro.md | 96 +++++++++++++++++++-
2 files changed, 96 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54d2e6be/LICENSE.txt
----------------------------------------------------------------------
diff --git a/LICENSE.txt b/LICENSE.txt
index 3f50521..46ee108 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1776,6 +1776,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
The binary distribution of this product bundles these dependencies under the
following license:
FindBugs-jsr305 3.0.0
+dnsjava 2.1.7, Copyright (c) 1998-2011, Brian Wellington. All rights reserved.
--------------------------------------------------------------------------------
(2-clause BSD)
Redistribution and use in source and binary forms, with or without
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54d2e6be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
index 89fefe9..e6a4e91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
@@ -10,4 +10,98 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
--->
\ No newline at end of file
+-->
+
+# Introduction: YARN Native Services
+
+## Overview
+YARN Native Services provides first class framework support and APIs to host long running services natively in YARN. In addition to launching services, the new APIs support performing lifecycle management operations, such as flex service components up/down, manage lifetime, upgrade the service to a newer version, and stop/restart/delete the service.
+
+The native services capabilities are built on the existing low-level resource management API provided by YARN that can support any type of application. Other application frameworks like Hadoop MapReduce already expose higher level APIs that users can leverage to run applications on top of YARN. With the advent of containerization technologies like Docker, providing first class support and APIs for long running services at the framework level made sense.
+
+Relying on a framework has the advantage of exposing a simpler usage model to the user by enabling service configuration and launch through specification (without writing new code), as well as hiding complex low-level details including state management and fault-tolerance etc. Users/operators of existing services typically like to avoid modifying an existing service to be aware of YARN. With first class support capable of running a single Docker image as well as complex assemblies comprised of multiple Docker images, there is no need for service owners to be aware of YARN. Developers of new services do not have to worry about YARN internals and only need to focus on containerization of their service(s).
+
+## First class support for services
+In order to natively provide first class support for long running services, several new features and improvements have been made at the framework level.
+
+### Incorporate Apache Slider into Apache YARN
+Apache Slider, which existed as a separate incubator project has been merged into YARN to kick start the first class support. Apache Slider is a universal Application Master (AM) which had several key features built in - fault tolerance of service containers and AM, work-preserving AM restarts, service logs management, service management like flex up/down, stop/start, and rolling upgrade to newer service versions, etc. Of course lot more work has been done on top of what Apache Slider brought in, details of which follow.
+
+### Native Services API
+A significant effort has gone into simplifying the user facing story for building services. In the past, bringing a new service to YARN was not a pleasant experience. The APIs of existing frameworks are either too low-level (native YARN), require writing new code (for frameworks with programmatic APIs) or require writing a complex spec (for declarative frameworks).
+
+The new REST APIs are very simple to use. The REST layer acts as a single point of entry for creation and lifecycle management of YARN services. Services here can range from simple single-component apps to the most complex, multi-component applications needing special orchestration needs.
+
+Plan is to make this a unified REST based entry point for other important features like resource-profile management ([YARN-3926](https://issues.apache.org/jira/browse/YARN-4793)), package-definitions' lifecycle-management and service-discovery ([YARN-913](https://issues.apache.org/jira/browse/YARN-913)/[YARN-4757](https://issues.apache.org/jira/browse/YARN-4757)).
+
+### Native Services Discovery
+The new discovery solution exposes the registry information through a more generic and widely used mechanism: DNS. Service Discovery via DNS uses the well-known DNS interfaces to browse the network for services. Having the registry information exposed via DNS simplifies the life of services.
+
+The previous read mechanisms of YARN Service Registry were limited to a registry specific (java) API and a REST interface. In practice, this made it very difficult for wiring up existing clients and services. For e.g., dynamic configuration of dependent endpoints of a service was not easy to implement using the registry-read mechanisms, **without** code-changes to existing services. These are solved by the DNS based service discovery.
+
+### Scheduling
+[YARN-6592](https://issues.apache.org/jira/browse/YARN-6592) covers a host of scheduling features that are useful for short-running applications and services alike. Below, are a few very important YARN core features that help schedule services better. Without these, running services on YARN is a hassle.
+
+* Affinity (TBD)
+* Anti-affinity (TBD)
+* Gang scheduling (TBD)
+* Malleable container sizes ([YARN-1197](https://issues.apache.org/jira/browse/YARN-1197))
+
+### Resource Profiles
+YARN always had support for memory as a resource, inheriting it from Hadoop-(1.x)’s MapReduce platform. Later support for CPU as a resource ([YARN-2](https://issues.apache.org/jira/browse/YARN-2)/[YARN-3](https://issues.apache.org/jira/browse/YARN-3)) was added. Multiple efforts added support for various other resource-types in YARN such as disk ([YARN-2139](https://issues.apache.org/jira/browse/YARN-2139)), and network ([YARN-2140](https://issues.apache.org/jira/browse/YARN-2140)), specifically benefiting long running services.
+
+In many systems outside of YARN, users are already accustomed to specifying their desired ‘box’ of requirements where each box comes with a predefined amount of each resources. Admins would define various available box-sizes (small, medium, large etc) and users would pick the ones they desire and everybody is happy. In [YARN-3926](https://issues.apache.org/jira/browse/YARN-3926), YARN introduces Resource Profiles which extends the YARN resource model for easier resource-type management and profiles. This helps in two ways - the system can schedule applications better and it can perform intelligent over-subscription of resources where applicable.
+
+Resource profiles are all the more important for services since -
+* Similar to short running apps, you don’t have to fiddle with varying resource-requirements for each container type
+* Services usually end up planning for peak usages, leaving a lot of possibility of barren utilization
+
+### Special handling of preemption and container reservations
+TBD.
+
+Preemption and reservation of long running containers have different implications from regular ones. Preemption of resources in YARN today works by killing of containers. For long-lived services this is unacceptable. Also, scheduler should avoid allocating long running containers on borrowed resources. [YARN-4724](https://issues.apache.org/jira/browse/YARN-4724) will address some of these special recognition of service containers.
+
+### Container auto-restarts
+If a service container dies, expiring container's allocation and releasing the allocation is undesirable in many cases. Long running containers may exit for various reasons, crash and need to restart but forcing them to go through the complete scheduling cycle, resource localization, etc. is both unnecessary and expensive.
+
+Services can enable app-specific policies to prevent NodeManagers to automatically restart containers. [YARN-3998](https://issues.apache.org/jira/browse/YARN-3998) implements a retry-policy to let NM re-launch a service container when it fails.
+
+### Container allocation re-use for application upgrades
+TBD.
+
+Auto-restart of containers will support upgrade of service containers without reclaiming the resources first. During an upgrade, with multitude of other applications running in the system, giving up and getting back resources allocated to the service is hard to manage. Node-Labels help this cause but are not straight-forward to use to address the app-specific use-cases. The umbrella [YARN-4726](https://issues.apache.org/jira/browse/YARN-4726) along with [YARN-5620](https://issues.apache.org/jira/browse/YARN-5620) and [YARN-4470](https://issues.apache.org/jira/browse/YARN-4470) will take care of this.
+
+### Dynamic Configurations
+Most production-level services require dynamic configurations to manage and simplify their lifecycle. Container’s resource size, local/work dirs and log-dirs are the most basic information services need. Service's endpoint details (host/port), their inter-component dependencies, health-check endpoints, etc. are all critical to the success of today's real-life services.
+
+### Resource re-localization for reconfiguration/upgrades
+TBD
+
+### Service Registry
+TBD
+
+### Service persistent storage and volume support
+TBD
+
+### Packaging
+TBD
+
+### Container image registry (private, public and hybrid)
+TBD
+
+### Container image management and APIs
+TBD
+
+### Container image storage
+TBD
+
+### Monitoring
+TBD
+
+### Metrics
+TBD
+
+### Service Logs
+TBD
+
+
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[76/86] [abbrv] hadoop git commit: Rebase onto latest trunk. minor
conflicts
Posted by ji...@apache.org.
Rebase onto latest trunk. minor conflicts
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38190819
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38190819
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38190819
Branch: refs/heads/yarn-native-services
Commit: 3819081963d1f28f7a18c5eb00f7b9fe9c6558bf
Parents: d5fad4e
Author: Jian He <ji...@apache.org>
Authored: Wed Aug 30 22:48:35 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 25 16:37:24 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/util/Apps.java | 2 ++
.../rmapp/attempt/RMAppAttemptImpl.java | 35 ++------------------
2 files changed, 4 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38190819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
index 75b4633..e88d2b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
@@ -150,6 +150,8 @@ public class Apps {
}
// Check if should black list the node based on container exit status
+ @Private
+ @Unstable
public static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) {
switch (exitStatus) {
case ContainerExitStatus.PREEMPTED:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38190819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index fa29e00..a0aa284 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -109,6 +109,7 @@ import org.apache.hadoop.yarn.state.MultipleArcTransition;
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.BoundedAppender;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -1556,38 +1557,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
}
}
- private static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) {
- switch (exitStatus) {
- case ContainerExitStatus.PREEMPTED:
- case ContainerExitStatus.KILLED_BY_RESOURCEMANAGER:
- case ContainerExitStatus.KILLED_BY_APPMASTER:
- case ContainerExitStatus.KILLED_AFTER_APP_COMPLETION:
- case ContainerExitStatus.ABORTED:
- // Neither the app's fault nor the system's fault. This happens by design,
- // so no need for skipping nodes
- return false;
- case ContainerExitStatus.DISKS_FAILED:
- // This container is marked with this exit-status means that the node is
- // already marked as unhealthy given that most of the disks failed. So, no
- // need for any explicit skipping of nodes.
- return false;
- case ContainerExitStatus.KILLED_EXCEEDED_VMEM:
- case ContainerExitStatus.KILLED_EXCEEDED_PMEM:
- // No point in skipping the node as it's not the system's fault
- return false;
- case ContainerExitStatus.SUCCESS:
- return false;
- case ContainerExitStatus.INVALID:
- // Ideally, this shouldn't be considered for skipping a node. But in
- // reality, it seems like there are cases where we are not setting
- // exit-code correctly and so it's better to be conservative. See
- // YARN-4284.
- return true;
- default:
- return true;
- }
- }
-
private static final class UnmanagedAMAttemptSavedTransition
extends AMLaunchedTransition {
@Override
@@ -1971,7 +1940,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
containerFinishedEvent.getContainerStatus();
if (containerStatus != null) {
int exitStatus = containerStatus.getExitStatus();
- if (shouldCountTowardsNodeBlacklisting(exitStatus)) {
+ if (Apps.shouldCountTowardsNodeBlacklisting(exitStatus)) {
appAttempt.addAMNodeToBlackList(nodeId);
}
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
deleted file mode 100644
index 15333e4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ /dev/null
@@ -1,1123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.server.appmaster.management.BoolMetric;
-import org.apache.slider.server.appmaster.management.Timestamp;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation;
-import org.apache.slider.server.avro.LoadedRoleHistory;
-import org.apache.slider.server.avro.NodeEntryRecord;
-import org.apache.slider.server.avro.RoleHistoryHeader;
-import org.apache.slider.server.avro.RoleHistoryWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * The Role History.
- * <p>
- * Synchronization policy: all public operations are synchronized.
- * Protected methods are in place for testing -no guarantees are made.
- * <p>
- * Inner classes have no synchronization guarantees; they should be manipulated
- * in these classes and not externally.
- * <p>
- * Note that as well as some methods marked visible for testing, there
- * is the option for the time generator method, {@link #now()} to
- * be overridden so that a repeatable time series can be used.
- *
- */
-public class RoleHistory {
- protected static final Logger log =
- LoggerFactory.getLogger(RoleHistory.class);
- private final List<ProviderRole> providerRoles;
- /** the roles in here are shared with App State */
- private final Map<Integer, RoleStatus> roleStatusMap = new HashMap<>();
- private final AbstractClusterServices recordFactory;
-
- private long startTime;
-
- /** Time when saved */
- private final Timestamp saveTime = new Timestamp(0);
-
- /** If the history was loaded, the time at which the history was saved.
- * That is: the time the data was valid */
- private final Timestamp thawedDataTime = new Timestamp(0);
-
- private NodeMap nodemap;
- private int roleSize;
- private final BoolMetric dirty = new BoolMetric(false);
- private FileSystem filesystem;
- private Path historyPath;
- private RoleHistoryWriter historyWriter = new RoleHistoryWriter();
-
- /**
- * When were the nodes updated in a {@link #onNodesUpdated(List)} call?
- * If zero: never.
- */
- private final Timestamp nodesUpdatedTime = new Timestamp(0);
- private final BoolMetric nodeUpdateReceived = new BoolMetric(false);
-
- private OutstandingRequestTracker outstandingRequests =
- new OutstandingRequestTracker();
-
- /**
- * For each role, lists nodes that are available for data-local allocation,
- * ordered by more recently released - to accelerate node selection.
- * That is, they are "recently used nodes"
- */
- private Map<Integer, LinkedList<NodeInstance>> recentNodes;
-
- /**
- * Instantiate
- * @param roles initial role list
- * @param recordFactory yarn record factory
- * @throws BadConfigException
- */
- public RoleHistory(Collection<RoleStatus> roles, AbstractClusterServices recordFactory) throws BadConfigException {
- this.recordFactory = recordFactory;
- roleSize = roles.size();
- providerRoles = new ArrayList<>(roleSize);
- for (RoleStatus role : roles) {
- addNewRole(role);
- }
- reset();
- }
-
- /**
- * Reset the variables -this does not adjust the fixed attributes
- * of the history, but the nodemap and failed node map are cleared.
- */
- protected synchronized void reset() throws BadConfigException {
-
- nodemap = new NodeMap(roleSize);
- resetAvailableNodeLists();
- outstandingRequests = new OutstandingRequestTracker();
- }
-
-
- /**
- * safety check: make sure the role is unique amongst
- * the role stats...which is extended with the new role
- * @param roleStatus role
- * @throws ArrayIndexOutOfBoundsException
- * @throws BadConfigException
- */
- protected void putRole(RoleStatus roleStatus) throws BadConfigException {
- int index = roleStatus.getKey();
- if (index < 0) {
- throw new BadConfigException("Provider " + roleStatus + " id is out of range");
- }
- if (roleStatusMap.get(index) != null) {
- throw new BadConfigException(
- roleStatus.toString() + " id duplicates that of " +
- roleStatusMap.get(index));
- }
- roleStatusMap.put(index, roleStatus);
- }
-
- /**
- * Add a new role
- * @param roleStatus new role
- */
- public void addNewRole(RoleStatus roleStatus) throws BadConfigException {
- log.debug("Validating/adding new role to role history: {} ", roleStatus);
- putRole(roleStatus);
- this.providerRoles.add(roleStatus.getProviderRole());
- }
-
- /**
- * Lookup a role by ID
- * @param roleId role Id
- * @return role or null if not found
- */
- public ProviderRole lookupRole(int roleId) {
- for (ProviderRole role : providerRoles) {
- if (role.id == roleId) {
- return role;
- }
- }
- return null;
- }
-
- /**
- * Clear the lists of available nodes
- */
- private synchronized void resetAvailableNodeLists() {
- recentNodes = new ConcurrentHashMap<>(roleSize);
- }
-
- /**
- * Prepare the history for re-reading its state.
- * <p>
- * This intended for use by the RoleWriter logic.
- * @throws BadConfigException if there is a problem rebuilding the state
- */
- private void prepareForReading(RoleHistoryHeader header)
- throws BadConfigException {
- reset();
-
- int roleCountInSource = header.getRoles();
- if (roleCountInSource != roleSize) {
- log.warn("Number of roles in source {}"
- +" does not match the expected number of {}",
- roleCountInSource,
- roleSize);
- }
- //record when the data was loaded
- setThawedDataTime(header.getSaved());
- }
-
- /**
- * rebuild the placement history from the loaded role history
- * @param loadedRoleHistory loaded history
- * @return the number of entries discarded
- * @throws BadConfigException if there is a problem rebuilding the state
- */
- @VisibleForTesting
- public synchronized int rebuild(LoadedRoleHistory loadedRoleHistory) throws BadConfigException {
- RoleHistoryHeader header = loadedRoleHistory.getHeader();
- prepareForReading(header);
- int discarded = 0;
- Long saved = header.getSaved();
- for (NodeEntryRecord nodeEntryRecord : loadedRoleHistory.records) {
- Integer roleId = nodeEntryRecord.getRole();
- NodeEntry nodeEntry = new NodeEntry(roleId);
- nodeEntry.setLastUsed(nodeEntryRecord.getLastUsed());
- if (nodeEntryRecord.getActive()) {
- //if active at the time of save, make the last used time the save time
- nodeEntry.setLastUsed(saved);
- }
- String hostname = SliderUtils.sequenceToString(nodeEntryRecord.getHost());
- ProviderRole providerRole = lookupRole(roleId);
- if (providerRole == null) {
- // discarding entry
- log.info("Discarding history entry with unknown role: {} on host {}",
- roleId, hostname);
- discarded ++;
- } else {
- NodeInstance instance = getOrCreateNodeInstance(hostname);
- instance.set(roleId, nodeEntry);
- }
- }
- return discarded;
- }
-
- public synchronized long getStartTime() {
- return startTime;
- }
-
- public synchronized long getSaveTime() {
- return saveTime.get();
- }
-
- public long getThawedDataTime() {
- return thawedDataTime.get();
- }
-
- public void setThawedDataTime(long thawedDataTime) {
- this.thawedDataTime.set(thawedDataTime);
- }
-
- public synchronized int getRoleSize() {
- return roleSize;
- }
-
- /**
- * Get the total size of the cluster -the number of NodeInstances
- * @return a count
- */
- public synchronized int getClusterSize() {
- return nodemap.size();
- }
-
- public synchronized boolean isDirty() {
- return dirty.get();
- }
-
- public synchronized void setDirty(boolean dirty) {
- this.dirty.set(dirty);
- }
-
- /**
- * Tell the history that it has been saved; marks itself as clean
- * @param timestamp timestamp -updates the savetime field
- */
- public synchronized void saved(long timestamp) {
- setDirty(false);
- saveTime.set(timestamp);
- }
-
- /**
- * Get a clone of the nodemap.
- * The instances inside are not cloned
- * @return the map
- */
- public synchronized NodeMap cloneNodemap() {
- return (NodeMap) nodemap.clone();
- }
-
- /**
- * Get snapshot of the node map
- * @return a snapshot of the current node state
- * @param naming naming map of priority to enty name; entries must be unique.
- * It's OK to be incomplete, for those the list falls back to numbers.
- */
- public synchronized Map<String, NodeInformation> getNodeInformationSnapshot(
- Map<Integer, String> naming) {
- Map<String, NodeInformation> result = new HashMap<>(nodemap.size());
- for (Map.Entry<String, NodeInstance> entry : nodemap.entrySet()) {
- result.put(entry.getKey(), entry.getValue().serialize(naming));
- }
- return result;
- }
-
- /**
- * Get the information on a node
- * @param hostname hostname
- * @param naming naming map of priority to enty name; entries must be unique.
- * It's OK to be incomplete, for those the list falls back to numbers.
- * @return the information about that host, or null if there is none
- */
- public synchronized NodeInformation getNodeInformation(String hostname,
- Map<Integer, String> naming) {
- NodeInstance nodeInstance = nodemap.get(hostname);
- return nodeInstance != null ? nodeInstance.serialize(naming) : null;
- }
-
- /**
- * Get the node instance for the specific node -creating it if needed
- * @param hostname node address
- * @return the instance
- */
- public synchronized NodeInstance getOrCreateNodeInstance(String hostname) {
- //convert to a string
- return nodemap.getOrCreate(hostname);
- }
-
- /**
- * Insert a list of nodes into the map; overwrite any with that name.
- * This is a bulk operation for testing.
- * Important: this does not update the available node lists, these
- * must be rebuilt afterwards.
- * @param nodes collection of nodes.
- */
- @VisibleForTesting
- public synchronized void insert(Collection<NodeInstance> nodes) {
- nodemap.insert(nodes);
- }
-
- /**
- * Get current time. overrideable for test subclasses
- * @return current time in millis
- */
- protected long now() {
- return System.currentTimeMillis();
- }
-
- /**
- * Mark ourselves as dirty
- */
- public void touch() {
- setDirty(true);
- try {
- saveHistoryIfDirty();
- } catch (IOException e) {
- log.warn("Failed to save history file ", e);
- }
- }
-
- /**
- * reset the failed recently counters
- */
- public synchronized void resetFailedRecently() {
- log.info("Resetting failure history");
- nodemap.resetFailedRecently();
- }
-
- /**
- * Get the path used for history files
- * @return the directory used for history files
- */
- public Path getHistoryPath() {
- return historyPath;
- }
-
- /**
- * Save the history to its location using the timestamp as part of
- * the filename. The saveTime and dirty fields are updated
- * @param time timestamp timestamp to use as the save time
- * @return the path saved to
- * @throws IOException IO problems
- */
- @VisibleForTesting
- public synchronized Path saveHistory(long time) throws IOException {
- Path filename = historyWriter.createHistoryFilename(historyPath, time);
- historyWriter.write(filesystem, filename, true, this, time);
- saved(time);
- return filename;
- }
-
- /**
- * Save the history with the current timestamp if it is dirty;
- * return the path saved to if this is the case
- * @return the path or null if the history was not saved
- * @throws IOException failed to save for some reason
- */
- public synchronized Path saveHistoryIfDirty() throws IOException {
- if (isDirty()) {
- return saveHistory(now());
- } else {
- return null;
- }
- }
-
- /**
- * Start up
- * @param fs filesystem
- * @param historyDir path in FS for history
- * @return true if the history was thawed
- */
- public boolean onStart(FileSystem fs, Path historyDir) throws BadConfigException {
- assert filesystem == null;
- filesystem = fs;
- historyPath = historyDir;
- startTime = now();
- //assume the history is being thawed; this will downgrade as appropriate
- return onThaw();
- }
-
- /**
- * Handler for bootstrap event: there was no history to thaw
- */
- public void onBootstrap() {
- log.debug("Role history bootstrapped");
- }
-
- /**
- * Handle the start process <i>after the history has been rebuilt</i>,
- * and after any gc/purge
- */
- public synchronized boolean onThaw() throws BadConfigException {
- assert filesystem != null;
- assert historyPath != null;
- boolean thawSuccessful = false;
- //load in files from data dir
-
- LoadedRoleHistory loadedRoleHistory = null;
- try {
- loadedRoleHistory = historyWriter.loadFromHistoryDir(filesystem, historyPath);
- } catch (IOException e) {
- log.warn("Exception trying to load history from {}", historyPath, e);
- }
- if (loadedRoleHistory != null) {
- rebuild(loadedRoleHistory);
- thawSuccessful = true;
- Path loadPath = loadedRoleHistory.getPath();
- log.debug("loaded history from {}", loadPath);
- // delete any old entries
- try {
- int count = historyWriter.purgeOlderHistoryEntries(filesystem, loadPath);
- log.debug("Deleted {} old history entries", count);
- } catch (IOException e) {
- log.info("Ignoring exception raised while trying to delete old entries",
- e);
- }
-
- //start is then completed
- buildRecentNodeLists();
- } else {
- //fallback to bootstrap procedure
- onBootstrap();
- }
- return thawSuccessful;
- }
-
-
- /**
- * (After the start), rebuild the availability data structures
- */
- @VisibleForTesting
- public synchronized void buildRecentNodeLists() {
- resetAvailableNodeLists();
- // build the list of available nodes
- for (Map.Entry<String, NodeInstance> entry : nodemap.entrySet()) {
- NodeInstance ni = entry.getValue();
- for (int i = 0; i < roleSize; i++) {
- NodeEntry nodeEntry = ni.get(i);
- if (nodeEntry != null && nodeEntry.isAvailable()) {
- log.debug("Adding {} for role {}", ni, i);
- listRecentNodesForRoleId(i).add(ni);
- }
- }
- }
- // sort the resulting arrays
- for (int i = 0; i < roleSize; i++) {
- sortRecentNodeList(i);
- }
- }
-
- /**
- * Get the nodes for an ID -may be null
- * @param id role ID
- * @return potentially null list
- */
- @VisibleForTesting
- public List<NodeInstance> getRecentNodesForRoleId(int id) {
- return recentNodes.get(id);
- }
-
- /**
- * Get a possibly empty list of suggested nodes for a role.
- * @param id role ID
- * @return list
- */
- private LinkedList<NodeInstance> listRecentNodesForRoleId(int id) {
- LinkedList<NodeInstance> instances = recentNodes.get(id);
- if (instances == null) {
- synchronized (this) {
- // recheck in the synchronized block and recreate
- if (recentNodes.get(id) == null) {
- recentNodes.put(id, new LinkedList<NodeInstance>());
- }
- instances = recentNodes.get(id);
- }
- }
- return instances;
- }
-
- /**
- * Sort a the recent node list for a single role
- * @param role role to sort
- */
- private void sortRecentNodeList(int role) {
- List<NodeInstance> nodesForRoleId = getRecentNodesForRoleId(role);
- if (nodesForRoleId != null) {
- Collections.sort(nodesForRoleId, new NodeInstance.Preferred(role));
- }
- }
-
- public synchronized UpdateBlacklistOperation updateBlacklist(
- Collection<RoleStatus> roleStatuses) {
- List<String> blacklistAdditions = new ArrayList<>();
- List<String> blacklistRemovals = new ArrayList<>();
- for (Entry<String, NodeInstance> nodeInstanceEntry : nodemap.entrySet()) {
- boolean shouldBeBlacklisted = false;
- String nodeHost = nodeInstanceEntry.getKey();
- NodeInstance nodeInstance = nodeInstanceEntry.getValue();
- for (RoleStatus roleStatus : roleStatuses) {
- if (nodeInstance.exceedsFailureThreshold(roleStatus)) {
- shouldBeBlacklisted = true;
- break;
- }
- }
- if (shouldBeBlacklisted) {
- if (!nodeInstance.isBlacklisted()) {
- blacklistAdditions.add(nodeHost);
- nodeInstance.setBlacklisted(true);
- }
- } else {
- if (nodeInstance.isBlacklisted()) {
- blacklistRemovals.add(nodeHost);
- nodeInstance.setBlacklisted(false);
- }
- }
- }
- if (blacklistAdditions.isEmpty() && blacklistRemovals.isEmpty()) {
- return null;
- }
- return new UpdateBlacklistOperation(blacklistAdditions, blacklistRemovals);
- }
-
- /**
- * Find a node for use
- * @param role role
- * @return the instance, or null for none
- */
- @VisibleForTesting
- public synchronized NodeInstance findRecentNodeForNewInstance(RoleStatus role) {
- if (!role.isPlacementDesired()) {
- // no data locality policy
- return null;
- }
- int roleId = role.getKey();
- boolean strictPlacement = role.isStrictPlacement();
- NodeInstance nodeInstance = null;
- // Get the list of possible targets.
- // This is a live list: changes here are preserved
- List<NodeInstance> targets = getRecentNodesForRoleId(roleId);
- if (targets == null) {
- // nothing to allocate on
- return null;
- }
-
- int cnt = targets.size();
- log.debug("There are {} node(s) to consider for {}", cnt, role.getName());
- for (int i = 0; i < cnt && nodeInstance == null; i++) {
- NodeInstance candidate = targets.get(i);
- if (candidate.getActiveRoleInstances(roleId) == 0) {
- // no active instances: check failure statistics
- if (strictPlacement
- || (candidate.isOnline() && !candidate.exceedsFailureThreshold(role))) {
- targets.remove(i);
- // exit criteria for loop is now met
- nodeInstance = candidate;
- } else {
- // too many failures for this node
- log.info("Recent node failures is higher than threshold {}. Not requesting host {}",
- role.getNodeFailureThreshold(), candidate.hostname);
- }
- }
- }
-
- if (nodeInstance == null) {
- log.info("No node found for {}", role.getName());
- }
- return nodeInstance;
- }
-
- /**
- * Find a node for use
- * @param role role
- * @return the instance, or null for none
- */
- @VisibleForTesting
- public synchronized List<NodeInstance> findNodeForNewAAInstance(RoleStatus role) {
- // all nodes that are live and can host the role; no attempt to exclude ones
- // considered failing
- return nodemap.findAllNodesForRole(role.getKey(), role.getLabelExpression());
- }
-
- /**
- * Request an instance on a given node.
- * An outstanding request is created & tracked, with the
- * relevant node entry for that role updated.
- *<p>
- * The role status entries will also be tracked
- * <p>
- * Returns the request that is now being tracked.
- * If the node instance is not null, it's details about the role is incremented
- *
- * @param node node to target or null for "any"
- * @param role role to request
- * @return the request
- */
- public synchronized OutstandingRequest requestInstanceOnNode(
- NodeInstance node, RoleStatus role, Resource resource) {
- OutstandingRequest outstanding = outstandingRequests.newRequest(node, role.getKey());
- outstanding.buildContainerRequest(resource, role, now());
- return outstanding;
- }
-
- /**
- * Find a node for a role and request an instance on that (or a location-less
- * instance)
- * @param role role status
- * @return a request ready to go, or null if this is an AA request and no
- * location can be found.
- */
- public synchronized OutstandingRequest requestContainerForRole(RoleStatus role) {
-
- if (role.isAntiAffinePlacement()) {
- return requestContainerForAARole(role);
- } else {
- Resource resource = recordFactory.newResource();
- role.copyResourceRequirements(resource);
- NodeInstance node = findRecentNodeForNewInstance(role);
- return requestInstanceOnNode(node, role, resource);
- }
- }
-
- /**
- * Find a node for an AA role and request an instance on that (or a location-less
- * instance)
- * @param role role status
- * @return a request ready to go, or null if no location can be found.
- */
- public synchronized OutstandingRequest requestContainerForAARole(RoleStatus role) {
- List<NodeInstance> nodes = findNodeForNewAAInstance(role);
- if (!nodes.isEmpty()) {
- OutstandingRequest outstanding = outstandingRequests.newAARequest(
- role.getKey(), nodes, role.getLabelExpression());
- Resource resource = recordFactory.newResource();
- role.copyResourceRequirements(resource);
- outstanding.buildContainerRequest(resource, role, now());
- return outstanding;
- } else {
- log.warn("No suitable location for {}", role.getName());
- return null;
- }
- }
- /**
- * Get the list of active nodes ... walks the node map so
- * is {@code O(nodes)}
- * @param role role index
- * @return a possibly empty list of nodes with an instance of that node
- */
- public synchronized List<NodeInstance> listActiveNodes(int role) {
- return nodemap.listActiveNodes(role);
- }
-
- /**
- * Get the node entry of a container
- * @param container container to look up
- * @return the entry
- * @throws RuntimeException if the container has no hostname
- */
- public NodeEntry getOrCreateNodeEntry(Container container) {
- return getOrCreateNodeInstance(container).getOrCreate(container);
- }
-
- /**
- * Get the node instance of a container -always returns something
- * @param container container to look up
- * @return a (possibly new) node instance
- * @throws RuntimeException if the container has no hostname
- */
- public synchronized NodeInstance getOrCreateNodeInstance(Container container) {
- return nodemap.getOrCreate(RoleHistoryUtils.hostnameOf(container));
- }
-
- /**
- * Get the node instance of a host if defined
- * @param hostname hostname to look up
- * @return a node instance or null
- * @throws RuntimeException if the container has no hostname
- */
- public synchronized NodeInstance getExistingNodeInstance(String hostname) {
- return nodemap.get(hostname);
- }
-
- /**
- * Get the node instance of a container <i>if there's an entry in the history</i>
- * @param container container to look up
- * @return a node instance or null
- * @throws RuntimeException if the container has no hostname
- */
- public synchronized NodeInstance getExistingNodeInstance(Container container) {
- return nodemap.get(RoleHistoryUtils.hostnameOf(container));
- }
-
- /**
- * Perform any pre-allocation operations on the list of allocated containers
- * based on knowledge of system state.
- * Currently this places requested hosts ahead of unrequested ones.
- * @param allocatedContainers list of allocated containers
- * @return list of containers potentially reordered
- */
- public synchronized List<Container> prepareAllocationList(List<Container> allocatedContainers) {
-
- //partition into requested and unrequested
- List<Container> requested =
- new ArrayList<>(allocatedContainers.size());
- List<Container> unrequested =
- new ArrayList<>(allocatedContainers.size());
- outstandingRequests.partitionRequests(this, allocatedContainers, requested, unrequested);
-
- //give the unrequested ones lower priority
- requested.addAll(unrequested);
- return requested;
- }
-
- /**
- * A container has been allocated on a node -update the data structures
- * @param container container
- * @param desiredCount desired #of instances
- * @param actualCount current count of instances
- * @return The allocation outcome
- */
- public synchronized ContainerAllocationResults onContainerAllocated(Container container,
- long desiredCount,
- long actualCount) {
- int role = ContainerPriority.extractRole(container);
-
- String hostname = RoleHistoryUtils.hostnameOf(container);
- List<NodeInstance> nodeInstances = listRecentNodesForRoleId(role);
- ContainerAllocationResults outcome =
- outstandingRequests.onContainerAllocated(role, hostname, container);
- if (desiredCount <= actualCount) {
- // all outstanding requests have been satisfied
- // clear all the lists, so returning nodes to the available set
- List<NodeInstance> hosts = outstandingRequests.resetOutstandingRequests(role);
- if (!hosts.isEmpty()) {
- //add the list
- log.info("Adding {} hosts for role {}", hosts.size(), role);
- nodeInstances.addAll(hosts);
- sortRecentNodeList(role);
- }
- }
- return outcome;
- }
-
- /**
- * A container has been assigned to a role instance on a node -update the data structures
- * @param container container
- */
- public void onContainerAssigned(Container container) {
- NodeInstance node = getOrCreateNodeInstance(container);
- NodeEntry nodeEntry = node.getOrCreate(container);
- nodeEntry.onStarting();
- log.debug("Node {} has updated NodeEntry {}", node, nodeEntry);
- }
-
- /**
- * Event: a container start has been submitted
- * @param container container being started
- * @param instance instance bound to the container
- */
- public void onContainerStartSubmitted(Container container,
- RoleInstance instance) {
- // no actions here
- }
-
- /**
- * Container start event
- * @param container container that just started
- */
- public void onContainerStarted(Container container) {
- NodeEntry nodeEntry = getOrCreateNodeEntry(container);
- nodeEntry.onStartCompleted();
- touch();
- }
-
- /**
- * A container failed to start: update the node entry state
- * and return the container to the queue
- * @param container container that failed
- * @return true if the node was queued
- */
- public boolean onNodeManagerContainerStartFailed(Container container) {
- return markContainerFinished(container, false, true, ContainerOutcome.Failed);
- }
-
- /**
- * Does the RoleHistory have enough information about the YARN cluster
- * to start placing AA requests? That is: has it the node map and
- * any label information needed?
- * @return true if the caller can start requesting AA nodes
- */
- public boolean canPlaceAANodes() {
- return nodeUpdateReceived.get();
- }
-
- /**
- * Get the last time the nodes were updated from YARN
- * @return the update time or zero if never updated.
- */
- public long getNodesUpdatedTime() {
- return nodesUpdatedTime.get();
- }
-
- /**
- * Update failedNodes and nodemap based on the node state
- *
- * @param updatedNodes list of updated nodes
- * @return true if a review should be triggered.
- */
- public synchronized boolean onNodesUpdated(List<NodeReport> updatedNodes) {
- log.debug("Updating {} nodes", updatedNodes.size());
- nodesUpdatedTime.set(now());
- nodeUpdateReceived.set(true);
- int printed = 0;
- boolean triggerReview = false;
- for (NodeReport updatedNode : updatedNodes) {
- String hostname = updatedNode.getNodeId() == null
- ? ""
- : updatedNode.getNodeId().getHost();
- NodeState nodeState = updatedNode.getNodeState();
- if (hostname.isEmpty() || nodeState == null) {
- log.warn("Ignoring incomplete update");
- continue;
- }
- if (log.isDebugEnabled() && printed++ < 10) {
- // log the first few, but avoid overloading the logs for a full cluster
- // update
- log.debug("Node \"{}\" is in state {}", hostname, nodeState);
- }
- // update the node; this also creates an instance if needed
- boolean updated = nodemap.updateNode(hostname, updatedNode);
- triggerReview |= updated;
- }
- return triggerReview;
- }
-
- /**
- * A container release request was issued
- * @param container container submitted
- */
- public void onContainerReleaseSubmitted(Container container) {
- NodeEntry nodeEntry = getOrCreateNodeEntry(container);
- nodeEntry.release();
- }
-
- /**
- * App state notified of a container completed
- * @param container completed container
- * @return true if the node was queued
- */
- public boolean onReleaseCompleted(Container container) {
- return markContainerFinished(container, true, false, ContainerOutcome.Failed);
- }
-
- /**
- * App state notified of a container completed -but as
- * it wasn't being released it is marked as failed
- *
- * @param container completed container
- * @param shortLived was the container short lived?
- * @param outcome
- * @return true if the node is considered available for work
- */
- public boolean onFailedContainer(Container container,
- boolean shortLived,
- ContainerOutcome outcome) {
- return markContainerFinished(container, false, shortLived, outcome);
- }
-
- /**
- * Mark a container finished; if it was released then that is treated
- * differently. history is {@code touch()}-ed
- *
- *
- * @param container completed container
- * @param wasReleased was the container released?
- * @param shortLived was the container short lived?
- * @param outcome
- * @return true if the node was queued
- */
- protected synchronized boolean markContainerFinished(Container container,
- boolean wasReleased,
- boolean shortLived,
- ContainerOutcome outcome) {
- NodeEntry nodeEntry = getOrCreateNodeEntry(container);
- log.info("Finished container for node {}, released={}, shortlived={}",
- nodeEntry.rolePriority, wasReleased, shortLived);
- boolean available;
- if (shortLived) {
- nodeEntry.onStartFailed();
- available = false;
- } else {
- available = nodeEntry.containerCompleted(wasReleased, outcome);
- maybeQueueNodeForWork(container, nodeEntry, available);
- }
- touch();
- return available;
- }
-
- /**
- * If the node is marked as available; queue it for assignments.
- * Unsynced: requires caller to be in a sync block.
- * @param container completed container
- * @param nodeEntry node
- * @param available available flag
- * @return true if the node was queued
- */
- private boolean maybeQueueNodeForWork(Container container,
- NodeEntry nodeEntry,
- boolean available) {
- if (available) {
- //node is free
- nodeEntry.setLastUsed(now());
- NodeInstance ni = getOrCreateNodeInstance(container);
- int roleId = ContainerPriority.extractRole(container);
- log.debug("Node {} is now available for role id {}", ni, roleId);
- listRecentNodesForRoleId(roleId).addFirst(ni);
- }
- return available;
- }
-
- /**
- * Print the history to the log. This is for testing and diagnostics
- */
- public synchronized void dump() {
- for (ProviderRole role : providerRoles) {
- log.info(role.toString());
- List<NodeInstance> instances = listRecentNodesForRoleId(role.id);
- log.info(" available: " + instances.size()
- + " " + SliderUtils.joinWithInnerSeparator(" ", instances));
- }
-
- log.info("Nodes in Cluster: {}", getClusterSize());
- for (NodeInstance node : nodemap.values()) {
- log.info(node.toFullString());
- }
- }
-
- /**
- * Build the mapping entry for persisting to the role history
- * @return a mapping object
- */
- public synchronized Map<CharSequence, Integer> buildMappingForHistoryFile() {
- Map<CharSequence, Integer> mapping = new HashMap<>(getRoleSize());
- for (ProviderRole role : providerRoles) {
- mapping.put(role.name, role.id);
- }
- return mapping;
- }
-
- /**
- * Get a clone of the available list
- * @param role role index
- * @return a clone of the list
- */
- @VisibleForTesting
- public List<NodeInstance> cloneRecentNodeList(int role) {
- return new LinkedList<>(listRecentNodesForRoleId(role));
- }
-
- /**
- * Get a snapshot of the outstanding placed request list
- * @return a list of the requests outstanding at the time of requesting
- */
- @VisibleForTesting
- public List<OutstandingRequest> listPlacedRequests() {
- return outstandingRequests.listPlacedRequests();
- }
-
- /**
- * Get a snapshot of the outstanding placed request list
- * @return a list of the requests outstanding at the time of requesting
- */
- @VisibleForTesting
- public List<OutstandingRequest> listOpenRequests() {
- return outstandingRequests.listOpenRequests();
- }
-
- /**
- * Escalate operation as triggered by external timer.
- * @return a (usually empty) list of cancel/request operations.
- */
- public synchronized List<AbstractRMOperation> escalateOutstandingRequests() {
- return outstandingRequests.escalateOutstandingRequests(now());
- }
- /**
- * Escalate operation as triggered by external timer.
- * @return a (usually empty) list of cancel/request operations.
- */
- public List<AbstractRMOperation> cancelOutstandingAARequests() {
- return outstandingRequests.cancelOutstandingAARequests();
- }
-
- /**
- * Cancel a number of outstanding requests for a role -that is, not
- * actual containers, just requests for new ones.
- * @param role role
- * @param toCancel number to cancel
- * @return a list of cancellable operations.
- */
- public List<AbstractRMOperation> cancelRequestsForRole(RoleStatus role, int toCancel) {
- return role.isAntiAffinePlacement() ?
- cancelRequestsForAARole(role, toCancel)
- : cancelRequestsForSimpleRole(role, toCancel);
- }
-
- /**
- * Build the list of requests to cancel from the outstanding list.
- * @param role role
- * @param toCancel number to cancel
- * @return a list of cancellable operations.
- */
- private synchronized List<AbstractRMOperation> cancelRequestsForSimpleRole(RoleStatus role, int toCancel) {
- Preconditions.checkArgument(toCancel > 0,
- "trying to cancel invalid number of requests: " + toCancel);
- List<AbstractRMOperation> results = new ArrayList<>(toCancel);
- // first scan through the unplaced request list to find all of a role
- int roleId = role.getKey();
- List<OutstandingRequest> requests =
- outstandingRequests.extractOpenRequestsForRole(roleId, toCancel);
-
- // are there any left?
- int remaining = toCancel - requests.size();
- // ask for some placed nodes
- requests.addAll(outstandingRequests.extractPlacedRequestsForRole(roleId, remaining));
-
- // build cancellations
- for (OutstandingRequest request : requests) {
- results.add(request.createCancelOperation());
- }
- return results;
- }
-
- /**
- * Build the list of requests to cancel for an AA role. This reduces the number
- * of outstanding pending requests first, then cancels any active request,
- * before finally asking for any placed containers
- * @param role role
- * @param toCancel number to cancel
- * @return a list of cancellable operations.
- */
- private synchronized List<AbstractRMOperation> cancelRequestsForAARole(RoleStatus role, int toCancel) {
- List<AbstractRMOperation> results = new ArrayList<>(toCancel);
- int roleId = role.getKey();
- List<OutstandingRequest> requests = new ArrayList<>(toCancel);
- // there may be pending requests which can be cancelled here
- long pending = role.getAAPending();
- if (pending > 0) {
- // there are some pending ones which can be cancelled first
- long pendingToCancel = Math.min(pending, toCancel);
- log.info("Cancelling {} pending AA allocations, leaving {}", toCancel,
- pendingToCancel);
- role.setAAPending(pending - pendingToCancel);
- toCancel -= pendingToCancel;
- }
- if (toCancel > 0 && role.isAARequestOutstanding()) {
- // not enough
- log.info("Cancelling current AA request");
- // find the single entry which may be running
- requests = outstandingRequests.extractOpenRequestsForRole(roleId, toCancel);
- role.cancelOutstandingAARequest();
- toCancel--;
- }
-
- // ask for some excess nodes
- if (toCancel > 0) {
- requests.addAll(outstandingRequests.extractPlacedRequestsForRole(roleId, toCancel));
- }
-
- // build cancellations
- for (OutstandingRequest request : requests) {
- results.add(request.createCancelOperation());
- }
- return results;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistoryUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistoryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistoryUtils.java
deleted file mode 100644
index ea6197b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistoryUtils.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.slider.common.tools.SliderUtils;
-
-public class RoleHistoryUtils {
-
- public static String hostnameOf(Container container) {
- NodeId nodeId = container.getNodeId();
- if (nodeId== null) {
- throw new RuntimeException("Container has no node ID: %s" +
- SliderUtils.containerToString(container));
- }
- return nodeId.getHost();
- }
-
- /**
- * Decrement a value but hold it at zero. Usually a sanity check
- * on counters tracking outstanding operations
- * @param val value
- * @return decremented value
- */
- public static int decToFloor(int val) {
- int v = val-1;
- if (v < 0) {
- v = 0;
- }
- return v;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHostnamePair.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHostnamePair.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHostnamePair.java
deleted file mode 100644
index 920887a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHostnamePair.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import java.util.Objects;
-
-public class RoleHostnamePair {
-
- /**
- * requested role
- */
- public final int roleId;
-
- /**
- * hostname -will be null if node==null
- */
- public final String hostname;
-
- public RoleHostnamePair(int roleId, String hostname) {
- this.roleId = roleId;
- this.hostname = hostname;
- }
-
- public int getRoleId() {
- return roleId;
- }
-
- public String getHostname() {
- return hostname;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (!(o instanceof RoleHostnamePair)) {
- return false;
- }
- RoleHostnamePair that = (RoleHostnamePair) o;
- return Objects.equals(roleId, that.roleId) &&
- Objects.equals(hostname, that.hostname);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(roleId, hostname);
- }
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder(
- "RoleHostnamePair{");
- sb.append("roleId=").append(roleId);
- sb.append(", hostname='").append(hostname).append('\'');
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
deleted file mode 100644
index 070f2f8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ /dev/null
@@ -1,343 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ProtocolTypes;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.proto.Messages;
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.providers.ProviderRole;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Tracking information about a container
- */
-public final class RoleInstance implements Cloneable {
-
- public Container container;
- public ProviderRole providerRole;
- public long componentId = -1;
- public String compInstanceName = null;
- /**
- * Container ID
- */
- public final String id;
- public long createTime;
- public long startTime;
- /**
- * flag set when it is released, to know if it has
- * already been targeted for termination
- */
- public boolean released;
-
- /**
- * Name of the role
- */
- public String role;
-
- /**
- * Version of the app
- */
- public String appVersion;
-
- /**
- * Role Id; matches priority in resources.json
- */
- public int roleId;
-
- /**
- * state from StateValues
- */
- public int state;
-
- /**
- * Exit code: only valid if the state >= STOPPED
- */
- public int exitCode;
-
- /**
- * what was the command executed?
- */
- public String command;
-
- /**
- * Any diagnostics
- */
- public String diagnostics;
-
- /**
- * What is the tail output from the executed process (or [] if not started
- * or the log cannot be picked up
- */
- public String[] output;
-
- /**
- * Any environment details
- */
- public String[] environment;
-
- public String ip;
- public String hostname;
- public String host;
- public String hostURL;
- public ContainerAllocationOutcome placement;
- public Path compInstanceDir;
-
- /**
- * A list of registered endpoints.
- */
- private List<Endpoint> endpoints =
- new ArrayList<>(2);
-
- public RoleInstance(Container container, ProviderRole role) {
- this(container);
- if (role.componentIdCounter != null) {
- componentId = role.componentIdCounter.getAndIncrement();
- compInstanceName = role.name + componentId;
- } else {
- compInstanceName = role.name;
- }
- compInstanceName = compInstanceName.toLowerCase().replaceAll("_", "-");
- this.providerRole = role;
- }
-
- public RoleInstance(Container container, RoleInstance failedInstance) {
- this(container);
- this.componentId = failedInstance.componentId;
- this.compInstanceName = failedInstance.compInstanceName;
- this.providerRole = failedInstance.providerRole;
- }
-
- /**
- * Create an instance to track an allocated container
- * @param container a container which must be non null, and have a non-null Id field.
- */
- public RoleInstance(Container container) {
- Preconditions.checkNotNull(container, "Null container");
- Preconditions.checkState(container.getId() != null,
- "Null container ID");
-
- this.container = container;
- id = container.getId().toString();
- if (container.getNodeId() != null) {
- host = container.getNodeId().getHost();
- }
- if (container.getNodeHttpAddress() != null) {
- hostURL = "http://" + container.getNodeHttpAddress();
- }
- }
-
- public NodeId getHost() {
- return container.getNodeId();
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("RoleInstance{");
- sb.append("role='").append(role).append('\'');
- sb.append(", id='").append(id).append('\'');
- sb.append(", instanceName='").append(compInstanceName).append('\'');
- sb.append(", container=").append(SliderUtils.containerToString(container));
- sb.append(", createTime=").append(createTime);
- sb.append(", startTime=").append(startTime);
- sb.append(", released=").append(released);
- sb.append(", roleId=").append(roleId);
- sb.append(", host=").append(host);
- sb.append(", hostURL=").append(hostURL);
- sb.append(", state=").append(state);
- sb.append(", placement=").append(placement);
- sb.append(", exitCode=").append(exitCode);
- sb.append(", command='").append(command).append('\'');
- sb.append(", diagnostics='").append(diagnostics).append('\'');
- sb.append(", output=").append(Arrays.toString(output));
- sb.append(", environment=").append(Arrays.toString(environment));
- sb.append('}');
- return sb.toString();
- }
-
- public ContainerId getContainerId() {
- return container.getId();
- }
-
- /**
- * Generate the protobuf format of a request
- * @return protobuf format. This excludes the Container info
- */
- public Messages.RoleInstanceState toProtobuf() {
- Messages.RoleInstanceState.Builder builder =
- Messages.RoleInstanceState.newBuilder();
- if (container != null) {
- builder.setName(container.getId().toString());
- } else {
- builder.setName("unallocated instance");
- }
- if (command != null) {
- builder.setCommand(command);
- }
- if (environment != null) {
- builder.addAllEnvironment(Arrays.asList(environment));
- }
- if (diagnostics != null) {
- builder.setDiagnostics(diagnostics);
- }
- builder.setExitCode(exitCode);
-
- if (output != null) {
- builder.addAllOutput(Arrays.asList(output));
- }
- if (role != null) {
- builder.setRole(role);
- }
- builder.setRoleId(roleId);
- builder.setState(state);
-
- builder.setReleased(released);
- builder.setCreateTime(createTime);
- builder.setStartTime(startTime);
- builder.setHost(host);
- builder.setHostURL(hostURL);
- if (appVersion != null) {
- builder.setAppVersion(appVersion);
- }
- return builder.build();
- }
-
- /**
- * Build a serializable ClusterNode structure from this instance.
- * This operation is unsynchronized.
- * @return a serialized value.
- */
- public ClusterNode toClusterNode() {
- ClusterNode node;
- if (container != null) {
- node = new ClusterNode(container.getId());
- } else {
- node = new ClusterNode();
- node.name = "unallocated instance";
- }
- node.command = command;
- node.createTime = createTime;
- node.diagnostics = diagnostics;
- if (environment != null) {
- node.environment = Arrays.copyOf(environment, environment.length);
- }
- node.exitCode = exitCode;
- node.ip = ip;
- node.hostname = hostname;
- node.host = host;
- node.hostUrl = hostURL;
- if (output != null) {
- node.output = Arrays.copyOf(output, output.length);
- }
- node.released = released;
- node.role = role;
- node.roleId = roleId;
- node.startTime = startTime ;
- node.state = state;
-
- return node;
- }
-
- /**
- * Clone operation clones all the simple values but shares the
- * Container object into the cloned copy -same with the output,
- * diagnostics and env arrays.
- * @return a clone of the object
- * @throws CloneNotSupportedException
- */
- @Override
- public Object clone() throws CloneNotSupportedException {
- RoleInstance cloned = (RoleInstance) super.clone();
- // clone the endpoint list, but not the values
- cloned.endpoints = new ArrayList<Endpoint>(this.endpoints);
- return cloned;
- }
-
- /**
- * Get the list of endpoints.
- * @return the endpoint list.
- */
- public List<Endpoint> getEndpoints() {
- return endpoints;
- }
-
- /**
- * Add an endpoint registration
- * @param endpoint endpoint (non-null)
- */
- public void addEndpoint(Endpoint endpoint) {
- Preconditions.checkArgument(endpoint != null);
- endpoints.add(endpoint);
- }
-
- /**
- * Register a port endpoint as an inet-addr formatted endpoint, using the
- * hostname as the first part of the address
- * @param port port port
- * @param api API API name
- */
- public void registerPortEndpoint(int port, String api) {
- Endpoint epr =
- RegistryTypeUtils.inetAddrEndpoint(api,
- ProtocolTypes.PROTOCOL_TCP, host, port);
- addEndpoint(epr);
- }
-
- /**
- * Serialize. Some data structures (e.g output)
- * may be shared
- * @return a serialized form for marshalling as JSON
- */
- public ContainerInformation serialize() {
- ContainerInformation info = new ContainerInformation();
- info.containerId = id;
- info.component = role;
- info.appVersion = appVersion;
- info.startTime = startTime;
- info.createTime = createTime;
- info.diagnostics = diagnostics;
- info.state = state;
- info.host = host;
- info.hostURL = hostURL;
- info.released = released ? Boolean.TRUE : null;
- if (placement != null) {
- info.placement = placement.toString();
- }
- if (output != null) {
- info.output = output;
- }
- return info;
- }
-
- public String getCompInstanceName() {
- return compInstanceName;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
deleted file mode 100644
index ec0ff25..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ /dev/null
@@ -1,367 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.codahale.metrics.Metric;
-import com.codahale.metrics.MetricSet;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.slider.api.types.ComponentInformation;
-import org.apache.slider.api.types.RoleStatistics;
-import org.apache.slider.providers.PlacementPolicy;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.server.appmaster.management.BoolMetricPredicate;
-import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
-
-import java.io.Serializable;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Models the ongoing status of all nodes in an application.
- *
- * These structures are shared across the {@link AppState} and {@link RoleHistory} structures,
- * and must be designed for synchronous access. Atomic counters are preferred to anything which
- * requires synchronization. Where synchronized access is good is that it allows for
- * the whole instance to be locked, for updating multiple entries.
- */
-public final class RoleStatus implements MetricSet {
-
- private final String name;
-
- /**
- * Role priority
- */
- private final int key;
- private final ProviderRole providerRole;
-
- /** resource requirements */
- private Resource resourceRequirements;
- private ServiceMetrics componentMetrics;
-
- /** any pending AA request */
- private volatile OutstandingRequest outstandingAArequest = null;
-
-
- private String failureMessage = "";
-
- public RoleStatus(ProviderRole providerRole) {
- this.providerRole = providerRole;
- this.name = providerRole.name;
- this.key = providerRole.id;
- componentMetrics =
- ServiceMetrics.register(this.name, "Metrics for component " + this.name);
- componentMetrics
- .tag("type", "Metrics type [component or service]", "component");
- }
-
- public ServiceMetrics getComponentMetrics() {
- return this.componentMetrics;
- }
-
- @Override
- public Map<String, Metric> getMetrics() {
- Map<String, Metric> metrics = new HashMap<>(15);
- metrics.put("outstandingAArequest",
- new BoolMetricPredicate(new BoolMetricPredicate.Eval() {
- @Override
- public boolean eval() {
- return isAARequestOutstanding();
- }
- }));
- return metrics;
- }
-
- public String getName() {
- return name;
- }
-
- public int getKey() {
- return key;
- }
-
- public int getPriority() {
- return getKey();
- }
-
- /**
- * Get the placement policy enum, from the values in
- * {@link PlacementPolicy}
- * @return the placement policy for this role
- */
- public int getPlacementPolicy() {
- return providerRole.placementPolicy;
- }
-
- public long getPlacementTimeoutSeconds() {
- return providerRole.placementTimeoutSeconds;
- }
-
- /**
- * The number of failures on a specific node that can be tolerated
- * before selecting a different node for placement
- * @return
- */
- public int getNodeFailureThreshold() {
- return providerRole.nodeFailureThreshold;
- }
-
- public boolean isExcludeFromFlexing() {
- return hasPlacementPolicy(PlacementPolicy.EXCLUDE_FROM_FLEXING);
- }
-
- public boolean isStrictPlacement() {
- return hasPlacementPolicy(PlacementPolicy.STRICT);
- }
-
- public boolean isAntiAffinePlacement() {
- return hasPlacementPolicy(PlacementPolicy.ANTI_AFFINITY_REQUIRED);
- }
-
- public boolean hasPlacementPolicy(int policy) {
- return 0 != (getPlacementPolicy() & policy);
- }
-
- public boolean isPlacementDesired() {
- return !hasPlacementPolicy(PlacementPolicy.ANYWHERE);
- }
-
- /**
- * Probe for an outstanding AA request being true
- * @return true if there is an outstanding AA Request
- */
- public boolean isAARequestOutstanding() {
- return outstandingAArequest != null;
- }
-
- /**
- * expose the predicate {@link #isAARequestOutstanding()} as an integer,
- * which is very convenient in tests
- * @return 1 if there is an outstanding request; 0 if not
- */
- public int getOutstandingAARequestCount() {
- return isAARequestOutstanding()? 1: 0;
- }
- /**
- * Note that a role failed, text will
- * be used in any diagnostics if an exception
- * is later raised.
- * @param text text about the failure
- */
- public synchronized void noteFailed(String text) {
- if (text != null) {
- failureMessage = text;
- }
- }
-
-
- public void setOutstandingAArequest(OutstandingRequest outstandingAArequest) {
- this.outstandingAArequest = outstandingAArequest;
- }
-
- /**
- * Complete the outstanding AA request (there's no check for one in progress, caller
- * expected to have done that).
- */
- public void completeOutstandingAARequest() {
- setOutstandingAArequest(null);
- }
-
- /**
- * Cancel any outstanding AA request. Harmless if the role is non-AA, or
- * if there are no outstanding requests.
- */
- public void cancelOutstandingAARequest() {
- if (outstandingAArequest != null) {
- setOutstandingAArequest(null);
- }
- }
-
- public long getDesired() {
- return componentMetrics.containersDesired.value();
- }
-
- public void setDesired(int desired) {
- componentMetrics.containersDesired.set(desired);
- }
-
- public long getRunning() {
- return componentMetrics.containersRunning.value();
- }
-
- public long getRequested() {
- return componentMetrics.containersRequested.value();
- }
-
- public long getAAPending() {
- return componentMetrics.pendingAAContainers.value();
- }
-
- void decAAPending() {
- componentMetrics.pendingAAContainers.decr();
- }
-
- void setAAPending(long n) {
- componentMetrics.pendingAAContainers.set((int)n);
- }
-
- public long getLimitsExceeded() {
- return componentMetrics.containersLimitsExceeded.value();
- }
-
- public long getPreempted() {
- return componentMetrics.containersPreempted.value();
- }
-
- public long getDiskFailed() {
- return componentMetrics.containersDiskFailure.value();
- }
-
- public long getFailedRecently() {
-// return componentMetrics.failedSinceLastThreshold.value();
- return 0;
- }
-
- public long resetFailedRecently() {
-// long count =
-// componentMetrics.failedSinceLastThreshold.value();
-// componentMetrics.failedSinceLastThreshold.set(0);
- return 0;
- }
-
- public long getFailed() {
- return componentMetrics.containersFailed.value();
- }
-
- String getFailureMessage() {
- return this.failureMessage;
- }
- /**
- * Get the number of roles we are short of.
- * nodes released are ignored.
- * @return the positive or negative number of roles to add/release.
- * 0 means "do nothing".
- */
- public long getDelta() {
- long inuse = getActualAndRequested();
- long delta = getDesired() - inuse;
- if (delta < 0) {
- // TODO this doesn't do anything now that we're not tracking releasing
- // containers -- maybe we need releasing
- //if we are releasing, remove the number that are already released.
- //but never switch to a positive
- // TODO, WHY is this min operation even needed ??? if delta is negative, it's always < 0 ???
- delta = Math.min(delta, 0);
- }
- return delta;
- }
-
- /**
- * Get count of actual and requested containers.
- * @return the size of the application when outstanding requests are included.
- */
- public long getActualAndRequested() {
- return getRunning() + getRequested();
- }
-
- /**
- * Get the provider role
- * @return the provider role
- */
- public ProviderRole getProviderRole() {
- return providerRole;
- }
-
- /**
- * Produced a serialized form which can be served up as JSON
- * @return a summary of the current role status.
- */
- public synchronized ComponentInformation serialize() {
- ComponentInformation info = new ComponentInformation();
- info.name = name;
- return info;
- }
-
- /**
- * Get the (possibly null) label expression for this role
- * @return a string or null
- */
- public String getLabelExpression() {
- return providerRole.labelExpression;
- }
-
- public Resource getResourceRequirements() {
- return resourceRequirements;
- }
-
- public void setResourceRequirements(Resource resourceRequirements) {
- this.resourceRequirements = resourceRequirements;
- }
-
- /**
- * Compare two role status entries by name
- */
- public static class CompareByName implements Comparator<RoleStatus>,
- Serializable {
- @Override
- public int compare(RoleStatus o1, RoleStatus o2) {
- return o1.getName().compareTo(o2.getName());
- }
- }
-
- /**
- * Compare two role status entries by key
- */
- public static class CompareByKey implements Comparator<RoleStatus>,
- Serializable {
- @Override
- public int compare(RoleStatus o1, RoleStatus o2) {
- return (o1.getKey() < o2.getKey() ? -1 : (o1.getKey() == o2.getKey() ? 0 : 1));
- }
- }
-
- /**
- * Given a resource, set its requirements to those this role needs
- * @param resource resource to configure
- * @return the resource
- */
- public Resource copyResourceRequirements(Resource resource) {
- Preconditions.checkNotNull(resourceRequirements,
- "Role resource requirements have not been set");
- resource.setMemory(resourceRequirements.getMemory());
- resource.setVirtualCores(resourceRequirements.getVirtualCores());
- return resource;
- }
-
- public synchronized RoleStatistics getStatistics() {
- RoleStatistics stats = new RoleStatistics();
- stats.activeAA = getOutstandingAARequestCount();
- stats.actual = getRunning();
- stats.desired = getDesired();
- stats.failed = getFailed();
- stats.limitsExceeded = getLimitsExceeded();
- stats.nodeFailed = getDiskFailed();
- stats.preempted = getPreempted();
- stats.requested = getRequested();
- stats.started = getRunning();
- return stats;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java
deleted file mode 100644
index b848096..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import java.util.List;
-
-/**
- * Simplest release selector simply returns the list
- */
-public class SimpleReleaseSelector implements ContainerReleaseSelector {
-
- @Override
- public List<RoleInstance> sortCandidates(int roleId,
- List<RoleInstance> candidates) {
- return candidates;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
deleted file mode 100644
index 90221cb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.google.common.cache.LoadingCache;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.StatusKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.api.types.ApplicationLivenessInformation;
-import org.apache.slider.api.types.ComponentInformation;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.api.types.RoleStatistics;
-import org.apache.slider.core.exceptions.NoSuchNodeException;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
-import org.apache.slider.core.registry.docstore.PublishedExportsSet;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-/**
- * The methods to offer state access to the providers and other parts of
- * the system which want read-only access to the state.
- */
-public interface StateAccessForProviders {
-
- /**
- * Get a map of role status entries by role Id
- * @return the map of currently defined roles.
- */
- Map<Integer, RoleStatus> getRoleStatusMap();
-
- /**
- * Get the name of the application
- * @return the name
- */
- String getApplicationName();
-
- /**
- * Get the published configurations
- * @return the configuration set
- */
- PublishedConfigSet getPublishedSliderConfigurations();
-
- /**
- * Get the published exports set
- * @return
- */
- PublishedExportsSet getPublishedExportsSet();
-
- /**
- * Get a named published config set
- * @param name name to look up
- * @return the instance or null
- */
- PublishedConfigSet getPublishedConfigSet(String name);
-
- /**
- * Get a named published config set, creating it if need be.
- * @param name name to look up
- * @return the instance -possibly a new one
- */
- PublishedConfigSet getOrCreatePublishedConfigSet(String name);
-
- /**
- * List the config sets -this takes a clone of the current set
- * @return a list of config sets
- */
- List<String> listConfigSets();
-
- /**
- * Get a map of all the failed containers
- * @return map of recorded failed containers
- */
- Map<ContainerId, RoleInstance> getFailedContainers();
-
- /**
- * Get the live containers.
- *
- * @return the live nodes
- */
- Map<ContainerId, RoleInstance> getLiveContainers();
-
- /**
- * Get the current cluster description
- * @return the actual state of the cluster
- */
- Application getApplication();
-
- /**
- * Flag set to indicate the application is live -this only happens
- * after the buildInstance operation
- */
- boolean isApplicationLive();
-
- /**
- * Look up a role from its key -or fail
- *
- * @param key key to resolve
- * @return the status
- * @throws YarnRuntimeException on no match
- */
- RoleStatus lookupRoleStatus(int key);
-
- /**
- * Look up a role from its key -or fail
- *
- * @param c container in a role
- * @return the status
- * @throws YarnRuntimeException on no match
- */
- RoleStatus lookupRoleStatus(Container c) throws YarnRuntimeException;
-
- /**
- * Look up a role from its key -or fail
- *
- * @param name container in a role
- * @return the status
- * @throws YarnRuntimeException on no match
- */
- RoleStatus lookupRoleStatus(String name) throws YarnRuntimeException;
-
- /**
- * Clone a list of active containers
- * @return the active containers at the time
- * the call was made
- */
- List<RoleInstance> cloneOwnedContainerList();
-
- /**
- * Get the number of active containers
- * @return the number of active containers the time the call was made
- */
- int getNumOwnedContainers();
-
- /**
- * Get any active container with the given ID
- * @param id container Id
- * @return the active container or null if it is not found
- */
- RoleInstance getOwnedContainer(ContainerId id);
-
- /**
- * Get any active container with the given ID
- * @param id container Id
- * @return the active container or null if it is not found
- */
- RoleInstance getOwnedContainer(String id) throws NoSuchNodeException;
-
- /**
- * Create a clone of the list of live cluster nodes.
- * @return the list of nodes, may be empty
- */
- List<RoleInstance> cloneLiveContainerInfoList();
-
- /**
- * Get the {@link RoleInstance} details on a container.
- * This is an O(n) operation
- * @param containerId the container ID
- * @return null if there is no such node
- * @throws NoSuchNodeException if the node cannot be found
- */
- RoleInstance getLiveInstanceByContainerID(String containerId)
- throws NoSuchNodeException;
-
- /**
- * Get the details on a list of instaces referred to by ID.
- * Unknown nodes are not returned
- * <i>Important: the order of the results are undefined</i>
- * @param containerIDs the containers
- * @return list of instances
- */
- List<RoleInstance> getLiveInstancesByContainerIDs(
- Collection<String> containerIDs);
-
- /**
- * Update the cluster description with anything interesting
- */
- Application refreshClusterStatus();
-
-
- /**
- * get application liveness information
- * @return a snapshot of the current liveness information
- */
- ApplicationLivenessInformation getApplicationLivenessInformation();
-
- /**
- * Get a snapshot of component information.
- * <p>
- * This does <i>not</i> include any container list, which
- * is more expensive to create.
- * @return a map of current role status values.
- */
- Map<String, ComponentInformation> getComponentInfoSnapshot();
-
- /**
- * Find out about the nodes for specific roles
- * Component_name -> ContainerId -> ClusterNode
- * @return
- */
- Map<String, Map<String, ClusterNode>> getRoleClusterNodeMapping();
-
- /**
- * Enum all role instances by role.
- * @param role role, or "" for all roles
- * @return a list of instances, may be empty
- */
- List<RoleInstance> enumLiveInstancesInRole(String role);
-
- /**
- * Look up all containers of a specific component name
- * @param component component/role name
- * @return list of instances. This is a snapshot
- */
- List<RoleInstance> lookupRoleContainers(String component);
-
- /**
- * Get the JSON serializable information about a component
- * @param component component to look up
- * @return a structure describing the component.
- */
- ComponentInformation getComponentInformation(String component);
-
-
- /**
- * Get a clone of the nodemap.
- * The instances inside are not cloned
- * @return a possibly empty map of hostname top info
- */
- Map<String, NodeInformation> getNodeInformationSnapshot();
-
- /**
- * get information on a node
- * @param hostname hostname to look up
- * @return the information, or null if there is no information held.
- */
- NodeInformation getNodeInformation(String hostname);
-
- /**
- * Get the aggregate statistics across all roles
- * @return role statistics
- */
- RoleStatistics getRoleStatistics();
-
- /**
- * Get global substitution tokens.
- */
- Map<String, String> getGlobalSubstitutionTokens();
-
- /**
- * Get config file cache.
- */
- LoadingCache<ConfigFile, Object> getConfigFileCache();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java
deleted file mode 100644
index be8960d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web;
-
-/*
-
-
- , );
- long now = System.currentTimeMillis();
- httpRes.addDateHeader ( "Expires", now );
- httpRes.addDateHeader ( "Date", now );
- httpRes.addHeader ( "Pragma", "no-cache" );
- */
-public interface HttpCacheHeaders {
- String HTTP_HEADER_CACHE_CONTROL = "Cache-Control";
- String HTTP_HEADER_CACHE_CONTROL_NONE = "no-cache";
- String HTTP_HEADER_PRAGMA = "Pragma";
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMController.java
deleted file mode 100644
index c3c6e60..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMController.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web;
-
-import com.google.inject.Inject;
-import org.apache.hadoop.yarn.webapp.Controller;
-import org.apache.slider.server.appmaster.web.layout.AppLayout;
-import org.apache.slider.server.appmaster.web.layout.ClusterSpecificationView;
-import org.apache.slider.server.appmaster.web.layout.ContainerStatsView;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-public class SliderAMController extends Controller {
- private static final Logger log = LoggerFactory.getLogger(SliderAMController.class);
-
- private final WebAppApi slider;
-
- @Inject
- public SliderAMController(WebAppApi slider, RequestContext ctx) {
- super(ctx);
- this.slider = slider;
- }
-
- @Override
- public void index() {
- setTitle("Slider App Master");
-
- updateAppState();
-
- render(AppLayout.class);
- }
-
- public void containerStats() {
- setTitle("Container Statistics");
-
- updateAppState();
-
- render(ContainerStatsView.class);
- }
-
- public void specification() {
- setTitle("Cluster Specification");
-
- render(ClusterSpecificationView.class);
- }
-
- private void updateAppState() {
- //TODO don't do this on every request?
- slider.getAppState().refreshClusterStatus();
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
deleted file mode 100644
index 1fd49ab..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.client.ClientRMProxy;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.conf.HAUtil;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.text.DateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hadoop.security.UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
-
-/**
- * Utils to work with credentials and tokens.
- *
- * Designed to be movable to Hadoop core
- */
-public final class CredentialUtils {
-
- public static final String JOB_CREDENTIALS_BINARY
- = SliderXmlConfKeys.MAPREDUCE_JOB_CREDENTIALS_BINARY;
-
- private CredentialUtils() {
- }
-
- private static final Logger LOG =
- LoggerFactory.getLogger(CredentialUtils.class);
-
- /**
- * Save credentials to a byte buffer. Returns null if there were no
- * credentials to save
- * @param credentials credential set
- * @return a byte buffer of serialized tokens
- * @throws IOException if the credentials could not be written to the stream
- */
- public static ByteBuffer marshallCredentials(Credentials credentials) throws IOException {
- ByteBuffer buffer = null;
- if (!credentials.getAllTokens().isEmpty()) {
- DataOutputBuffer dob = new DataOutputBuffer();
- try {
- credentials.writeTokenStorageToStream(dob);
- } finally {
- dob.close();
- }
- buffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
- }
- return buffer;
- }
-
- public static File locateEnvCredentials(Map<String, String> env,
- Configuration conf,
- StringBuffer sourceTextOut) throws FileNotFoundException {
- String tokenFilename = env.get(HADOOP_TOKEN_FILE_LOCATION);
- String source = "environment variable " + HADOOP_TOKEN_FILE_LOCATION;
- if (tokenFilename == null) {
- tokenFilename = conf.get(JOB_CREDENTIALS_BINARY);
- source = "configuration option " + JOB_CREDENTIALS_BINARY;
- }
- if (tokenFilename != null) {
- // use delegation tokens, i.e. from Oozie
- File file = new File(tokenFilename.trim());
- String details = String.format(
- "Token File %s from %s",
- file,
- source);
- if (!file.exists()) {
- throw new FileNotFoundException("No " + details);
- }
- if (!file.isFile() && !file.canRead()) {
- throw new FileNotFoundException("Cannot read " + details);
- }
- sourceTextOut.append(details);
- return file;
- } else {
- return null;
- }
- }
-
- /**
- * Load the credentials from the environment. This looks at
- * the value of {@link UserGroupInformation#HADOOP_TOKEN_FILE_LOCATION}
- * and attempts to read in the value
- * @param env environment to resolve the variable from
- * @param conf configuration use when reading the tokens
- * @return a set of credentials, or null if the environment did not
- * specify any
- * @throws IOException if a location for credentials was defined, but
- * the credentials could not be loaded.
- */
- public static Credentials loadTokensFromEnvironment(Map<String, String> env,
- Configuration conf)
- throws IOException {
- StringBuffer origin = new StringBuffer();
- File file = locateEnvCredentials(env, conf, origin);
- if (file != null) {
- LOG.debug("Using {}", origin);
- return Credentials.readTokenStorageFile(file, conf);
- } else {
- return null;
- }
- }
-
- /**
- * Save credentials to a file
- * @param file file to save to (will be overwritten)
- * @param credentials credentials to write
- * @throws IOException
- */
- public static void saveTokens(File file,
- Credentials credentials) throws IOException {
- try(DataOutputStream daos = new DataOutputStream(
- new FileOutputStream(file))) {
- credentials.writeTokenStorageToStream(daos);
- }
- }
-
- /**
- * Look up and return the resource manager's principal. This method
- * automatically does the <code>_HOST</code> replacement in the principal and
- * correctly handles HA resource manager configurations.
- *
- * From: YARN-4629
- * @param conf the {@link Configuration} file from which to read the
- * principal
- * @return the resource manager's principal string
- * @throws IOException thrown if there's an error replacing the host name
- */
- public static String getRMPrincipal(Configuration conf) throws IOException {
- String principal = conf.get(RM_PRINCIPAL, "");
- String hostname;
- Preconditions.checkState(!principal.isEmpty(), "Not set: " + RM_PRINCIPAL);
-
- if (HAUtil.isHAEnabled(conf)) {
- YarnConfiguration yarnConf = new YarnConfiguration(conf);
- if (yarnConf.get(RM_HA_ID) == null) {
- // If RM_HA_ID is not configured, use the first of RM_HA_IDS.
- // Any valid RM HA ID should work.
- String[] rmIds = yarnConf.getStrings(RM_HA_IDS);
- Preconditions.checkState((rmIds != null) && (rmIds.length > 0),
- "Not set " + RM_HA_IDS);
- yarnConf.set(RM_HA_ID, rmIds[0]);
- }
-
- hostname = yarnConf.getSocketAddr(
- RM_ADDRESS,
- DEFAULT_RM_ADDRESS,
- DEFAULT_RM_PORT).getHostName();
- } else {
- hostname = conf.getSocketAddr(
- RM_ADDRESS,
- DEFAULT_RM_ADDRESS,
- DEFAULT_RM_PORT).getHostName();
- }
- return SecurityUtil.getServerPrincipal(principal, hostname);
- }
-
- /**
- * Create and add any filesystem delegation tokens with
- * the RM(s) configured to be able to renew them. Returns null
- * on an insecure cluster (i.e. harmless)
- * @param conf configuration
- * @param fs filesystem
- * @param credentials credentials to update
- * @return a list of all added tokens.
- * @throws IOException
- */
- public static Token<?>[] addRMRenewableFSDelegationTokens(Configuration conf,
- FileSystem fs,
- Credentials credentials) throws IOException {
- Preconditions.checkArgument(conf != null);
- Preconditions.checkArgument(credentials != null);
- if (UserGroupInformation.isSecurityEnabled()) {
- return fs.addDelegationTokens(CredentialUtils.getRMPrincipal(conf),
- credentials);
- }
- return null;
- }
-
- /**
- * Add an FS delegation token which can be renewed by the current user
- * @param fs filesystem
- * @param credentials credentials to update
- * @throws IOException problems.
- */
- public static void addSelfRenewableFSDelegationTokens(
- FileSystem fs,
- Credentials credentials) throws IOException {
- Preconditions.checkArgument(fs != null);
- Preconditions.checkArgument(credentials != null);
- fs.addDelegationTokens(
- getSelfRenewer(),
- credentials);
- }
-
- public static String getSelfRenewer() throws IOException {
- return UserGroupInformation.getLoginUser().getShortUserName();
- }
-
- /**
- * Create and add an RM delegation token to the credentials
- * @param yarnClient Yarn Client
- * @param credentials to add token to
- * @return the token which was added
- * @throws IOException
- * @throws YarnException
- */
- public static Token<TokenIdentifier> addRMDelegationToken(YarnClient yarnClient,
- Credentials credentials)
- throws IOException, YarnException {
- Configuration conf = yarnClient.getConfig();
- Text rmPrincipal = new Text(CredentialUtils.getRMPrincipal(conf));
- Text rmDTService = ClientRMProxy.getRMDelegationTokenService(conf);
- Token<TokenIdentifier> rmDelegationToken =
- ConverterUtils.convertFromYarn(
- yarnClient.getRMDelegationToken(rmPrincipal),
- rmDTService);
- credentials.addToken(rmDelegationToken.getService(), rmDelegationToken);
- return rmDelegationToken;
- }
-
- public static Token<TimelineDelegationTokenIdentifier> maybeAddTimelineToken(
- Configuration conf,
- Credentials credentials)
- throws IOException, YarnException {
- if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
- LOG.debug("Timeline service enabled -fetching token");
-
- try(TimelineClient timelineClient = TimelineClient.createTimelineClient()) {
- timelineClient.init(conf);
- timelineClient.start();
- Token<TimelineDelegationTokenIdentifier> token =
- timelineClient.getDelegationToken(
- CredentialUtils.getRMPrincipal(conf));
- credentials.addToken(token.getService(), token);
- return token;
- }
- } else {
- LOG.debug("Timeline service is disabled");
- return null;
- }
- }
-
- /**
- * Filter a list of tokens from a set of credentials
- * @param credentials credential source (a new credential set os re
- * @param filter List of tokens to strip out
- * @return a new, filtered, set of credentials
- */
- public static Credentials filterTokens(Credentials credentials,
- List<Text> filter) {
- Credentials result = new Credentials(credentials);
- Iterator<Token<? extends TokenIdentifier>> iter =
- result.getAllTokens().iterator();
- while (iter.hasNext()) {
- Token<? extends TokenIdentifier> token = iter.next();
- LOG.debug("Token {}", token.getKind());
- if (filter.contains(token.getKind())) {
- LOG.debug("Filtering token {}", token.getKind());
- iter.remove();
- }
- }
- return result;
- }
-
- public static String dumpTokens(Credentials credentials, String separator) {
- ArrayList<Token<? extends TokenIdentifier>> sorted =
- new ArrayList<>(credentials.getAllTokens());
- Collections.sort(sorted, new TokenComparator());
- StringBuilder buffer = new StringBuilder(sorted.size()* 128);
- for (Token<? extends TokenIdentifier> token : sorted) {
- buffer.append(tokenToString(token)).append(separator);
- }
- return buffer.toString();
- }
-
- /**
- * Create a string for people to look at
- * @param token token to convert to a string form
- * @return a printable view of the token
- */
- public static String tokenToString(Token<? extends TokenIdentifier> token) {
- DateFormat df = DateFormat.getDateTimeInstance(
- DateFormat.SHORT, DateFormat.SHORT);
- StringBuilder buffer = new StringBuilder(128);
- buffer.append(token.toString());
- try {
- TokenIdentifier ti = token.decodeIdentifier();
- buffer.append("; ").append(ti);
- if (ti instanceof AbstractDelegationTokenIdentifier) {
- // details in human readable form, and compensate for information HDFS DT omits
- AbstractDelegationTokenIdentifier dt = (AbstractDelegationTokenIdentifier) ti;
- buffer.append("; Renewer: ").append(dt.getRenewer());
- buffer.append("; Issued: ")
- .append(df.format(new Date(dt.getIssueDate())));
- buffer.append("; Max Date: ")
- .append(df.format(new Date(dt.getMaxDate())));
- }
- } catch (IOException e) {
- //marshall problem; not ours
- LOG.debug("Failed to decode {}: {}", token, e, e);
- }
- return buffer.toString();
- }
-
- /**
- * Get the expiry time of a token.
- * @param token token to examine
- * @return the time in milliseconds after which the token is invalid.
- * @throws IOException
- */
- public static long getTokenExpiryTime(Token token) throws IOException {
- TokenIdentifier identifier = token.decodeIdentifier();
- Preconditions.checkState(identifier instanceof AbstractDelegationTokenIdentifier,
- "Token %s of type: %s has an identifier which cannot be examined: %s",
- token, token.getClass(), identifier);
- AbstractDelegationTokenIdentifier id =
- (AbstractDelegationTokenIdentifier) identifier;
- return id.getMaxDate();
- }
-
- private static class TokenComparator
- implements Comparator<Token<? extends TokenIdentifier>>, Serializable {
- @Override
- public int compare(Token<? extends TokenIdentifier> left,
- Token<? extends TokenIdentifier> right) {
- return left.getKind().toString().compareTo(right.getKind().toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
deleted file mode 100644
index b8aa4c6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
-
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- * Command line builder purely for the Java CLI.
- * Some of the <code>define</code> methods are designed to work with Hadoop tool and
- * Slider launcher applications.
- */
-public class JavaCommandLineBuilder extends CommandLineBuilder {
-
- public JavaCommandLineBuilder() {
- add(getJavaBinary());
- }
-
- /**
- * Get the java binary. This is called in the constructor so don't try and
- * do anything other than return a constant.
- * @return the path to the Java binary
- */
- protected String getJavaBinary() {
- return ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java";
- }
-
- /**
- * Set the size of the heap if a non-empty heap is passed in.
- * @param heap empty string or something like "128M" ,"1G" etc. The value is
- * trimmed.
- */
- public void setJVMHeap(String heap) {
- if (SliderUtils.isSet(heap)) {
- add("-Xmx" + heap.trim());
- }
- }
-
- /**
- * Turn Java assertions on
- */
- public void enableJavaAssertions() {
- add("-ea");
- add("-esa");
- }
-
- /**
- * Add a system property definition -must be used before setting the main entry point
- * @param property
- * @param value
- */
- public void sysprop(String property, String value) {
- Preconditions.checkArgument(property != null, "null property name");
- Preconditions.checkArgument(value != null, "null value");
- add("-D" + property + "=" + value);
- }
-
- public JavaCommandLineBuilder forceIPv4() {
- sysprop("java.net.preferIPv4Stack", "true");
- return this;
- }
-
- public JavaCommandLineBuilder headless() {
- sysprop("java.awt.headless", "true");
- return this;
- }
-
- public boolean addConfOption(Configuration conf, String key) {
- return defineIfSet(key, conf.get(key));
- }
-
- /**
- * Add a varargs list of configuration parameters —if they are present
- * @param conf configuration source
- * @param keys keys
- */
- public void addConfOptions(Configuration conf, String... keys) {
- for (String key : keys) {
- addConfOption(conf, key);
- }
- }
-
- /**
- * Add all configuration options which match the prefix
- * @param conf configuration
- * @param prefix prefix, e.g {@code "slider."}
- * @return the number of entries copied
- */
- public int addPrefixedConfOptions(Configuration conf, String prefix) {
- int copied = 0;
- for (Map.Entry<String, String> entry : conf) {
- if (entry.getKey().startsWith(prefix)) {
- define(entry.getKey(), entry.getValue());
- copied++;
- }
- }
- return copied;
- }
-
- /**
- * Ass a configuration option to the command line of the application
- * @param conf configuration
- * @param key key
- * @param defVal default value
- * @return the resolved configuration option
- * @throws IllegalArgumentException if key is null or the looked up value
- * is null (that is: the argument is missing and devVal was null.
- */
- public String addConfOptionToCLI(Configuration conf,
- String key,
- String defVal) {
- Preconditions.checkArgument(key != null, "null key");
- String val = conf.get(key, defVal);
- define(key, val);
- return val;
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI. This is very Hadoop API
- * @param key key
- * @param val value
- * @throws IllegalArgumentException if either argument is null
- */
- public void define(String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- Preconditions.checkArgument(val != null, "null value");
- add("-D", key + "=" + val);
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
- * is not null
- * @param key key
- * @param val value
- */
- public boolean defineIfSet(String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- if (val != null) {
- define(key, val);
- return true;
- } else {
- return false;
- }
- }
-
- /**
- * Add a mandatory config option
- * @param conf configuration
- * @param key key
- * @throws BadConfigException if the key is missing
- */
- public void addMandatoryConfOption(Configuration conf,
- String key) throws BadConfigException {
- if (!addConfOption(conf, key)) {
- throw new BadConfigException("Missing configuration option: " + key);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java
deleted file mode 100644
index 8e0ef5a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.slider.core.persist.ApplicationReportSerDeser;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.io.IOException;
-
-/**
- * Serialized form of an application report which can be persisted
- * and then parsed. It can not be converted back into a
- * real YARN application report
- *
- * Useful for testing
- */
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-
-public class SerializedApplicationReport {
-
- public String applicationId;
- public String applicationAttemptId;
- public String name;
- public String applicationType;
- public String user;
- public String queue;
- public String host;
- public Integer rpcPort;
- public String state;
- public String diagnostics;
- public String url;
- /**
- * This value is non-null only when a report is generated from a submission context.
- * The YARN {@link ApplicationReport} structure does not propagate this value
- * from the RM.
- */
- public Long submitTime;
- public Long startTime;
- public Long finishTime;
- public String finalStatus;
- public String origTrackingUrl;
- public Float progress;
-
- public SerializedApplicationReport() {
- }
-
- public SerializedApplicationReport(ApplicationReport report) {
- this.applicationId = report.getApplicationId().toString();
- ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId();
- this.applicationAttemptId = attemptId != null ? attemptId.toString() : "N/A";
- this.name = report.getName();
- this.applicationType = report.getApplicationType();
- this.user = report.getUser();
- this.queue = report.getQueue();
- this.host = report.getHost();
- this.rpcPort = report.getRpcPort();
- this.state = report.getYarnApplicationState().toString();
- this.diagnostics = report.getDiagnostics();
- this.startTime = report.getStartTime();
- this.finishTime = report.getFinishTime();
- FinalApplicationStatus appStatus = report.getFinalApplicationStatus();
- this.finalStatus = appStatus == null ? "" : appStatus.toString();
- this.progress = report.getProgress();
- this.url = report.getTrackingUrl();
- this.origTrackingUrl= report.getOriginalTrackingUrl();
- }
-
- @Override
- public String toString() {
- try {
- return ApplicationReportSerDeser.toString(this);
- } catch (IOException e) {
- return super.toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ExitCodeProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ExitCodeProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ExitCodeProvider.java
deleted file mode 100644
index 503b9b9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ExitCodeProvider.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.main;
-
-/**
- * Get the exit code of an exception. Making it an interface allows
- * us to retrofit exit codes onto existing classes
- */
-public interface ExitCodeProvider {
-
- /**
- * Method to get the exit code
- * @return the exit code
- */
- int getExitCode();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/IrqHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/IrqHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/IrqHandler.java
deleted file mode 100644
index 42442d1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/IrqHandler.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.main;
-
-import sun.misc.Signal;
-import sun.misc.SignalHandler;
-
-import java.io.IOException;
-
-/**
- * This class bundles up all the compiler warnings about abuse of sun.misc
- * interrupt handling code
- * into one place.
- */
-@SuppressWarnings("UseOfSunClasses")
-public final class IrqHandler implements SignalHandler {
-
- public static final String CONTROL_C = "INT";
- public static final String SIGTERM = "TERM";
-
- private final String name;
- private final Interrupted handler;
-
- /**
- * Create an IRQ handler bound to the specific interrupt
- * @param name signal name
- * @param handler handler
- * @throws IOException
- */
- public IrqHandler(String name, Interrupted handler) throws IOException {
- this.handler = handler;
- this.name = name;
- try {
- Signal.handle(new Signal(name), this);
- } catch (IllegalArgumentException e) {
- throw new IOException(
- "Could not set handler for signal \"" + name + "\"."
- + "This can happen if the JVM has the -Xrs set.",
- e);
- }
- }
-
- @Override
- public String toString() {
- return "IrqHandler for signal " + name ;
- }
-
- /**
- * Handler for the JVM API for signal handling
- * @param signal signal raised
- */
-// @Override
- public void handle(Signal signal) {
- InterruptData data = new InterruptData(signal.getName(), signal.getNumber());
- handler.interrupted(data);
- }
-
- /**
- * Interrupt data to pass on.
- */
- public static class InterruptData {
- public final String name;
- public final int number;
-
- public InterruptData(String name, int number) {
- this.name = name;
- this.number = number;
- }
-
- @Override
- public String toString() {
- return "signal " + name + '(' + number + ')';
- }
- }
-
- /**
- * Callback on interruption
- */
- public interface Interrupted {
-
- /**
- * Handle an interrupt
- * @param interruptData data
- */
- void interrupted(InterruptData interruptData);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/LauncherExitCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/LauncherExitCodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/LauncherExitCodes.java
deleted file mode 100644
index 83e89f0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/LauncherExitCodes.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.main;
-
-/*
- * Common Exit codes
- * <p>
- * Exit codes from 64 up are application specific.
- * <p>
- * Many of the exit codes are designed to resemble HTTP error codes,
- * squashed into a single byte. e.g 44 , "not found" is the equivalent
- * of 404
- * <pre>
- * 0-10: general command issues
- * 30-39: equivalent to the 3XX responses, where those responses are
- * considered errors by the application.
- * 40-49: request-related errors
- * 50-59: server-side problems. These may be triggered by the request.
- * 64- : application specific error codes
- * </pre>
- */
-public interface LauncherExitCodes {
-
- /**
- * 0: success
- */
- int EXIT_SUCCESS = 0;
-
- /**
- * -1: generic "false" response. The operation worked but
- * the result was not true
- */
- int EXIT_FALSE = -1;
-
- /**
- * Exit code when a client requested service termination: {@value}
- */
- int EXIT_CLIENT_INITIATED_SHUTDOWN = 1;
-
- /**
- * Exit code when targets could not be launched: {@value}
- */
- int EXIT_TASK_LAUNCH_FAILURE = 2;
-
- /**
- * Exit code when a control-C, kill -3, signal was picked up: {@value}
- */
- int EXIT_INTERRUPTED = 3;
-
- /**
- * Exit code when a usage message was printed: {@value}
- */
- int EXIT_USAGE = 4;
-
- /**
- * Exit code when something happened but we can't be specific: {@value}
- */
- int EXIT_OTHER_FAILURE = 5;
-
- /**
- * Exit code on connectivity problems: {@value}
- */
- int EXIT_MOVED = 31;
-
- /**
- * found: {@value}.
- * <p>
- * This is low value as in HTTP it is normally a success/redirect;
- * whereas on the command line 0 is the sole success code.
- * <p>
- * <code>302 Found</code>
- */
- int EXIT_FOUND = 32;
-
- /**
- * Exit code on a request where the destination has not changed
- * and (somehow) the command specified that this is an error.
- * That is, this exit code is somehow different from a "success"
- * : {@value}
- * <p>
- * <code>304 Not Modified </code>
- */
- int EXIT_NOT_MODIFIED = 34;
-
- /**
- * Exit code when the command line doesn't parse: {@value}, or
- * when it is otherwise invalid.
- * <p>
- * <code>400 BAD REQUEST</code>
- */
- int EXIT_COMMAND_ARGUMENT_ERROR = 40;
-
- /**
- * The request requires user authentication: {@value}
- * <p>
- * <code>401 Unauthorized</code>
- */
- int EXIT_UNAUTHORIZED = 41;
-
- /**
- * Forbidden action: {@value}
- * <p>
- * <code>403: Forbidden</code>
- */
- int EXIT_FORBIDDEN = 43;
-
- /**
- * Something was not found: {@value}
- * <p>
- * <code>404: NOT FOUND</code>
- */
- int EXIT_NOT_FOUND = 44;
-
- /**
- * The operation is not allowed: {@value}
- * <p>
- * <code>405: NOT ALLOWED</code>
- */
- int EXIT_OPERATION_NOT_ALLOWED = 45;
-
- /**
- * The command is somehow not acceptable: {@value}
- * <p>
- * <code>406: NOT ACCEPTABLE</code>
- */
- int EXIT_NOT_ACCEPTABLE = 46;
-
- /**
- * Exit code on connectivity problems: {@value}
- * <p>
- * <code>408: Request Timeout</code>
- */
- int EXIT_CONNECTIVITY_PROBLEM = 48;
-
- /**
- * The request could not be completed due to a conflict with the current
- * state of the resource. {@value}
- * <p>
- * <code>409: conflict</code>
- */
- int EXIT_CONFLICT = 49;
-
- /**
- * internal error: {@value}
- * <p>
- * <code>500 Internal Server Error</code>
- */
- int EXIT_INTERNAL_ERROR = 50;
-
- /**
- * Unimplemented feature: {@value}
- * <p>
- * <code>501: Not Implemented</code>
- */
- int EXIT_UNIMPLEMENTED = 51;
-
- /**
- * Service Unavailable; it may be available later: {@value}
- * <p>
- * <code>503 Service Unavailable</code>
- */
- int EXIT_SERVICE_UNAVAILABLE = 53;
-
- /**
- * The application does not support, or refuses to support this version: {@value}.
- * If raised, this is expected to be raised server-side and likely due
- * to client/server version incompatibilities.
- * <p>
- * <code> 505: Version Not Supported</code>
- */
- int EXIT_UNSUPPORTED_VERSION = 55;
-
- /**
- * Exit code when an exception was thrown from the service: {@value}
- * <p>
- * <code>5XX</code>
- */
- int EXIT_EXCEPTION_THROWN = 56;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/RunService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/RunService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/RunService.java
deleted file mode 100644
index c3a1d0e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/RunService.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.main;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.Service;
-
-/**
- * An interface which services can implement to have their
- * execution managed by the ServiceLauncher.
- * The command line options will be passed down before the
- * {@link Service#init(Configuration)} operation is invoked via an
- * invocation of {@link RunService#bindArgs(Configuration, String...)}
- * After the service has been successfully started via {@link Service#start()}
- * the {@link RunService#runService()} method is called to execute the
- * service. When this method returns, the service launcher will exit, using
- * the return code from the method as its exit option.
- */
-public interface RunService extends Service {
-
- /**
- * Propagate the command line arguments.
- * This method is called before {@link Service#init(Configuration)};
- * the configuration that is returned from this operation
- * is the one that is passed on to the init operation.
- * This permits implemenations to change the configuration before
- * the init operation.n
- *
- *
- * @param config the initial configuration build up by the
- * service launcher.
- * @param args argument list list of arguments passed to the command line
- * after any launcher-specific commands have been stripped.
- * @return the configuration to init the service with. This MUST NOT be null.
- * Recommended: pass down the config parameter with any changes
- * @throws Exception any problem
- */
- Configuration bindArgs(Configuration config, String... args) throws Exception;
-
- /**
- * Run a service. This called after {@link Service#start()}
- * @return the exit code
- * @throws Throwable any exception to report
- */
- int runService() throws Throwable ;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java
deleted file mode 100644
index 27813b7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.main;
-
-
-import org.apache.hadoop.yarn.exceptions.YarnException;
-
-/**
- * A service launch exception that includes an exit code;
- * when caught by the ServiceLauncher, it will convert that
- * into a process exit code.
- */
-public class ServiceLaunchException extends YarnException
- implements ExitCodeProvider, LauncherExitCodes {
-
- private final int exitCode;
-
- /**
- * Create an exception with the specific exit code
- * @param exitCode exit code
- * @param cause cause of the exception
- */
- public ServiceLaunchException(int exitCode, Throwable cause) {
- super(cause);
- this.exitCode = exitCode;
- }
-
- /**
- * Create an exception with the specific exit code and text
- * @param exitCode exit code
- * @param message message to use in exception
- */
- public ServiceLaunchException(int exitCode, String message) {
- super(message);
- this.exitCode = exitCode;
- }
-
- /**
- * Create an exception with the specific exit code, text and cause
- * @param exitCode exit code
- * @param message message to use in exception
- * @param cause cause of the exception
- */
- public ServiceLaunchException(int exitCode, String message, Throwable cause) {
- super(message, cause);
- this.exitCode = exitCode;
- }
-
- /**
- * Get the exit code
- * @return the exit code
- */
- @Override
- public int getExitCode() {
- return exitCode;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
deleted file mode 100644
index f192ec8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
+++ /dev/null
@@ -1,642 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.main;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.net.MalformedURLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * A class to launch any service by name.
- *
- * It's designed to be subclassed for custom entry points.
- *
- *
- * Workflow
- * <ol>
- * <li>An instance of the class is created</li>
- * <li>If it implements RunService, it is given the binding args off the CLI</li>
- * <li>Its service.init() and service.start() methods are called.</li>
- * <li>If it implements RunService, runService() is called and its return
- * code used as the exit code.</li>
- * <li>Otherwise: it waits for the service to stop, assuming in its start() method
- * it begins work</li>
- * <li>If an exception returned an exit code, that becomes the exit code of the
- * command.</li>
- * </ol>
- * Error and warning messages are logged to stderr. Why? If the classpath
- * is wrong & logger configurations not on it, then no error messages by
- * the started app will be seen and the caller is left trying to debug
- * using exit codes.
- *
- */
-@SuppressWarnings("UseOfSystemOutOrSystemErr")
-public class ServiceLauncher<S extends Service>
- implements LauncherExitCodes, IrqHandler.Interrupted,
- Thread.UncaughtExceptionHandler {
- private static final Logger LOG = LoggerFactory.getLogger(
- ServiceLauncher.class);
-
- protected static final int PRIORITY = 30;
-
- public static final String NAME = "ServiceLauncher";
-
- /**
- * Name of the "--conf" argument.
- */
- public static final String ARG_CONF = "--conf";
-
- public static final String USAGE_MESSAGE =
- "Usage: " + NAME + " classname [" + ARG_CONF +
- "<conf file>] <service arguments> | ";
- static final int SHUTDOWN_TIME_ON_INTERRUPT = 30 * 1000;
-
- private volatile S service;
- private int serviceExitCode;
- @SuppressWarnings("MismatchedQueryAndUpdateOfCollection")
- private final List<IrqHandler> interruptHandlers = new ArrayList<>(1);
- private Configuration configuration;
- private String serviceClassName;
- private static AtomicBoolean signalAlreadyReceived = new AtomicBoolean(false);
-
-
- /**
- * Create an instance of the launcher
- * @param serviceClassName classname of the service
- */
- public ServiceLauncher(String serviceClassName) {
- this.serviceClassName = serviceClassName;
- }
-
- /**
- * Get the service. Null until and unless
- * {@link #launchService(Configuration, String[], boolean)} has completed
- * @return the service
- */
- public S getService() {
- return service;
- }
-
- /**
- * Get the configuration constructed from the command line arguments
- * @return the configuration used to create the service
- */
- public Configuration getConfiguration() {
- return configuration;
- }
-
- /**
- * The exit code from a successful service execution
- * @return the exit code.
- */
- public int getServiceExitCode() {
- return serviceExitCode;
- }
-
- @Override
- public String toString() {
- return "ServiceLauncher for " + serviceClassName;
- }
-
- /**
- * Launch the service, by creating it, initing it, starting it and then
- * maybe running it. {@link RunService#bindArgs(Configuration, String...)} is invoked
- * on the service between creation and init.
- *
- * All exceptions that occur are propagated upwards.
- *
- * If the method returns a status code, it means that it got as far starting
- * the service, and if it implements {@link RunService}, that the
- * method {@link RunService#runService()} has completed.
- *
- * At this point, the service is returned by {@link #getService()}.
- *
- * @param conf configuration
- * @param processedArgs arguments after the configuration parameters
- * have been stripped out.
- * @param addProcessHooks should process failure handlers be added to
- * terminate this service on shutdown. Tests should set this to false.
- * @throws ClassNotFoundException classname not on the classpath
- * @throws IllegalAccessException not allowed at the class
- * @throws InstantiationException not allowed to instantiate it
- * @throws InterruptedException thread interrupted
- * @throws Throwable any other failure
- */
- public int launchService(Configuration conf,
- String[] processedArgs,
- boolean addProcessHooks)
- throws Throwable {
-
- instantiateService(conf);
-
- // add any process shutdown hooks
- if (addProcessHooks) {
- ServiceShutdownHook shutdownHook = new ServiceShutdownHook(service);
- ShutdownHookManager.get().addShutdownHook(shutdownHook, PRIORITY);
- }
- RunService runService = null;
-
- if (service instanceof RunService) {
- //if its a runService, pass in the conf and arguments before init)
- runService = (RunService) service;
- configuration = runService.bindArgs(configuration, processedArgs);
- Preconditions.checkNotNull(configuration,
- "null configuration returned by bindArgs()");
- }
-
- //some class constructors init; here this is picked up on.
- if (!service.isInState(Service.STATE.INITED)) {
- service.init(configuration);
- }
- service.start();
- int exitCode = EXIT_SUCCESS;
- if (runService != null) {
- //assume that runnable services are meant to run from here
- exitCode = runService.runService();
- LOG.debug("Service exited with exit code {}", exitCode);
-
- } else {
- //run the service until it stops or an interrupt happens on a different thread.
- LOG.debug("waiting for service threads to terminate");
- service.waitForServiceToStop(0);
- }
- //exit
- serviceExitCode = exitCode;
- return serviceExitCode;
- }
-
- /**
- * Instantiate the service defined in <code>serviceClassName</code>
- * . Sets the <code>configuration</code> field
- * to the configuration, and <code>service</code> to the service.
- *
- * @param conf configuration to use
- * @throws ClassNotFoundException classname not on the classpath
- * @throws IllegalAccessException not allowed at the class
- * @throws InstantiationException not allowed to instantiate it
- */
- @SuppressWarnings("unchecked")
- public Service instantiateService(Configuration conf)
- throws ClassNotFoundException, InstantiationException, IllegalAccessException,
- ExitUtil.ExitException, NoSuchMethodException, InvocationTargetException {
- Preconditions.checkArgument(conf != null, "null conf");
- configuration = conf;
-
- //Instantiate the class -this requires the service to have a public
- // zero-argument constructor
- Class<?> serviceClass =
- this.getClass().getClassLoader().loadClass(serviceClassName);
- Object instance = serviceClass.getConstructor().newInstance();
- if (!(instance instanceof Service)) {
- //not a service
- throw new ExitUtil.ExitException(EXIT_COMMAND_ARGUMENT_ERROR,
- "Not a Service class: " + serviceClassName);
- }
-
- service = (S) instance;
- return service;
- }
-
- /**
- * Register this class as the handler for the control-C interrupt.
- * Can be overridden for testing.
- */
- protected void registerInterruptHandler() {
- try {
- interruptHandlers.add(new IrqHandler(IrqHandler.CONTROL_C, this));
- interruptHandlers.add(new IrqHandler(IrqHandler.SIGTERM, this));
- } catch (IOException e) {
- error("Signal handler setup failed : {}" + e, e);
- }
- }
-
- /**
- * The service has been interrupted -try to shut down the service.
- * Give the service time to do this before the exit operation is called
- * @param interruptData the interrupted data.
- */
- @Override
- public void interrupted(IrqHandler.InterruptData interruptData) {
- String message = "Service interrupted by " + interruptData.toString();
- warn(message);
- if (!signalAlreadyReceived.compareAndSet(false, true)) {
- warn("Repeated interrupt: escalating to a JVM halt");
- // signal already received. On a second request to a hard JVM
- // halt and so bypass any blocking shutdown hooks.
- ExitUtil.halt(EXIT_INTERRUPTED, message);
- }
- int shutdownTimeMillis = SHUTDOWN_TIME_ON_INTERRUPT;
- //start an async shutdown thread with a timeout
- ServiceForcedShutdown forcedShutdown =
- new ServiceForcedShutdown(shutdownTimeMillis);
- Thread thread = new Thread(forcedShutdown);
- thread.setDaemon(true);
- thread.start();
- //wait for that thread to finish
- try {
- thread.join(shutdownTimeMillis);
- } catch (InterruptedException ignored) {
- //ignored
- }
- if (!forcedShutdown.isServiceStopped()) {
- warn("Service did not shut down in time");
- }
- exit(EXIT_INTERRUPTED, message);
- }
-
- /**
- * Uncaught exception handler.
- * If an error is raised: shutdown
- * The state of the system is unknown at this point -attempting
- * a clean shutdown is dangerous. Instead: exit
- * @param thread thread that failed
- * @param exception exception
- */
- @Override
- public void uncaughtException(Thread thread, Throwable exception) {
- if (ShutdownHookManager.get().isShutdownInProgress()) {
- LOG.error("Thread {} threw an error during shutdown: {}.",
- thread.toString(),
- exception,
- exception);
- } else if (exception instanceof Error) {
- try {
- LOG.error("Thread {} threw an error: {}. Shutting down",
- thread.toString(),
- exception,
- exception);
- } catch (Throwable err) {
- // We don't want to not exit because of an issue with logging
- }
- if (exception instanceof OutOfMemoryError) {
- // After catching an OOM java says it is undefined behavior, so don't
- // even try to clean up or we can get stuck on shutdown.
- try {
- System.err.println("Halting due to Out Of Memory Error...");
- } catch (Throwable err) {
- // Again we don't want to exit because of logging issues.
- }
- ExitUtil.halt(EXIT_EXCEPTION_THROWN);
- } else {
- // error other than OutOfMemory
- exit(convertToExitException(exception));
- }
- } else {
- // simple exception in a thread. There's a policy decision here:
- // terminate the service vs. keep going after a thread has failed
- LOG.error("Thread {} threw an exception: {}",
- thread.toString(),
- exception,
- exception);
- }
- }
-
- /**
- * Print a warning: currently this goes to stderr
- * @param text
- */
- protected void warn(String text) {
- System.err.println(text);
- }
-
- /**
- * Report an error. The message is printed to stderr; the exception
- * is logged via the current logger.
- * @param message message for the user
- * @param thrown the exception thrown
- */
- protected void error(String message, Throwable thrown) {
- String text = "Exception: " + message;
- warn(text);
- LOG.error(text, thrown);
- }
-
- /**
- * Exit the code.
- * This is method can be overridden for testing, throwing an
- * exception instead. Any subclassed method MUST raise an
- * <code>ExitUtil.ExitException</code> instance.
- * The service launcher code assumes that after this method is invoked,
- * no other code in the same method is called.
- * @param exitCode code to exit
- */
- protected void exit(int exitCode, String message) {
- ExitUtil.terminate(exitCode, message);
- }
-
- /**
- * Exit off an exception. This can be subclassed for testing
- * @param ee exit exception
- */
- protected void exit(ExitUtil.ExitException ee) {
- ExitUtil.terminate(ee.status, ee);
- }
-
- /**
- * Get the service name via {@link Service#getName()}.
- * If the service is not instantiated, the classname is returned instead.
- * @return the service name
- */
- public String getServiceName() {
- Service s = service;
- String name = null;
- if (s != null) {
- try {
- name = s.getName();
- } catch (Exception ignored) {
- // ignored
- }
- }
- if (name != null) {
- return "service " + name;
- } else {
- return "service classname " + serviceClassName;
- }
- }
-
- /**
- * Parse the command line, building a configuration from it, then
- * launch the service and wait for it to finish. finally, exit
- * passing the status code to the #exit(int) method.
- * @param args arguments to the service. arg[0] is
- * assumed to be the service classname and is automatically
- */
- public void launchServiceAndExit(List<String> args) {
-
- registerInterruptHandler();
- //Currently the config just the default
- Configuration conf = new Configuration();
- String[] processedArgs = extractConfigurationArgs(conf, args);
- ExitUtil.ExitException ee = launchServiceRobustly(conf, processedArgs);
- System.out.flush();
- System.err.flush();
- exit(ee);
- }
-
- /**
- * Extract the configuration arguments and apply them to the configuration,
- * building an array of processed arguments to hand down to the service.
- *
- * @param conf configuration to update
- * @param args main arguments. args[0] is assumed to be the service
- * classname and is skipped
- * @return the processed list.
- */
- public static String[] extractConfigurationArgs(Configuration conf,
- List<String> args) {
-
- //convert args to a list
- int argCount = args.size();
- if (argCount <= 1 ) {
- return new String[0];
- }
- List<String> argsList = new ArrayList<String>(argCount);
- ListIterator<String> arguments = args.listIterator();
- //skip that first entry
- arguments.next();
- while (arguments.hasNext()) {
- String arg = arguments.next();
- if (arg.equals(ARG_CONF)) {
- //the argument is a --conf file tuple: extract the path and load
- //it in as a configuration resource.
-
- //increment the loop iterator
- if (!arguments.hasNext()) {
- //overshot the end of the file
- exitWithMessage(EXIT_COMMAND_ARGUMENT_ERROR,
- ARG_CONF + ": missing configuration file after ");
- }
- File file = new File(arguments.next());
- if (!file.exists()) {
- exitWithMessage(EXIT_COMMAND_ARGUMENT_ERROR,
- ARG_CONF + ": configuration file not found: " + file);
- }
- try {
- conf.addResource(file.toURI().toURL());
- } catch (MalformedURLException e) {
- LOG.debug("File {} cannot be converted to URL", file, e);
- exitWithMessage(EXIT_COMMAND_ARGUMENT_ERROR,
- ARG_CONF + ": configuration file path invalid: " + file);
- }
- } else {
- argsList.add(arg);
- }
- }
- String[] processedArgs = new String[argsList.size()];
- argsList.toArray(processedArgs);
- return processedArgs;
- }
-
- /**
- * Launch a service catching all exceptions and downgrading them to exit codes
- * after logging.
- * @param conf configuration to use
- * @param processedArgs command line after the launcher-specific arguments have
- * been stripped out
- * @return an exit exception, which will have a status code of 0 if it worked
- */
- public ExitUtil.ExitException launchServiceRobustly(Configuration conf,
- String[] processedArgs) {
- ExitUtil.ExitException exitException;
- try {
- int exitCode = launchService(conf, processedArgs, true);
- if (service != null) {
- Throwable failure = service.getFailureCause();
- if (failure != null) {
- //the service exited with a failure.
- //check what state it is in
- Service.STATE failureState = service.getFailureState();
- if (failureState == Service.STATE.STOPPED) {
- //the failure occurred during shutdown, not important enough to bother
- //the user as it may just scare them
- LOG.debug("Failure during shutdown:{} ", failure, failure);
- } else {
- //throw it for the catch handlers to deal with
- throw failure;
- }
- }
- }
- exitException = new ExitUtil.ExitException(exitCode,
- "In " + serviceClassName);
- // either the service succeeded, or an error raised during shutdown,
- // which we don't worry that much about
- } catch (ExitUtil.ExitException ee) {
- exitException = ee;
- } catch (Throwable thrown) {
- exitException = convertToExitException(thrown);
- }
- return exitException;
- }
-
- /**
- * Convert the exception to one that can be handed off to ExitUtils;
- * if it is of the write type it is passed throw as is. If not, a
- * new exception with the exit code {@link #EXIT_EXCEPTION_THROWN}
- * is created, with the argument <code>thrown</code> as the inner cause
- * @param thrown the exception thrown
- * @return an exception to terminate the process with
- */
- protected ExitUtil.ExitException convertToExitException(Throwable thrown) {
- ExitUtil.ExitException exitException;
- int exitCode;
- String message = thrown.getMessage();
- if (message == null) {
- message = thrown.toString();
- }
- if (thrown instanceof ExitCodeProvider) {
- exitCode = ((ExitCodeProvider) thrown).getExitCode();
- if (LOG.isDebugEnabled()) {
- LOG.debug("While running {}: {}", getServiceName(), message, thrown);
- }
- LOG.error(message);
- } else {
- // not any of the service launcher exceptions -assume something worse
- error(message, thrown);
- exitCode = EXIT_EXCEPTION_THROWN;
- }
- exitException = new ExitUtil.ExitException(exitCode, message);
- exitException.initCause(thrown);
- return exitException;
- }
-
-
- /**
- * Build a log message for starting up and shutting down.
- * This was grabbed from the ToolRunner code.
- * @param classname the class of the server
- * @param args arguments
- */
- public static String startupShutdownMessage(String classname,
- List<String> args) {
- final String hostname = NetUtils.getHostname();
-
- return toStartupShutdownString("STARTUP_MSG: ", new String[]{
- "Starting " + classname,
- " host = " + hostname,
- " args = " + args,
- " version = " + VersionInfo.getVersion(),
- " classpath = " + System.getProperty("java.class.path"),
- " build = " + VersionInfo.getUrl() + " -r "
- + VersionInfo.getRevision()
- + "; compiled by '" + VersionInfo.getUser()
- + "' on " + VersionInfo.getDate(),
- " java = " + System.getProperty("java.version")
- });
- }
-
- /**
- * Exit with a printed message
- * @param status status code
- * @param message message
- */
- private static void exitWithMessage(int status, String message) {
- System.err.println(message);
- ExitUtil.terminate(status);
- }
-
- private static String toStartupShutdownString(String prefix, String[] msg) {
- StringBuilder b = new StringBuilder(prefix);
- b.append("\n/************************************************************");
- for (String s : msg) {
- b.append("\n").append(prefix).append(s);
- }
- b.append("\n************************************************************/");
- return b.toString();
- }
-
- /**
- * forced shutdown runnable.
- */
- protected class ServiceForcedShutdown implements Runnable {
-
- private final int shutdownTimeMillis;
- private boolean serviceStopped;
-
- public ServiceForcedShutdown(int shutdownTimeoutMillis) {
- this.shutdownTimeMillis = shutdownTimeoutMillis;
- }
-
- @Override
- public void run() {
- if (service != null) {
- service.stop();
- serviceStopped = service.waitForServiceToStop(shutdownTimeMillis);
- } else {
- serviceStopped = true;
- }
- }
-
- private boolean isServiceStopped() {
- return serviceStopped;
- }
- }
-
- /**
- * The real main function, which takes the arguments as a list
- * arg 0 must be the service classname
- * @param argsList the list of arguments
- */
- public static void serviceMain(List<String> argsList) {
- if (argsList.isEmpty()) {
- exitWithMessage(EXIT_USAGE, USAGE_MESSAGE);
- } else {
- String serviceClassName = argsList.get(0);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(startupShutdownMessage(serviceClassName, argsList));
- StringBuilder builder = new StringBuilder();
- for (String arg : argsList) {
- builder.append('"').append(arg).append("\" ");
- }
- LOG.debug(builder.toString());
- }
- Thread.setDefaultUncaughtExceptionHandler(
- new YarnUncaughtExceptionHandler());
-
- ServiceLauncher serviceLauncher = new ServiceLauncher<>(serviceClassName);
- serviceLauncher.launchServiceAndExit(argsList);
- }
- }
-
- /**
- * This is the main entry point for the service launcher.
- * @param args command line arguments.
- */
- public static void main(String[] args) {
- List<String> argsList = Arrays.asList(args);
- serviceMain(argsList);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceShutdownHook.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceShutdownHook.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceShutdownHook.java
deleted file mode 100644
index de55789..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceShutdownHook.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.main;
-
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.util.ShutdownHookManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.ref.WeakReference;
-
-/**
- * JVM Shutdown hook for Service which will stop the
- * Service gracefully in case of JVM shutdown.
- * This hook uses a weak reference to the service, so
- * does not cause services to be retained after they have
- * been stopped and deferenced elsewhere.
- */
-public class ServiceShutdownHook implements Runnable {
- private static final Logger LOG = LoggerFactory.getLogger(
- ServiceShutdownHook.class);
-
- private final WeakReference<Service> serviceRef;
- private Runnable hook;
-
- public ServiceShutdownHook(Service service) {
- serviceRef = new WeakReference<>(service);
- }
-
- public void register(int priority) {
- unregister();
- hook = this;
- ShutdownHookManager.get().addShutdownHook(hook, priority);
- }
-
- public synchronized void unregister() {
- if (hook != null) {
- try {
- ShutdownHookManager.get().removeShutdownHook(hook);
- } catch (IllegalStateException e) {
- LOG.info("Failed to unregister shutdown hook: {}", e, e);
- }
- hook = null;
- }
- }
-
- @Override
- public void run() {
- Service service;
- synchronized (this) {
- service = serviceRef.get();
- serviceRef.clear();
- }
- if (service == null) {
- return;
- }
- try {
- // Stop the Service
- service.stop();
- } catch (Throwable t) {
- LOG.info("Error stopping {}", service.getName(), t);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/ApplicationReportSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/ApplicationReportSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/ApplicationReportSerDeser.java
deleted file mode 100644
index a8c72ce..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/ApplicationReportSerDeser.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.persist;
-
-import org.apache.slider.core.launch.SerializedApplicationReport;
-import org.codehaus.jackson.JsonGenerationException;
-import org.codehaus.jackson.JsonParseException;
-import org.codehaus.jackson.map.JsonMappingException;
-
-import java.io.IOException;
-
-/**
- * Persistence of {@link SerializedApplicationReport}
- *
- */
-public class ApplicationReportSerDeser
- extends JsonSerDeser<SerializedApplicationReport> {
- public ApplicationReportSerDeser() {
- super(SerializedApplicationReport.class);
- }
-
-
- private static final ApplicationReportSerDeser
- staticinstance = new ApplicationReportSerDeser();
-
- /**
- * Convert an instance to a JSON string -sync access to a shared ser/deser
- * object instance
- * @param instance object to convert
- * @return a JSON string description
- * @throws JsonParseException parse problems
- * @throws JsonMappingException O/J mapping problems
- */
- public static String toString(SerializedApplicationReport instance)
- throws IOException, JsonGenerationException, JsonMappingException {
- synchronized (staticinstance) {
- return staticinstance.toJson(instance);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/Filenames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/Filenames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/Filenames.java
deleted file mode 100644
index 06ecc51..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/Filenames.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.persist;
-
-public interface Filenames {
-
- String RESOURCES = "resources.json";
- String APPCONF = "app_config.json";
- String INTERNAL = "internal.json";
- String WRITELOCK = "writelock";
- String READLOCK = "readlock";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
deleted file mode 100644
index 8fe2549..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.persist;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.codehaus.jackson.JsonGenerationException;
-import org.codehaus.jackson.JsonParseException;
-import org.codehaus.jackson.map.DeserializationConfig;
-import org.codehaus.jackson.map.JsonMappingException;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.PropertyNamingStrategy;
-import org.codehaus.jackson.map.SerializationConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-/**
- * Support for marshalling objects to and from JSON.
- * This class is NOT thread safe; it constructs an object mapper
- * as an instance field.
- * @param <T>
- */
-public class JsonSerDeser<T> {
-
- private static final Logger log = LoggerFactory.getLogger(JsonSerDeser.class);
- private static final String UTF_8 = "UTF-8";
-
- private final Class<T> classType;
- private final ObjectMapper mapper;
-
- /**
- * Create an instance bound to a specific type
- * @param classType class type
- */
- public JsonSerDeser(Class<T> classType) {
- this.classType = classType;
- this.mapper = new ObjectMapper();
- mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- public JsonSerDeser(Class<T> classType, PropertyNamingStrategy namingStrategy) {
- this(classType);
- mapper.setPropertyNamingStrategy(namingStrategy);
- }
-
- /**
- * Convert from JSON
- * @param json input
- * @return the parsed JSON
- * @throws IOException IO
- * @throws JsonMappingException failure to map from the JSON to this class
- */
- public T fromJson(String json)
- throws IOException, JsonParseException, JsonMappingException {
- try {
- return mapper.readValue(json, classType);
- } catch (IOException e) {
- log.error("Exception while parsing json : " + e + "\n" + json, e);
- throw e;
- }
- }
-
- /**
- * Convert from a JSON file
- * @param jsonFile input file
- * @return the parsed JSON
- * @throws IOException IO problems
- * @throws JsonMappingException failure to map from the JSON to this class
- */
- public T fromFile(File jsonFile)
- throws IOException, JsonParseException, JsonMappingException {
- File absoluteFile = jsonFile.getAbsoluteFile();
- try {
- return mapper.readValue(absoluteFile, classType);
- } catch (IOException e) {
- log.error("Exception while parsing json file {}", absoluteFile, e);
- throw e;
- }
- }
-
- /**
- * Convert from a JSON file
- * @param resource input file
- * @return the parsed JSON
- * @throws IOException IO problems
- * @throws JsonMappingException failure to map from the JSON to this class
- */
- public T fromResource(String resource)
- throws IOException, JsonParseException, JsonMappingException {
- try(InputStream resStream = this.getClass().getResourceAsStream(resource)) {
- if (resStream == null) {
- throw new FileNotFoundException(resource);
- }
- return (T) (mapper.readValue(resStream, classType));
- } catch (IOException e) {
- log.error("Exception while parsing json resource {}", resource, e);
- throw e;
- }
- }
-
- /**
- * Convert from an input stream, closing the stream afterwards.
- * @param stream
- * @return the parsed JSON
- * @throws IOException IO problems
- */
- public T fromStream(InputStream stream) throws IOException {
- try {
- return (T) (mapper.readValue(stream, classType));
- } catch (IOException e) {
- log.error("Exception while parsing json input stream", e);
- throw e;
- } finally {
- IOUtils.closeStream(stream);
- }
- }
-
- /**
- * clone by converting to JSON and back again.
- * This is much less efficient than any Java clone process.
- * @param instance instance to duplicate
- * @return a new instance
- * @throws IOException problems.
- */
- public T fromInstance(T instance) throws IOException {
- return fromJson(toJson(instance));
- }
-
- /**
- * Deserialize from a byte array
- * @param b
- * @return the deserialized value
- * @throws IOException parse problems
- */
- public T fromBytes(byte[] b) throws IOException {
- String json = new String(b, 0, b.length, UTF_8);
- return fromJson(json);
- }
-
- /**
- * Load from a Hadoop filesystem
- * @param fs filesystem
- * @param path path
- * @return a loaded CD
- * @throws IOException IO problems
- * @throws JsonParseException parse problems
- * @throws JsonMappingException O/J mapping problems
- */
- public T load(FileSystem fs, Path path)
- throws IOException, JsonParseException, JsonMappingException {
- FileStatus status = fs.getFileStatus(path);
- long len = status.getLen();
- byte[] b = new byte[(int) len];
- FSDataInputStream dataInputStream = fs.open(path);
- int count = dataInputStream.read(b);
- if (count != len) {
- throw new EOFException("Read of " + path +" finished prematurely");
- }
- return fromBytes(b);
- }
-
-
- /**
- * Save to a hadoop filesystem
- * @param fs filesystem
- * @param path path
- * @param instance instance to save
- * @param overwrite should any existing file be overwritten
- * @throws IOException IO exception
- */
- public void save(FileSystem fs, Path path, T instance,
- boolean overwrite) throws
- IOException {
- FSDataOutputStream dataOutputStream = fs.create(path, overwrite);
- writeJsonAsBytes(instance, dataOutputStream);
- }
-
- /**
- * Save an instance to a file
- * @param instance instance to save
- * @param file file
- * @throws IOException
- */
- public void save(T instance, File file) throws
- IOException {
- writeJsonAsBytes(instance, new FileOutputStream(file.getAbsoluteFile()));
- }
-
- /**
- * Write the json as bytes -then close the file
- * @param dataOutputStream an outout stream that will always be closed
- * @throws IOException on any failure
- */
- private void writeJsonAsBytes(T instance,
- OutputStream dataOutputStream) throws IOException {
- try {
- String json = toJson(instance);
- byte[] b = json.getBytes(UTF_8);
- dataOutputStream.write(b);
- dataOutputStream.flush();
- dataOutputStream.close();
- } finally {
- IOUtils.closeStream(dataOutputStream);
- }
- }
-
- /**
- * Convert an object to a JSON string
- * @param instance instance to convert
- * @return a JSON string description
- * @throws JsonParseException parse problems
- * @throws JsonMappingException O/J mapping problems
- */
- public String toJson(T instance) throws IOException,
- JsonGenerationException,
- JsonMappingException {
- mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
- return mapper.writeValueAsString(instance);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/LockHeldAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/LockHeldAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/LockHeldAction.java
deleted file mode 100644
index 6659687..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/LockHeldAction.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.persist;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-import java.io.IOException;
-
-/**
- * Optional action to add while the lock is held; this is needed to execute
- * some other persistent operations within the scope at the same lock
- * without inserting too much code into the persister
- */
-public interface LockHeldAction {
-
- /**
- * Execute the action
- * @throws IOException on any failure
- */
- public void execute() throws IOException, SliderException;
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[68/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
new file mode 100644
index 0000000..c64b1b5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class Error {
+
+ private Integer code = null;
+ private String message = null;
+ private String fields = null;
+
+ /**
+ **/
+ public Error code(Integer code) {
+ this.code = code;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "")
+ @JsonProperty("code")
+ public Integer getCode() {
+ return code;
+ }
+
+ public void setCode(Integer code) {
+ this.code = code;
+ }
+
+ /**
+ **/
+ public Error message(String message) {
+ this.message = message;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "")
+ @JsonProperty("message")
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ /**
+ **/
+ public Error fields(String fields) {
+ this.fields = fields;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "")
+ @JsonProperty("fields")
+ public String getFields() {
+ return fields;
+ }
+
+ public void setFields(String fields) {
+ this.fields = fields;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Error error = (Error) o;
+ return Objects.equals(this.code, error.code)
+ && Objects.equals(this.message, error.message)
+ && Objects.equals(this.fields, error.fields);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(code, message, fields);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Error {\n");
+
+ sb.append(" code: ").append(toIndentedString(code)).append("\n");
+ sb.append(" message: ").append(toIndentedString(message)).append("\n");
+ sb.append(" fields: ").append(toIndentedString(fields)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
new file mode 100644
index 0000000..6f6fe6f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Placement policy of an instance of an service. This feature is in the
+ * works in YARN-4902.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "Placement policy of an instance of an service. This feature is in the works in YARN-4902.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class PlacementPolicy implements Serializable {
+ private static final long serialVersionUID = 4341110649551172231L;
+
+ private String label = null;
+
+ /**
+ * Assigns a service to a named partition of the cluster where the service
+ * desires to run (optional). If not specified all services are submitted to
+ * a default label of the service owner. One or more labels can be setup for
+ * each service owner account with required constraints like no-preemption,
+ * sla-99999, preemption-ok, etc.
+ **/
+ public PlacementPolicy label(String label) {
+ this.label = label;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Assigns a service to a named partition of the cluster where the service desires to run (optional). If not specified all services are submitted to a default label of the service owner. One or more labels can be setup for each service owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.")
+ @JsonProperty("label")
+ public String getLabel() {
+ return label;
+ }
+
+ public void setLabel(String label) {
+ this.label = label;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PlacementPolicy placementPolicy = (PlacementPolicy) o;
+ return Objects.equals(this.label, placementPolicy.label);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(label);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class PlacementPolicy {\n");
+
+ sb.append(" label: ").append(toIndentedString(label)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
new file mode 100644
index 0000000..1a25a4c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * A custom command or a pluggable helper container to determine the readiness
+ * of a container of a component. Readiness for every service is different.
+ * Hence the need for a simple interface, with scope to support advanced
+ * usecases.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class ReadinessCheck implements Serializable {
+ private static final long serialVersionUID = -3836839816887186801L;
+
+ public enum TypeEnum {
+ HTTP("HTTP"),
+ PORT("PORT");
+
+ private String value;
+
+ TypeEnum(String value) {
+ this.value = value;
+ }
+
+ @Override
+ @JsonValue
+ public String toString() {
+ return value;
+ }
+ }
+
+ private TypeEnum type = null;
+ private Map<String, String> props = new HashMap<String, String>();
+ private Artifact artifact = null;
+
+ /**
+ * E.g. HTTP (YARN will perform a simple REST call at a regular interval and
+ * expect a 204 No content).
+ **/
+ public ReadinessCheck type(TypeEnum type) {
+ this.type = type;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).")
+ @JsonProperty("type")
+ public TypeEnum getType() {
+ return type;
+ }
+
+ public void setType(TypeEnum type) {
+ this.type = type;
+ }
+
+ public ReadinessCheck props(Map<String, String> props) {
+ this.props = props;
+ return this;
+ }
+
+ public ReadinessCheck putPropsItem(String key, String propsItem) {
+ this.props.put(key, propsItem);
+ return this;
+ }
+
+ /**
+ * A blob of key value pairs that will be used to configure the check.
+ * @return props
+ **/
+ @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be used to configure the check.")
+ public Map<String, String> getProps() {
+ return props;
+ }
+
+ public void setProps(Map<String, String> props) {
+ this.props = props;
+ }
+
+ /**
+ * Artifact of the pluggable readiness check helper container (optional). If
+ * specified, this helper container typically hosts the http uri and
+ * encapsulates the complex scripts required to perform actual container
+ * readiness check. At the end it is expected to respond a 204 No content just
+ * like the simplified use case. This pluggable framework benefits service
+ * owners who can run services without any packaging modifications. Note,
+ * artifacts of type docker only is supported for now.
+ **/
+ public ReadinessCheck artifact(Artifact artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now.")
+ @JsonProperty("artifact")
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void setArtifact(Artifact artifact) {
+ this.artifact = artifact;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ReadinessCheck readinessCheck = (ReadinessCheck) o;
+ return Objects.equals(this.type, readinessCheck.type) &&
+ Objects.equals(this.props, readinessCheck.props) &&
+ Objects.equals(this.artifact, readinessCheck.artifact);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(type, props, artifact);
+ }
+
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class ReadinessCheck {\n");
+
+ sb.append(" type: ").append(toIndentedString(type)).append("\n");
+ sb.append(" props: ").append(toIndentedString(props)).append("\n");
+ sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
new file mode 100644
index 0000000..cec9de9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Resource determines the amount of resources (vcores, memory, network, etc.)
+ * usable by a container. This field determines the resource to be applied for
+ * all the containers of a component or service. The resource specified at
+ * the service (or global) level can be overriden at the component level. Only one
+ * of profile OR cpu & memory are expected. It raises a validation
+ * exception otherwise.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class Resource extends BaseResource implements Cloneable {
+ private static final long serialVersionUID = -6431667797380250037L;
+
+ private String profile = null;
+ private Integer cpus = 1;
+ private String memory = null;
+
+ /**
+ * Each resource profile has a unique id which is associated with a
+ * cluster-level predefined memory, cpus, etc.
+ **/
+ public Resource profile(String profile) {
+ this.profile = profile;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.")
+ @JsonProperty("profile")
+ public String getProfile() {
+ return profile;
+ }
+
+ public void setProfile(String profile) {
+ this.profile = profile;
+ }
+
+ /**
+ * Amount of vcores allocated to each container (optional but overrides cpus
+ * in profile if specified).
+ **/
+ public Resource cpus(Integer cpus) {
+ this.cpus = cpus;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).")
+ @JsonProperty("cpus")
+ public Integer getCpus() {
+ return cpus;
+ }
+
+ public void setCpus(Integer cpus) {
+ this.cpus = cpus;
+ }
+
+ /**
+ * Amount of memory allocated to each container (optional but overrides memory
+ * in profile if specified). Currently accepts only an integer value and
+ * default unit is in MB.
+ **/
+ public Resource memory(String memory) {
+ this.memory = memory;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.")
+ @JsonProperty("memory")
+ public String getMemory() {
+ return memory;
+ }
+
+ public void setMemory(String memory) {
+ this.memory = memory;
+ }
+
+ public long getMemoryMB() {
+ if (this.memory == null) {
+ return 0;
+ }
+ return Long.parseLong(memory);
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Resource resource = (Resource) o;
+ return Objects.equals(this.profile, resource.profile)
+ && Objects.equals(this.cpus, resource.cpus)
+ && Objects.equals(this.memory, resource.memory);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(profile, cpus, memory);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Resource {\n");
+
+ sb.append(" profile: ").append(toIndentedString(profile)).append("\n");
+ sb.append(" cpus: ").append(toIndentedString(cpus)).append("\n");
+ sb.append(" memory: ").append(toIndentedString(memory)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ return super.clone();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
new file mode 100644
index 0000000..f3bbaa0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * An Service resource has the following attributes.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "An Service resource has the following attributes.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({ "name", "state", "resource", "number_of_containers",
+ "lifetime", "containers" })
+public class Service extends BaseResource {
+ private static final long serialVersionUID = -4491694636566094885L;
+
+ private String name = null;
+ private String id = null;
+ private Artifact artifact = null;
+ private Resource resource = null;
+ private String launchCommand = null;
+ private Date launchTime = null;
+ private Long numberOfContainers = null;
+ private Long numberOfRunningContainers = null;
+ private Long lifetime = null;
+ private PlacementPolicy placementPolicy = null;
+ private List<Component> components = new ArrayList<>();
+ private Configuration configuration = new Configuration();
+ private List<Container> containers = new ArrayList<>();
+ private ServiceState state = null;
+ private Map<String, String> quicklinks = new HashMap<>();
+ private String queue = null;
+
+ /**
+ * A unique service name.
+ **/
+ public Service name(String name) {
+ this.name = name;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", required = true, value = "A unique service name.")
+ @JsonProperty("name")
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * A unique service id.
+ **/
+ public Service id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A unique service id.")
+ @JsonProperty("id")
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
+ * Artifact of single-component services. Mandatory if components
+ * attribute is not specified.
+ **/
+ public Service artifact(Artifact artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact of single-component services. Mandatory if components attribute is not specified.")
+ @JsonProperty("artifact")
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void setArtifact(Artifact artifact) {
+ this.artifact = artifact;
+ }
+
+ /**
+ * Resource of single-component services or the global default for
+ * multi-component services. Mandatory if it is a single-component
+ * service and if cpus and memory are not specified at the Service
+ * level.
+ **/
+ public Service resource(Resource resource) {
+ this.resource = resource;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Resource of single-component services or the global default for multi-component services. Mandatory if it is a single-component service and if cpus and memory are not specified at the Service level.")
+ @JsonProperty("resource")
+ public Resource getResource() {
+ return resource;
+ }
+
+ public void setResource(Resource resource) {
+ this.resource = resource;
+ }
+
+ /**
+ * The custom launch command of an service component (optional). If not
+ * specified for services with docker images say, it will default to the
+ * default start command of the image. If there is a single component in this
+ * service, you can specify this without the need to have a 'components'
+ * section.
+ **/
+ public Service launchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The custom launch command of an service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section.")
+ @JsonProperty("launch_command")
+ public String getLaunchCommand() {
+ return launchCommand;
+ }
+
+ @XmlElement(name = "launch_command")
+ public void setLaunchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ }
+
+ /**
+ * The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.
+ **/
+ public Service launchTime(Date launchTime) {
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.")
+ @JsonProperty("launch_time")
+ public Date getLaunchTime() {
+ return launchTime == null ? null : (Date) launchTime.clone();
+ }
+
+ @XmlElement(name = "launch_time")
+ public void setLaunchTime(Date launchTime) {
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
+ }
+
+ /**
+ * Number of containers for each component in the service. Each
+ * component can further override this service-level global default.
+ **/
+ public Service numberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Number of containers for each component in the service. Each component can further override this service-level global default.")
+ @JsonProperty("number_of_containers")
+ public Long getNumberOfContainers() {
+ return numberOfContainers;
+ }
+
+ @XmlElement(name = "number_of_containers")
+ public void setNumberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ }
+
+ /**
+ * In get response this provides the total number of running containers for
+ * this service (across all components) at the time of request. Note, a
+ * subsequent request can return a different number as and when more
+ * containers get allocated until it reaches the total number of containers or
+ * if a flex request has been made between the two requests.
+ **/
+ public Service numberOfRunningContainers(Long numberOfRunningContainers) {
+ this.numberOfRunningContainers = numberOfRunningContainers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.")
+ @JsonProperty("number_of_running_containers")
+ public Long getNumberOfRunningContainers() {
+ return numberOfRunningContainers;
+ }
+
+ @XmlElement(name = "number_of_running_containers")
+ public void setNumberOfRunningContainers(Long numberOfRunningContainers) {
+ this.numberOfRunningContainers = numberOfRunningContainers;
+ }
+
+ /**
+ * Life time (in seconds) of the service from the time it reaches the
+ * RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For
+ * unlimited lifetime do not set a lifetime value.
+ **/
+ public Service lifetime(Long lifetime) {
+ this.lifetime = lifetime;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Life time (in seconds) of the service from the time it reaches the RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.")
+ @JsonProperty("lifetime")
+ public Long getLifetime() {
+ return lifetime;
+ }
+
+ public void setLifetime(Long lifetime) {
+ this.lifetime = lifetime;
+ }
+
+ /**
+ * Advanced scheduling and placement policies (optional). If not specified, it
+ * defaults to the default placement policy of the service owner. The design of
+ * placement policies are in the works. It is not very clear at this point,
+ * how policies in conjunction with labels be exposed to service owners.
+ * This is a placeholder for now. The advanced structure of this attribute
+ * will be determined by YARN-4902.
+ **/
+ public Service placementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.")
+ @JsonProperty("placement_policy")
+ public PlacementPolicy getPlacementPolicy() {
+ return placementPolicy;
+ }
+
+ @XmlElement(name = "placement_policy")
+ public void setPlacementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ }
+
+ /**
+ * Components of an service.
+ **/
+ public Service components(List<Component> components) {
+ this.components = components;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Components of an service.")
+ @JsonProperty("components")
+ public List<Component> getComponents() {
+ return components;
+ }
+
+ public void setComponents(List<Component> components) {
+ this.components = components;
+ }
+
+ public void addComponent(Component component) {
+ components.add(component);
+ }
+
+ public Component getComponent(String name) {
+ for (Component component : components) {
+ if (component.getName().equals(name)) {
+ return component;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Config properties of an service. Configurations provided at the
+ * service/global level are available to all the components. Specific
+ * properties can be overridden at the component level.
+ **/
+ public Service configuration(Configuration configuration) {
+ this.configuration = configuration;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Config properties of an service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.")
+ @JsonProperty("configuration")
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public void setConfiguration(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ /**
+ * Containers of a started service. Specifying a value for this attribute
+ * for the POST payload raises a validation error. This blob is available only
+ * in the GET response of a started service.
+ **/
+ public Service containers(List<Container> containers) {
+ this.containers = containers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.")
+ @JsonProperty("containers")
+ public List<Container> getContainers() {
+ return containers;
+ }
+
+ public void setContainers(List<Container> containers) {
+ this.containers = containers;
+ }
+
+ public void addContainer(Container container) {
+ this.containers.add(container);
+ }
+
+ /**
+ * State of the service. Specifying a value for this attribute for the
+ * POST payload raises a validation error. This attribute is available only in
+ * the GET response of a started service.
+ **/
+ public Service state(ServiceState state) {
+ this.state = state;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "State of the service. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started service.")
+ @JsonProperty("state")
+ public ServiceState getState() {
+ return state;
+ }
+
+ public void setState(ServiceState state) {
+ this.state = state;
+ }
+
+ /**
+ * A blob of key-value pairs of quicklinks to be exported for an service.
+ **/
+ public Service quicklinks(Map<String, String> quicklinks) {
+ this.quicklinks = quicklinks;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key-value pairs of quicklinks to be exported for an service.")
+ @JsonProperty("quicklinks")
+ public Map<String, String> getQuicklinks() {
+ return quicklinks;
+ }
+
+ public void setQuicklinks(Map<String, String> quicklinks) {
+ this.quicklinks = quicklinks;
+ }
+
+ /**
+ * The YARN queue that this service should be submitted to.
+ **/
+ public Service queue(String queue) {
+ this.queue = queue;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The YARN queue that this service should be submitted to.")
+ @JsonProperty("queue")
+ public String getQueue() {
+ return queue;
+ }
+
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Service service = (Service) o;
+ return Objects.equals(this.name, service.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Service {\n");
+
+ sb.append(" name: ").append(toIndentedString(name)).append("\n");
+ sb.append(" id: ").append(toIndentedString(id)).append("\n");
+ sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
+ sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
+ sb.append(" launchCommand: ").append(toIndentedString(launchCommand))
+ .append("\n");
+ sb.append(" launchTime: ").append(toIndentedString(launchTime))
+ .append("\n");
+ sb.append(" numberOfContainers: ")
+ .append(toIndentedString(numberOfContainers)).append("\n");
+ sb.append(" numberOfRunningContainers: ")
+ .append(toIndentedString(numberOfRunningContainers)).append("\n");
+ sb.append(" lifetime: ").append(toIndentedString(lifetime)).append("\n");
+ sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy))
+ .append("\n");
+ sb.append(" components: ").append(toIndentedString(components))
+ .append("\n");
+ sb.append(" configuration: ").append(toIndentedString(configuration))
+ .append("\n");
+ sb.append(" containers: ").append(toIndentedString(containers))
+ .append("\n");
+ sb.append(" state: ").append(toIndentedString(state)).append("\n");
+ sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
+ .append("\n");
+ sb.append(" queue: ").append(toIndentedString(queue)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
new file mode 100644
index 0000000..a4509bd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * The current state of an service.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "The current state of an service.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public enum ServiceState {
+ ACCEPTED, STARTED, READY, STOPPED, FAILED;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceStatus.java
new file mode 100644
index 0000000..2cee23c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceStatus.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * The current status of a submitted service, returned as a response to the
+ * GET API.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "The current status of a submitted service, returned as a response to the GET API.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ServiceStatus extends BaseResource {
+ private static final long serialVersionUID = -3469885905347851034L;
+
+ private String diagnostics = null;
+ private ServiceState state = null;
+ private Integer code = null;
+
+ /**
+ * Diagnostic information (if any) for the reason of the current state of the
+ * service. It typically has a non-null value, if the service is in a
+ * non-running state.
+ **/
+ public ServiceStatus diagnostics(String diagnostics) {
+ this.diagnostics = diagnostics;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state.")
+ @JsonProperty("diagnostics")
+ public String getDiagnostics() {
+ return diagnostics;
+ }
+
+ public void setDiagnostics(String diagnostics) {
+ this.diagnostics = diagnostics;
+ }
+
+ /**
+ * Service state.
+ **/
+ public ServiceStatus state(ServiceState state) {
+ this.state = state;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Service state.")
+ @JsonProperty("state")
+ public ServiceState getState() {
+ return state;
+ }
+
+ public void setState(ServiceState state) {
+ this.state = state;
+ }
+
+ /**
+ * An error code specific to a scenario which service owners should be able to use
+ * to understand the failure in addition to the diagnostic information.
+ **/
+ public ServiceStatus code(Integer code) {
+ this.code = code;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.")
+ @JsonProperty("code")
+ public Integer getCode() {
+ return code;
+ }
+
+ public void setCode(Integer code) {
+ this.code = code;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ServiceStatus serviceStatus = (ServiceStatus) o;
+ return Objects.equals(this.diagnostics, serviceStatus.diagnostics)
+ && Objects.equals(this.state, serviceStatus.state)
+ && Objects.equals(this.code, serviceStatus.code);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(diagnostics, state, code);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class ServiceStatus {\n");
+
+ sb.append(" diagnostics: ").append(toIndentedString(diagnostics))
+ .append("\n");
+ sb.append(" state: ").append(toIndentedString(state)).append("\n");
+ sb.append(" code: ").append(toIndentedString(code)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java
new file mode 100644
index 0000000..e17c0c4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.client.ServerProxy;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+
+import java.net.InetSocketAddress;
+
+import static org.apache.hadoop.io.retry.RetryPolicies.TRY_ONCE_THEN_FAIL;
+
+public class ClientAMProxy extends ServerProxy{
+
+ public static <T> T createProxy(final Configuration conf,
+ final Class<T> protocol, final UserGroupInformation ugi,
+ final YarnRPC rpc, final InetSocketAddress serverAddress) {
+ Configuration confClone = new Configuration(conf);
+ confClone.setInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
+ confClone.setInt(CommonConfigurationKeysPublic.
+ IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
+ RetryPolicy retryPolicy;
+
+ if (conf.getLong(YarnServiceConf.CLIENT_AM_RETRY_MAX_WAIT_MS, 0) == 0) {
+ // by default no retry
+ retryPolicy = TRY_ONCE_THEN_FAIL;
+ } else {
+ retryPolicy =
+ createRetryPolicy(conf, YarnServiceConf.CLIENT_AM_RETRY_MAX_WAIT_MS,
+ 15 * 60 * 1000, YarnServiceConf.CLIENT_AM_RETRY_MAX_INTERVAL_MS,
+ 2 * 1000);
+ }
+ return createRetriableProxy(confClone, protocol, ugi, rpc, serverAddress,
+ retryPolicy);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
new file mode 100644
index 0000000..cb27021
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.client.params.ClientArgs;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.*;
+
+public class ServiceCLI {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceClient.class);
+ protected ServiceClient client;
+
+ int exec(ClientArgs args) throws Throwable {
+ if (StringUtils.isEmpty(args.getAction())) {
+ System.out.println(args.usage());
+ return -1;
+ }
+ switch (args.getAction()) {
+ case ACTION_BUILD: // Upload app json onto hdfs
+ client.actionBuild(args.getActionBuildArgs());
+ break;
+ case ACTION_START: // start the app with the pre-uploaded app json on hdfs
+ client.actionStart(args.getClusterName());
+ break;
+ case ACTION_CREATE: // create == build + start
+ client.actionCreate(args.getActionCreateArgs());
+ break;
+ case ACTION_STATUS:
+ Service app = client.getStatus(args.getClusterName());
+ System.out.println(app);
+ break;
+ case ACTION_FLEX:
+ try {
+ client.actionFlexByCLI(args);
+ } catch (FileNotFoundException e) {
+ System.err.println(
+ args.getClusterName() + " doesn't exist: " + e.getMessage());
+ return -1;
+ }
+ break;
+ case ACTION_STOP:
+ client.actionStop(args.getClusterName(), false);
+ break;
+ case ACTION_DESTROY: // Destroy can happen only if app is already stopped
+ client.actionDestroy(args.getClusterName());
+ break;
+ case ACTION_DEPENDENCY: // upload dependency jars
+ client.actionDependency(args.getActionDependencyArgs());
+ break;
+ case ACTION_UPDATE:
+ client.updateLifetime(args.getClusterName(),
+ args.getActionUpdateArgs().lifetime);
+ break;
+ case ACTION_HELP:
+ LOG.info(args.usage());
+ break;
+ default:
+ LOG.info("NOT IMPLEMENTED: " + args.getAction());
+ LOG.info(args.usage());
+ return -1;
+ }
+ return 0;
+ }
+
+ public ServiceCLI() {
+ createServiceClient();
+ }
+
+ protected void createServiceClient() {
+ client = new ServiceClient();
+ client.init(new YarnConfiguration());
+ client.start();
+ }
+
+ public static void main(String[] args) throws Throwable {
+ ClientArgs clientArgs = new ClientArgs(args);
+ try {
+ clientArgs.parse();
+ } catch (BadCommandArgumentsException e) {
+ System.err.println(e.getMessage());
+ System.exit(-1);
+ }
+ ServiceCLI cli = new ServiceCLI();
+ int res = cli.exec(clientArgs);
+ System.exit(res);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
deleted file mode 100644
index e89306c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import org.apache.commons.lang.StringUtils;
-import org.apache.slider.common.tools.SliderUtils;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * Set of configuration properties that can be injected into the application
- * components via envs, files and custom pluggable helper docker containers.
- * Files of several standard formats like xml, properties, json, yaml and
- * templates will be supported.
- **/
-
-@ApiModel(description = "Set of configuration properties that can be injected into the application components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Configuration implements Serializable {
- private static final long serialVersionUID = -4330788704981074466L;
-
- private Map<String, String> properties = new HashMap<String, String>();
- private Map<String, String> env = new HashMap<String, String>();
- private List<ConfigFile> files = new ArrayList<ConfigFile>();
-
- /**
- * A blob of key-value pairs of common application properties.
- **/
- public Configuration properties(Map<String, String> properties) {
- this.properties = properties;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A blob of key-value pairs of common application properties.")
- @JsonProperty("properties")
- public Map<String, String> getProperties() {
- return properties;
- }
-
- public void setProperties(Map<String, String> properties) {
- this.properties = properties;
- }
-
- /**
- * A blob of key-value pairs which will be appended to the default system
- * properties and handed off to the application at start time. All placeholder
- * references to properties will be substituted before injection.
- **/
- public Configuration env(Map<String, String> env) {
- this.env = env;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A blob of key-value pairs which will be appended to the default system properties and handed off to the application at start time. All placeholder references to properties will be substituted before injection.")
- @JsonProperty("env")
- public Map<String, String> getEnv() {
- return env;
- }
-
- public void setEnv(Map<String, String> env) {
- this.env = env;
- }
-
- /**
- * Array of list of files that needs to be created and made available as
- * volumes in the application component containers.
- **/
- public Configuration files(List<ConfigFile> files) {
- this.files = files;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Array of list of files that needs to be created and made available as volumes in the application component containers.")
- @JsonProperty("files")
- public List<ConfigFile> getFiles() {
- return files;
- }
-
- public void setFiles(List<ConfigFile> files) {
- this.files = files;
- }
-
- public long getPropertyLong(String name, long defaultValue) {
- String value = getProperty(name);
- if (StringUtils.isEmpty(value)) {
- return defaultValue;
- }
- return Long.parseLong(value);
- }
-
- public int getPropertyInt(String name, int defaultValue) {
- String value = getProperty(name);
- if (StringUtils.isEmpty(value)) {
- return defaultValue;
- }
- return Integer.parseInt(value);
- }
-
- public boolean getPropertyBool(String name, boolean defaultValue) {
- String value = getProperty(name);
- if (StringUtils.isEmpty(value)) {
- return defaultValue;
- }
- return Boolean.parseBoolean(value);
- }
-
- public String getProperty(String name, String defaultValue) {
- String value = getProperty(name);
- if (StringUtils.isEmpty(value)) {
- return defaultValue;
- }
- return value;
- }
-
- public void setProperty(String name, String value) {
- properties.put(name, value);
- }
-
- public String getProperty(String name) {
- return properties.get(name.trim());
- }
-
- public String getEnv(String name) {
- return env.get(name.trim());
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Configuration configuration = (Configuration) o;
- return Objects.equals(this.properties, configuration.properties)
- && Objects.equals(this.env, configuration.env)
- && Objects.equals(this.files, configuration.files);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(properties, env, files);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Configuration {\n");
-
- sb.append(" properties: ").append(toIndentedString(properties))
- .append("\n");
- sb.append(" env: ").append(toIndentedString(env)).append("\n");
- sb.append(" files: ").append(toIndentedString(files)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-
- /**
- * Merge all properties and envs from that configuration to this configration.
- * For ConfigFiles, all properties and envs of that ConfigFile are merged into
- * this ConfigFile.
- */
- public synchronized void mergeFrom(Configuration that) {
- SliderUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
- .getProperties());
- SliderUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
-
- Map<String, ConfigFile> thatMap = new HashMap<>();
- for (ConfigFile file : that.getFiles()) {
- thatMap.put(file.getDestFile(), file.copy());
- }
- for (ConfigFile thisFile : files) {
- if(thatMap.containsKey(thisFile.getDestFile())) {
- ConfigFile thatFile = thatMap.get(thisFile.getDestFile());
- SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProps(),
- thatFile.getProps());
- thatMap.remove(thisFile.getDestFile());
- }
- }
- // add remaining new files from that Configration
- for (ConfigFile thatFile : thatMap.values()) {
- files.add(thatFile.copy());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Container.java
deleted file mode 100644
index c5dc627..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Container.java
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.util.Date;
-import java.util.Objects;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * An instance of a running application container.
- **/
-
-@ApiModel(description = "An instance of a running application container")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@XmlRootElement
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Container extends BaseResource {
- private static final long serialVersionUID = -8955788064529288L;
-
- private String id = null;
- private Date launchTime = null;
- private String ip = null;
- private String hostname = null;
- private String bareHost = null;
- private ContainerState state = null;
- private String componentName = null;
- private Resource resource = null;
- private Artifact artifact = null;
- private Boolean privilegedContainer = null;
-
- /**
- * Unique container id of a running application, e.g.
- * container_e3751_1458061340047_0008_01_000002.
- **/
- public Container id(String id) {
- this.id = id;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Unique container id of a running application, e.g. container_e3751_1458061340047_0008_01_000002.")
- @JsonProperty("id")
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- /**
- * The time when the container was created, e.g. 2016-03-16T01:01:49.000Z.
- * This will most likely be different from cluster launch time.
- **/
- public Container launchTime(Date launchTime) {
- this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.")
- @JsonProperty("launch_time")
- public Date getLaunchTime() {
- return launchTime == null ? null : (Date) launchTime.clone();
- }
-
- @XmlElement(name = "launch_time")
- public void setLaunchTime(Date launchTime) {
- this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
- }
-
- /**
- * IP address of a running container, e.g. 172.31.42.141. The IP address and
- * hostname attribute values are dependent on the cluster/docker network setup
- * as per YARN-4007.
- **/
- public Container ip(String ip) {
- this.ip = ip;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.")
- @JsonProperty("ip")
- public String getIp() {
- return ip;
- }
-
- public void setIp(String ip) {
- this.ip = ip;
- }
-
- /**
- * Fully qualified hostname of a running container, e.g.
- * ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and
- * hostname attribute values are dependent on the cluster/docker network setup
- * as per YARN-4007.
- **/
- public Container hostname(String hostname) {
- this.hostname = hostname;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.")
- @JsonProperty("hostname")
- public String getHostname() {
- return hostname;
- }
-
- public void setHostname(String hostname) {
- this.hostname = hostname;
- }
-
- /**
- * The bare node or host in which the container is running, e.g.
- * cn008.example.com.
- **/
- public Container bareHost(String bareHost) {
- this.bareHost = bareHost;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The bare node or host in which the container is running, e.g. cn008.example.com.")
- @JsonProperty("bare_host")
- public String getBareHost() {
- return bareHost;
- }
-
- @XmlElement(name = "bare_host")
- public void setBareHost(String bareHost) {
- this.bareHost = bareHost;
- }
-
- /**
- * State of the container of an application.
- **/
- public Container state(ContainerState state) {
- this.state = state;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "State of the container of an application.")
- @JsonProperty("state")
- public ContainerState getState() {
- return state;
- }
-
- public void setState(ContainerState state) {
- this.state = state;
- }
-
- /**
- * Name of the component that this container instance belongs to.
- **/
- public Container componentName(String componentName) {
- this.componentName = componentName;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Name of the component that this container instance belongs to.")
- @JsonProperty("component_name")
- public String getComponentName() {
- return componentName;
- }
-
- @XmlElement(name = "component_name")
- public void setComponentName(String componentName) {
- this.componentName = componentName;
- }
-
- /**
- * Resource used for this container.
- **/
- public Container resource(Resource resource) {
- this.resource = resource;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Resource used for this container.")
- @JsonProperty("resource")
- public Resource getResource() {
- return resource;
- }
-
- public void setResource(Resource resource) {
- this.resource = resource;
- }
-
- /**
- * Artifact used for this container.
- **/
- public Container artifact(Artifact artifact) {
- this.artifact = artifact;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact used for this container.")
- @JsonProperty("artifact")
- public Artifact getArtifact() {
- return artifact;
- }
-
- public void setArtifact(Artifact artifact) {
- this.artifact = artifact;
- }
-
- /**
- * Container running in privileged mode or not.
- **/
- public Container privilegedContainer(Boolean privilegedContainer) {
- this.privilegedContainer = privilegedContainer;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Container running in privileged mode or not.")
- @JsonProperty("privileged_container")
- public Boolean getPrivilegedContainer() {
- return privilegedContainer;
- }
-
- public void setPrivilegedContainer(Boolean privilegedContainer) {
- this.privilegedContainer = privilegedContainer;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Container container = (Container) o;
- return Objects.equals(this.id, container.id);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(id);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Container {\n");
-
- sb.append(" id: ").append(toIndentedString(id)).append("\n");
- sb.append(" launchTime: ").append(toIndentedString(launchTime))
- .append("\n");
- sb.append(" ip: ").append(toIndentedString(ip)).append("\n");
- sb.append(" hostname: ").append(toIndentedString(hostname)).append("\n");
- sb.append(" bareHost: ").append(toIndentedString(bareHost)).append("\n");
- sb.append(" state: ").append(toIndentedString(state)).append("\n");
- sb.append(" componentName: ").append(toIndentedString(componentName))
- .append("\n");
- sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
- sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
- sb.append(" privilegedContainer: ")
- .append(toIndentedString(privilegedContainer)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ContainerState.java
deleted file mode 100644
index cd1ef4a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ContainerState.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-/**
- * The current state of the container of an application.
- **/
-public enum ContainerState {
- RUNNING_BUT_UNREADY, READY, STOPPED
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Error.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Error.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Error.java
deleted file mode 100644
index 3cf9b29..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Error.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import io.swagger.annotations.ApiModelProperty;
-
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public class Error {
-
- private Integer code = null;
- private String message = null;
- private String fields = null;
-
- /**
- **/
- public Error code(Integer code) {
- this.code = code;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "")
- @JsonProperty("code")
- public Integer getCode() {
- return code;
- }
-
- public void setCode(Integer code) {
- this.code = code;
- }
-
- /**
- **/
- public Error message(String message) {
- this.message = message;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "")
- @JsonProperty("message")
- public String getMessage() {
- return message;
- }
-
- public void setMessage(String message) {
- this.message = message;
- }
-
- /**
- **/
- public Error fields(String fields) {
- this.fields = fields;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "")
- @JsonProperty("fields")
- public String getFields() {
- return fields;
- }
-
- public void setFields(String fields) {
- this.fields = fields;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Error error = (Error) o;
- return Objects.equals(this.code, error.code)
- && Objects.equals(this.message, error.message)
- && Objects.equals(this.fields, error.fields);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(code, message, fields);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Error {\n");
-
- sb.append(" code: ").append(toIndentedString(code)).append("\n");
- sb.append(" message: ").append(toIndentedString(message)).append("\n");
- sb.append(" fields: ").append(toIndentedString(fields)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/PlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/PlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/PlacementPolicy.java
deleted file mode 100644
index 306338f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/PlacementPolicy.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Placement policy of an instance of an application. This feature is in the
- * works in YARN-4902.
- **/
-
-@ApiModel(description = "Placement policy of an instance of an application. This feature is in the works in YARN-4902.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public class PlacementPolicy implements Serializable {
- private static final long serialVersionUID = 4341110649551172231L;
-
- private String label = null;
-
- /**
- * Assigns an app to a named partition of the cluster where the application
- * desires to run (optional). If not specified all apps are submitted to a
- * default label of the app owner. One or more labels can be setup for each
- * application owner account with required constraints like no-preemption,
- * sla-99999, preemption-ok, etc.
- **/
- public PlacementPolicy label(String label) {
- this.label = label;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Assigns an app to a named partition of the cluster where the application desires to run (optional). If not specified all apps are submitted to a default label of the app owner. One or more labels can be setup for each application owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.")
- @JsonProperty("label")
- public String getLabel() {
- return label;
- }
-
- public void setLabel(String label) {
- this.label = label;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- PlacementPolicy placementPolicy = (PlacementPolicy) o;
- return Objects.equals(this.label, placementPolicy.label);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(label);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class PlacementPolicy {\n");
-
- sb.append(" label: ").append(toIndentedString(label)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ReadinessCheck.java
deleted file mode 100644
index b3c85bd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ReadinessCheck.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonValue;
-
-/**
- * A custom command or a pluggable helper container to determine the readiness
- * of a container of a component. Readiness for every application is different.
- * Hence the need for a simple interface, with scope to support advanced
- * usecases.
- **/
-
-@ApiModel(description = "A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every application is different. Hence the need for a simple interface, with scope to support advanced usecases.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public class ReadinessCheck implements Serializable {
- private static final long serialVersionUID = -3836839816887186801L;
-
- public enum TypeEnum {
- HTTP("HTTP"),
- PORT("PORT");
-
- private String value;
-
- TypeEnum(String value) {
- this.value = value;
- }
-
- @Override
- @JsonValue
- public String toString() {
- return value;
- }
- }
-
- private TypeEnum type = null;
- private Map<String, String> props = new HashMap<String, String>();
- private Artifact artifact = null;
-
- /**
- * E.g. HTTP (YARN will perform a simple REST call at a regular interval and
- * expect a 204 No content).
- **/
- public ReadinessCheck type(TypeEnum type) {
- this.type = type;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).")
- @JsonProperty("type")
- public TypeEnum getType() {
- return type;
- }
-
- public void setType(TypeEnum type) {
- this.type = type;
- }
-
- public ReadinessCheck props(Map<String, String> props) {
- this.props = props;
- return this;
- }
-
- public ReadinessCheck putPropsItem(String key, String propsItem) {
- this.props.put(key, propsItem);
- return this;
- }
-
- /**
- * A blob of key value pairs that will be used to configure the check.
- * @return props
- **/
- @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be used to configure the check.")
- public Map<String, String> getProps() {
- return props;
- }
-
- public void setProps(Map<String, String> props) {
- this.props = props;
- }
-
- /**
- * Artifact of the pluggable readiness check helper container (optional). If
- * specified, this helper container typically hosts the http uri and
- * encapsulates the complex scripts required to perform actual container
- * readiness check. At the end it is expected to respond a 204 No content just
- * like the simplified use case. This pluggable framework benefits application
- * owners who can run applications without any packaging modifications. Note,
- * artifacts of type docker only is supported for now.
- **/
- public ReadinessCheck artifact(Artifact artifact) {
- this.artifact = artifact;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits application owners who can run applications without any packaging modifications. Note, artifacts of type docker only is supported for now.")
- @JsonProperty("artifact")
- public Artifact getArtifact() {
- return artifact;
- }
-
- public void setArtifact(Artifact artifact) {
- this.artifact = artifact;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- ReadinessCheck readinessCheck = (ReadinessCheck) o;
- return Objects.equals(this.type, readinessCheck.type) &&
- Objects.equals(this.props, readinessCheck.props) &&
- Objects.equals(this.artifact, readinessCheck.artifact);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(type, props, artifact);
- }
-
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class ReadinessCheck {\n");
-
- sb.append(" type: ").append(toIndentedString(type)).append("\n");
- sb.append(" props: ").append(toIndentedString(props)).append("\n");
- sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java
deleted file mode 100644
index 314dfbb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Resource determines the amount of resources (vcores, memory, network, etc.)
- * usable by a container. This field determines the resource to be applied for
- * all the containers of a component or application. The resource specified at
- * the app (or global) level can be overriden at the component level. Only one
- * of profile OR cpu & memory are exepected. It raises a validation
- * exception otherwise.
- **/
-
-@ApiModel(description = "Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public class Resource extends BaseResource implements Cloneable {
- private static final long serialVersionUID = -6431667797380250037L;
-
- private String profile = null;
- private Integer cpus = 1;
- private String memory = null;
-
- /**
- * Each resource profile has a unique id which is associated with a
- * cluster-level predefined memory, cpus, etc.
- **/
- public Resource profile(String profile) {
- this.profile = profile;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.")
- @JsonProperty("profile")
- public String getProfile() {
- return profile;
- }
-
- public void setProfile(String profile) {
- this.profile = profile;
- }
-
- /**
- * Amount of vcores allocated to each container (optional but overrides cpus
- * in profile if specified).
- **/
- public Resource cpus(Integer cpus) {
- this.cpus = cpus;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).")
- @JsonProperty("cpus")
- public Integer getCpus() {
- return cpus;
- }
-
- public void setCpus(Integer cpus) {
- this.cpus = cpus;
- }
-
- /**
- * Amount of memory allocated to each container (optional but overrides memory
- * in profile if specified). Currently accepts only an integer value and
- * default unit is in MB.
- **/
- public Resource memory(String memory) {
- this.memory = memory;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.")
- @JsonProperty("memory")
- public String getMemory() {
- return memory;
- }
-
- public void setMemory(String memory) {
- this.memory = memory;
- }
-
- public long getMemoryMB() {
- if (this.memory == null) {
- return 0;
- }
- return Long.valueOf(memory);
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Resource resource = (Resource) o;
- return Objects.equals(this.profile, resource.profile)
- && Objects.equals(this.cpus, resource.cpus)
- && Objects.equals(this.memory, resource.memory);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(profile, cpus, memory);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Resource {\n");
-
- sb.append(" profile: ").append(toIndentedString(profile)).append("\n");
- sb.append(" cpus: ").append(toIndentedString(cpus)).append("\n");
- sb.append(" memory: ").append(toIndentedString(memory)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException {
- return super.clone();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
deleted file mode 100644
index 687edd2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.types;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-/**
- * Serialized information about liveness
- * <p>
- * If true liveness probes are implemented, this
- * datatype can be extended to publish their details.
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class ApplicationLivenessInformation {
- /** flag set if the cluster is at size */
- public boolean allRequestsSatisfied;
-
- /** number of outstanding requests: those needed to satisfy */
- public int requestsOutstanding;
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("ApplicationLivenessInformation{");
- sb.append("allRequestsSatisfied=").append(allRequestsSatisfied);
- sb.append(", requestsOutstanding=").append(requestsOutstanding);
- sb.append('}');
- return sb.toString();
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
deleted file mode 100644
index d2fdd62..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.types;
-
-import org.apache.slider.api.StatusKeys;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Serializable version of component data.
- * <p>
- * This is sent in REST calls as a JSON object —but is also marshalled into
- * a protobuf structure. Look at {@link RestTypeMarshalling}
- * for the specifics there.
- * <p>
- * This means that if any fields are added here. they must be added to
- * <code>src/main/proto/SliderClusterMessages.proto</code> and
- * the protobuf structures rebuilt via a {@code mvn generate-sources -Pcompile-protobuf}
- *
- * See also {@link RoleStatus#serialize()}
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-
-public class ComponentInformation {
-
- public String name;
- public int priority;
- public int desired, actual, releasing;
- public int placementPolicy;
- public int requested;
- public int failed, started, startFailed, completed, totalRequested;
- public int nodeFailed, failedRecently, preempted;
- public int pendingAntiAffineRequestCount;
- public boolean isAARequestOutstanding;
-
- public String failureMessage;
- public List<String> containers;
-
- /**
- * Build the statistics map from the current data
- * @return a map for use in statistics reports
- */
- public Map<String, Integer> buildStatistics() {
- Map<String, Integer> stats = new HashMap<>();
- stats.put(StatusKeys.STATISTICS_CONTAINERS_ACTIVE_REQUESTS, requested);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_ANTI_AFFINE_PENDING, pendingAntiAffineRequestCount);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED, completed);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_DESIRED, desired);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED, failed);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_NODE, nodeFailed);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_RECENTLY, failedRecently);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE, actual);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_PREEMPTED, preempted);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_REQUESTED, totalRequested);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED, started);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED, startFailed);
- return stats;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("ComponentInformation{");
- sb.append(", name='").append(name).append('\'');
- sb.append(", actual=").append(actual);
- sb.append(", completed=").append(completed);
- sb.append(", desired=").append(desired);
- sb.append(", failed=").append(failed);
- sb.append(", failureMessage='").append(failureMessage).append('\'');
- sb.append(", placementPolicy=").append(placementPolicy);
- sb.append(", isAARequestOutstanding=").append(isAARequestOutstanding);
- sb.append(", pendingAntiAffineRequestCount=").append(pendingAntiAffineRequestCount);
- sb.append(", priority=").append(priority);
- sb.append(", releasing=").append(releasing);
- sb.append(", requested=").append(requested);
- sb.append(", started=").append(started);
- sb.append(", startFailed=").append(startFailed);
- sb.append(", totalRequested=").append(totalRequested);
- sb.append(", container count='")
- .append(containers == null ? 0 : containers.size())
- .append('\'');
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java
deleted file mode 100644
index 6991340..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.types;
-
-import org.apache.hadoop.registry.client.binding.JsonSerDeser;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-/**
- * Serializable version of component instance data
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class ContainerInformation {
-
- public String containerId;
- public String component;
- public String appVersion;
- public Boolean released;
- public int state;
- public Integer exitCode;
- public String diagnostics;
- public long createTime;
- public long startTime;
-
- public String host;
- public String hostURL;
- public String placement;
- /**
- * What is the tail output from the executed process (or [] if not started
- * or the log cannot be picked up
- */
- public String[] output;
-
- @Override
- public String toString() {
- JsonSerDeser<ContainerInformation> serDeser =
- new JsonSerDeser<>(
- ContainerInformation.class);
- return serDeser.toString(this);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java
deleted file mode 100644
index 8424be2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.types;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-/**
- * Serialized node entry information. Must be kept in sync with the protobuf equivalent.
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class NodeEntryInformation {
-
- /** incrementing counter of instances that failed */
- public int failed;
-
- /** Counter of "failed recently" events. */
- public int failedRecently;
-
- /** timestamp of last use */
- public long lastUsed;
-
- /** Number of live nodes. */
- public int live;
-
- /** incrementing counter of instances that have been pre-empted. */
- public int preempted;
-
- /** Priority */
- public int priority;
-
- /** instance explicitly requested on this node */
- public int requested;
-
- /** number of containers being released off this node */
- public int releasing;
-
- /** incrementing counter of instances that failed to start */
- public int startFailed;
-
- /** number of starting instances */
- public int starting;
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder(
- "NodeEntryInformation{");
- sb.append("priority=").append(priority);
- sb.append(", live=").append(live);
- sb.append(", requested=").append(requested);
- sb.append(", releasing=").append(releasing);
- sb.append(", starting=").append(starting);
- sb.append(", failed=").append(failed);
- sb.append(", failedRecently=").append(failedRecently);
- sb.append(", startFailed=").append(startFailed);
- sb.append(", preempted=").append(preempted);
- sb.append(", lastUsed=").append(lastUsed);
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java
deleted file mode 100644
index 4fe5b4c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.types;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Serialized node information. Must be kept in sync with the protobuf equivalent.
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class NodeInformation {
-
- public String hostname;
- public String state;
- public String labels;
- public String rackName;
- public String httpAddress;
- public String healthReport;
- public long lastUpdated;
- public Map<String, NodeEntryInformation> entries = new HashMap<>();
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder(
- "NodeInformation{");
- sb.append("hostname='").append(hostname).append('\'');
- sb.append(", state='").append(state).append('\'');
- sb.append(", labels='").append(labels).append('\'');
- sb.append(", rackName='").append(rackName).append('\'');
- sb.append(", httpAddress='").append(httpAddress).append('\'');
- sb.append(", healthReport='").append(healthReport).append('\'');
- sb.append(", lastUpdated=").append(lastUpdated);
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java
deleted file mode 100644
index 741523e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.types;
-
-import org.apache.slider.core.persist.JsonSerDeser;
-
-import java.util.ArrayList;
-import java.util.Collection;
-
-public class NodeInformationList extends ArrayList<NodeInformation> {
- public NodeInformationList() {
- }
-
- public NodeInformationList(Collection<? extends NodeInformation> c) {
- super(c);
- }
-
- public NodeInformationList(int initialCapacity) {
- super(initialCapacity);
- }
-
- public static JsonSerDeser<NodeInformationList> createSerializer() {
- return new JsonSerDeser<>(NodeInformationList.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/PingInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/PingInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/PingInformation.java
deleted file mode 100644
index 223edca..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/PingInformation.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.types;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-/**
- * Serialized information to/from Ping operations
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class PingInformation {
- public long time;
- public String text;
- public String verb;
- public String body;
-
- @Override
- public String toString() {
-
- final StringBuilder sb =
- new StringBuilder("PingResource{");
- sb.append("time=").append(time);
- sb.append(", verb=").append(verb);
- sb.append(", text='").append(text).append('\'');
- sb.append(", body='").append(body).append('\'');
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java
deleted file mode 100644
index bc3d526..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.types;
-
-import org.apache.slider.api.proto.Messages;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Class to handle marshalling of REST
- * types to/from Protobuf records.
- */
-public class RestTypeMarshalling {
-
- public static Messages.ApplicationLivenessInformationProto
- marshall(ApplicationLivenessInformation info) {
-
- Messages.ApplicationLivenessInformationProto.Builder builder =
- Messages.ApplicationLivenessInformationProto.newBuilder();
- builder.setAllRequestsSatisfied(info.allRequestsSatisfied);
- builder.setRequestsOutstanding(info.requestsOutstanding);
- return builder.build();
- }
-
- public static ApplicationLivenessInformation
- unmarshall(Messages.ApplicationLivenessInformationProto wire) {
- ApplicationLivenessInformation info = new ApplicationLivenessInformation();
- info.allRequestsSatisfied = wire.getAllRequestsSatisfied();
- info.requestsOutstanding = wire.getRequestsOutstanding();
- return info;
- }
-
- public static ComponentInformation
- unmarshall(Messages.ComponentInformationProto wire) {
- ComponentInformation info = new ComponentInformation();
- info.name = wire.getName();
- info.priority = wire.getPriority();
- info.placementPolicy = wire.getPlacementPolicy();
-
- info.actual = wire.getActual();
- info.completed = wire.getCompleted();
- info.desired = wire.getDesired();
- info.failed = wire.getFailed();
- info.releasing = wire.getReleasing();
- info.requested = wire.getRequested();
- info.started = wire.getStarted();
- info.startFailed = wire.getStartFailed();
- info.totalRequested = wire.getTotalRequested();
- info.containers = new ArrayList<>(wire.getContainersList());
- if (wire.hasFailureMessage()) {
- info.failureMessage = wire.getFailureMessage();
- }
- if (wire.hasPendingAntiAffineRequestCount()) {
- info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount();
- }
- if (wire.hasIsAARequestOutstanding()) {
- info.isAARequestOutstanding = wire.getIsAARequestOutstanding();
- }
- return info;
- }
- public static Messages.ComponentInformationProto marshall(ComponentInformation info) {
-
- Messages.ComponentInformationProto.Builder builder =
- Messages.ComponentInformationProto.newBuilder();
- builder.setName(info.name);
- builder.setPriority(info.priority);
- builder.setPlacementPolicy(info.placementPolicy);
-
- builder.setActual(info.actual);
- builder.setCompleted(info.completed);
- builder.setDesired(info.desired);
- builder.setFailed(info.failed);
- builder.setReleasing(info.releasing);
- builder.setRequested(info.requested);
- builder.setStarted(info.started);
- builder.setStartFailed(info.startFailed);
- builder.setTotalRequested(info.totalRequested);
- builder.setNodeFailed(info.nodeFailed);
- builder.setPreempted(info.preempted);
- builder.setFailedRecently(info.failedRecently);
- if (info.failureMessage != null) {
- builder.setFailureMessage(info.failureMessage);
- }
- if (info.containers != null) {
- builder.addAllContainers(info.containers);
- }
- builder.setPendingAntiAffineRequestCount(info.pendingAntiAffineRequestCount);
- builder.setIsAARequestOutstanding(info.isAARequestOutstanding);
- return builder.build();
- }
-
- public static Messages.NodeInformationProto marshall(NodeInformation info) {
-
- Messages.NodeInformationProto.Builder builder =
- Messages.NodeInformationProto.newBuilder();
- builder.setHostname(info.hostname);
- builder.setLastUpdated(info.lastUpdated);
- builder.setState(info.state != null? info.state : "unknown");
- builder.setRackName(info.rackName != null ? info.rackName : "");
- builder.setHealthReport(info.healthReport != null ? info.healthReport : "");
- builder.setHttpAddress(info.httpAddress != null ? info.httpAddress : "");
- builder.setLabels(info.labels != null ? info.labels: "");
-
-
- if (info.entries != null) {
- for (Map.Entry<String, NodeEntryInformation> elt : info.entries.entrySet()) {
- NodeEntryInformation entry = elt.getValue();
- Messages.NodeEntryInformationProto.Builder node =
- Messages.NodeEntryInformationProto.newBuilder();
- node.setPriority(entry.priority);
- node.setName(elt.getKey());
- node.setFailed(entry.failed);
- node.setFailedRecently(entry.failedRecently);
- node.setLive(entry.live);
- node.setLastUsed(entry.lastUsed);
- node.setPreempted(entry.preempted);
- node.setRequested(entry.requested);
- node.setReleasing(entry.releasing);
- node.setStartFailed(entry.startFailed);
- node.setStarting(entry.starting);
- builder.addEntries(node.build());
- }
- }
- return builder.build();
- }
-
- public static NodeInformation unmarshall(Messages.NodeInformationProto wire) {
- NodeInformation info = new NodeInformation();
- info.healthReport = wire.getHealthReport();
- info.hostname = wire.getHostname();
- info.httpAddress = wire.getHttpAddress();
- info.labels = wire.getLabels();
- info.lastUpdated = wire.getLastUpdated();
- info.rackName = wire.getRackName();
- info.state = wire.getState();
- List<Messages.NodeEntryInformationProto> entriesList = wire.getEntriesList();
- if (entriesList != null) {
- info.entries = new HashMap<>(entriesList.size());
- for (Messages.NodeEntryInformationProto entry : entriesList) {
- NodeEntryInformation nei = new NodeEntryInformation();
- nei.failed = entry.getFailed();
- nei.failedRecently = entry.getFailedRecently();
- nei.lastUsed = entry.getLastUsed();
- nei.live = entry.getLive();
- nei.preempted = entry.getPreempted();
- nei.priority = entry.getPriority();
- nei.requested = entry.getRequested();
- nei.releasing = entry.getReleasing();
- nei.startFailed = entry.getStartFailed();
- nei.starting = entry.getStarting();
- info.entries.put(entry.getName(), nei);
- }
- }
- return info;
- }
-
- public static ContainerInformation unmarshall(Messages.ContainerInformationProto wire) {
- ContainerInformation info = new ContainerInformation();
- info.containerId = wire.getContainerId();
- info.component = wire.getComponent();
- info.appVersion = wire.getAppVersion();
- info.state = wire.getState();
- if (wire.hasReleased()) {
- info.released = wire.getReleased();
- }
- if (wire.hasExitCode()) {
- info.exitCode = wire.getExitCode();
- }
- if (wire.hasDiagnostics()) {
- info.diagnostics = wire.getDiagnostics();
- }
- if (wire.hasHost()) {
- info.host = wire.getHost();
- }
- if (wire.hasHostURL()) {
- info.host = wire.getHostURL();
- }
- info.createTime = wire.getCreateTime();
- info.startTime = wire.getStartTime();
- info.output = wire.getOutputList().toArray(
- new String[wire.getOutputCount()]
- );
- if (wire.hasPlacement()) {
- info.placement = wire.getPlacement();
- }
- return info;
- }
-
- public static List<ContainerInformation> unmarshall(Messages.GetLiveContainersResponseProto wire) {
- List<ContainerInformation> infoList = new ArrayList<>(wire.getContainersList().size());
- for (Messages.ContainerInformationProto container : wire.getContainersList()) {
- infoList.add(unmarshall(container));
- }
- return infoList;
- }
-
- public static Messages.ContainerInformationProto marshall(ContainerInformation info) {
-
- Messages.ContainerInformationProto.Builder builder =
- Messages.ContainerInformationProto.newBuilder();
- if (info.containerId != null) {
- builder.setContainerId(info.containerId);
- }
- if (info.component != null) {
- builder.setComponent(info.component);
- }
- if (info.appVersion != null) {
- builder.setAppVersion(info.appVersion);
- }
- builder.setCreateTime(info.createTime);
- if (info.diagnostics != null) {
- builder.setDiagnostics(info.diagnostics);
- }
- if (info.host != null) {
- builder.setHost(info.host);
- }
- if (info.hostURL != null) {
- builder.setHostURL(info.hostURL);
- }
- if (info.output != null) {
- builder.addAllOutput(Arrays.asList(info.output));
- }
- if (info.released != null) {
- builder.setReleased(info.released);
- }
- if (info.placement != null) {
- builder.setPlacement(info.placement);
- }
- builder.setStartTime(info.startTime);
- builder.setState(info.state);
- return builder.build();
- }
-
- public static String unmarshall(Messages.WrappedJsonProto wire) {
- return wire.getJson();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java
deleted file mode 100644
index 25f4d9d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.types;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-
-/**
- * Simple role statistics for state views; can be generated by RoleStatus
- * instances, and aggregated for summary information.
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class RoleStatistics {
- public long activeAA = 0L;
- public long actual = 0L;
- public long completed = 0L;
- public long desired = 0L;
- public long failed = 0L;
- public long failedRecently = 0L;
- public long limitsExceeded = 0L;
- public long nodeFailed = 0L;
- public long preempted = 0L;
- public long requested = 0L;
- public long started = 0L;
-
- /**
- * Add another statistics instance
- * @param that the other value
- * @return this entry
- */
- public RoleStatistics add(final RoleStatistics that) {
- activeAA += that.activeAA;
- actual += that.actual;
- completed += that.completed;
- desired += that.desired;
- failed += that.failed;
- failedRecently += that.failedRecently;
- limitsExceeded += that.limitsExceeded;
- nodeFailed += that.nodeFailed;
- preempted += that.preempted;
- requested += that.requested;
- started += that.started;
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java
deleted file mode 100644
index 3b95f80..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.types;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-
-/**
- * Description of a slider instance
- */
-public class SliderInstanceDescription {
-
- public final String name;
- public final Path path;
- public final ApplicationReport applicationReport;
-
- public SliderInstanceDescription(String name,
- Path path,
- ApplicationReport applicationReport) {
- this.name = name;
- this.path = path;
- this.applicationReport = applicationReport;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("SliderInstanceDescription{");
- sb.append("name='").append(name).append('\'');
- sb.append(", path=").append(path);
- sb.append(", applicationReport: ")
- .append(applicationReport == null
- ? "null"
- : (" id " + applicationReport.getApplicationId()));
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientRegistryBinder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientRegistryBinder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientRegistryBinder.java
deleted file mode 100644
index da37d11..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientRegistryBinder.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
-import org.apache.hadoop.registry.client.impl.zk.RegistryInternalConstants;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.encodeForRegistry;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.convertUsername;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.getCurrentUsernameUnencoded;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
-
-/**
- * Generic code to get the URLs for clients via the registry
- */
-public class ClientRegistryBinder {
- private static final Logger log =
- LoggerFactory.getLogger(ClientRegistryBinder.class);
-
- private final RegistryOperations operations;
-
- public ClientRegistryBinder(RegistryOperations operations) {
- this.operations = operations;
- }
-
- /**
- * Buld the user path -switches to the system path if the user is "".
- * It also cross-converts the username to ascii via punycode
- * @param username username or ""
- * @return the path to the user
- */
- public static String homePathForUser(String username) {
- Preconditions.checkArgument(username != null, "null user");
-
- // catch recursion
- if (username.startsWith(RegistryConstants.PATH_USERS)) {
- return username;
- }
-
- if (username.isEmpty()) {
- return RegistryConstants.PATH_SYSTEM_SERVICES;
- }
-
- // convert username to registry name
- String convertedName = convertUsername(username);
-
- return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
- encodeForRegistry(convertedName));
- }
-
- /**
- * Get the current username, before any encoding has been applied.
- * @return the current user from the kerberos identity, falling back
- * to the user and/or env variables.
- */
- public static String currentUsernameUnencoded() {
- String env_hadoop_username = System.getenv(
- RegistryInternalConstants.HADOOP_USER_NAME);
- return getCurrentUsernameUnencoded(env_hadoop_username);
- }
-
- /**
- * Qualify a user.
- * <ol>
- * <li> <code>"~"</code> maps to user home path home</li>
- * <li> <code>"~user"</code> maps to <code>/users/$user</code></li>
- * <li> <code>"/"</code> maps to <code>/services/</code></li>
- * </ol>
- * @param user the username
- * @return the base path
- */
- public static String qualifyUser(String user) {
- // qualify the user
- String t = user.trim();
- if (t.startsWith("/")) {
- // already resolved
- return t;
- } else if (t.equals("~")) {
- // self
- return currentUsernameUnencoded();
- } else if (t.startsWith("~")) {
- // another user
- // convert username to registry name
- String convertedName = convertUsername(t.substring(1));
-
- return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
- encodeForRegistry(convertedName));
- } else {
- return "/" + t;
- }
- }
-
- /**
- * Look up an external REST API
- * @param user user which will be qualified as per {@link #qualifyUser(String)}
- * @param serviceClass service class
- * @param instance instance name
- * @param api API
- * @return the API, or an exception is raised.
- * @throws IOException
- */
- public String lookupExternalRestAPI(String user,
- String serviceClass,
- String instance,
- String api)
- throws IOException {
- String qualified = qualifyUser(user);
- String path = servicePath(qualified, serviceClass, instance);
- String restAPI = resolveExternalRestAPI(api, path);
- if (restAPI == null) {
- throw new PathNotFoundException(path + " API " + api);
- }
- return restAPI;
- }
-
- /**
- * Resolve a service record then return an external REST API exported it.
- *
- * @param api API to resolve
- * @param path path of the service record
- * @return null if the record exists but the API is absent or it has no
- * REST endpoints.
- * @throws IOException resolution problems, as covered in
- * {@link RegistryOperations#resolve(String)}
- */
- protected String resolveExternalRestAPI(String api, String path) throws
- IOException {
- ServiceRecord record = operations.resolve(path);
- return lookupRestAPI(record, api, true);
- }
-
- /**
- * Look up an external REST API endpoint
- * @param record service record
- * @param api URI of api
- * @param external flag to indicate this is an external record
- * @return the first endpoint of the implementation, or null if there
- * is no entry for the API, implementation or it's the wrong type.
- */
- public static String lookupRestAPI(ServiceRecord record,
- String api, boolean external) throws InvalidRecordException {
- try {
- String url = null;
- Endpoint endpoint = getEndpoint(record, api, external);
- List<String> addresses =
- RegistryTypeUtils.retrieveAddressesUriType(endpoint);
- if (addresses != null && !addresses.isEmpty()) {
- url = addresses.get(0);
- }
- return url;
- } catch (InvalidRecordException e) {
- log.debug("looking for API {}", api, e);
- return null;
- }
- }
-
- /**
- * Get an endpont by API
- * @param record service record
- * @param api API
- * @param external flag to indicate this is an external record
- * @return the endpoint or null
- */
- public static Endpoint getEndpoint(ServiceRecord record,
- String api,
- boolean external) {
- return external ? record.getExternalEndpoint(api)
- : record.getInternalEndpoint(api);
- }
-
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
new file mode 100644
index 0000000..e4eae20
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
+import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.utils.CoreFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static org.apache.hadoop.yarn.service.provider.docker.DockerKeys.DEFAULT_DOCKER_NETWORK;
+
+/**
+ * Launcher of applications: base class
+ */
+public class AbstractLauncher {
+ private static final Logger log =
+ LoggerFactory.getLogger(AbstractLauncher.class);
+ public static final String CLASSPATH = "CLASSPATH";
+ /**
+ * Filesystem to use for the launch
+ */
+ protected final CoreFileSystem coreFileSystem;
+ /**
+ * Env vars; set up at final launch stage
+ */
+ protected final Map<String, String> envVars = new HashMap<>();
+ protected final ContainerLaunchContext containerLaunchContext =
+ Records.newRecord(ContainerLaunchContext.class);
+ protected final List<String> commands = new ArrayList<>(20);
+ protected final Map<String, LocalResource> localResources = new HashMap<>();
+ protected final Map<String, String> mountPaths = new HashMap<>();
+ private final Map<String, ByteBuffer> serviceData = new HashMap<>();
+ // security
+ protected final Credentials credentials;
+ protected boolean yarnDockerMode = false;
+ protected String dockerImage;
+ protected String dockerNetwork = DEFAULT_DOCKER_NETWORK;
+ protected String dockerHostname;
+ protected String runPrivilegedContainer;
+
+
+ /**
+ * Create instance.
+ * @param coreFileSystem filesystem
+ * @param credentials initial set of credentials -null is permitted
+ */
+ public AbstractLauncher(
+ CoreFileSystem coreFileSystem,
+ Credentials credentials) {
+ this.coreFileSystem = coreFileSystem;
+ this.credentials = credentials != null ? credentials: new Credentials();
+ }
+
+ public void setYarnDockerMode(boolean yarnDockerMode){
+ this.yarnDockerMode = yarnDockerMode;
+ }
+
+ /**
+ * Get the env vars to work on
+ * @return env vars
+ */
+ public Map<String, String> getEnv() {
+ return envVars;
+ }
+
+ /**
+ * Get the launch commands.
+ * @return the live list of commands
+ */
+ public List<String> getCommands() {
+ return commands;
+ }
+
+ public void addLocalResource(String subPath, LocalResource resource) {
+ localResources.put(subPath, resource);
+ }
+
+ public void addLocalResource(String subPath, LocalResource resource, String mountPath) {
+ localResources.put(subPath, resource);
+ mountPaths.put(subPath, mountPath);
+ }
+
+ /**
+ * Accessor to the credentials
+ * @return the credentials associated with this launcher
+ */
+ public Credentials getCredentials() {
+ return credentials;
+ }
+
+
+ public void addCommand(String cmd) {
+ commands.add(cmd);
+ }
+
+ /**
+ * Complete the launch context (copy in env vars, etc).
+ * @return the container to launch
+ */
+ public ContainerLaunchContext completeContainerLaunch() throws IOException {
+
+ String cmdStr = SliderUtils.join(commands, " ", false);
+ log.debug("Completed setting up container command {}", cmdStr);
+ containerLaunchContext.setCommands(commands);
+
+ //env variables
+ if (log.isDebugEnabled()) {
+ log.debug("Environment variables");
+ for (Map.Entry<String, String> envPair : envVars.entrySet()) {
+ log.debug(" \"{}\"=\"{}\"", envPair.getKey(), envPair.getValue());
+ }
+ }
+ containerLaunchContext.setEnvironment(envVars);
+
+ //service data
+ if (log.isDebugEnabled()) {
+ log.debug("Service Data size");
+ for (Map.Entry<String, ByteBuffer> entry : serviceData.entrySet()) {
+ log.debug("\"{}\"=> {} bytes of data", entry.getKey(),
+ entry.getValue().array().length);
+ }
+ }
+ containerLaunchContext.setServiceData(serviceData);
+
+ // resources
+ dumpLocalResources();
+ containerLaunchContext.setLocalResources(localResources);
+
+ //tokens
+ log.debug("{} tokens", credentials.numberOfTokens());
+ containerLaunchContext.setTokens(CredentialUtils.marshallCredentials(
+ credentials));
+
+ if(yarnDockerMode){
+ Map<String, String> env = containerLaunchContext.getEnvironment();
+ env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker");
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage);
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", dockerNetwork);
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME",
+ dockerHostname);
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer);
+ StringBuilder sb = new StringBuilder();
+ for (Entry<String,String> mount : mountPaths.entrySet()) {
+ if (sb.length() > 0) {
+ sb.append(",");
+ }
+ sb.append(mount.getKey());
+ sb.append(":");
+ sb.append(mount.getValue());
+ }
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS", sb.toString());
+ log.info("yarn docker env var has been set {}", containerLaunchContext.getEnvironment().toString());
+ }
+
+ return containerLaunchContext;
+ }
+
+ public void setRetryContext(int maxRetries, int retryInterval) {
+ ContainerRetryContext retryContext = ContainerRetryContext
+ .newInstance(ContainerRetryPolicy.RETRY_ON_ALL_ERRORS, null, maxRetries,
+ retryInterval);
+ containerLaunchContext.setContainerRetryContext(retryContext);
+ }
+
+ /**
+ * Dump local resources at debug level
+ */
+ private void dumpLocalResources() {
+ if (log.isDebugEnabled()) {
+ log.debug("{} resources: ", localResources.size());
+ for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) {
+
+ String key = entry.getKey();
+ LocalResource val = entry.getValue();
+ log.debug(key + "=" + SliderUtils.stringify(val.getResource()));
+ }
+ }
+ }
+
+ /**
+ * This is critical for an insecure cluster -it passes
+ * down the username to YARN, and so gives the code running
+ * in containers the rights it needs to work with
+ * data.
+ * @throws IOException problems working with current user
+ */
+ protected void propagateUsernameInInsecureCluster() throws IOException {
+ //insecure cluster: propagate user name via env variable
+ String userName = UserGroupInformation.getCurrentUser().getUserName();
+ envVars.put(YarnServiceConstants.HADOOP_USER_NAME, userName);
+ }
+
+ /**
+ * Utility method to set up the classpath
+ * @param classpath classpath to use
+ */
+ public void setClasspath(ClasspathConstructor classpath) {
+ setEnv(CLASSPATH, classpath.buildClasspath());
+ }
+
+ /**
+ * Set an environment variable in the launch context
+ * @param var variable name
+ * @param value value (must be non null)
+ */
+ public void setEnv(String var, String value) {
+ Preconditions.checkArgument(var != null, "null variable name");
+ Preconditions.checkArgument(value != null, "null value");
+ envVars.put(var, value);
+ }
+
+
+ public void putEnv(Map<String, String> map) {
+ envVars.putAll(map);
+ }
+
+
+ public void setDockerImage(String dockerImage) {
+ this.dockerImage = dockerImage;
+ }
+
+ public void setDockerNetwork(String dockerNetwork) {
+ this.dockerNetwork = dockerNetwork;
+ }
+
+ public void setDockerHostname(String dockerHostname) {
+ this.dockerHostname = dockerHostname;
+ }
+
+ public void setRunPrivilegedContainer(boolean runPrivilegedContainer) {
+ if (runPrivilegedContainer) {
+ this.runPrivilegedContainer = Boolean.toString(true);
+ } else {
+ this.runPrivilegedContainer = Boolean.toString(false);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
new file mode 100644
index 0000000..22b3877
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * build a classpath -allows for entries to be injected in front of
+ * YARN classpath as well as behind, adds appropriate separators,
+ * extraction of local classpath, etc.
+ */
+public class ClasspathConstructor {
+
+ public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR;
+ private final List<String> pathElements = new ArrayList<>();
+
+ public ClasspathConstructor() {
+ }
+
+
+ /**
+ * Get the list of JARs from the YARN settings
+ * @param config configuration
+ */
+ public List<String> yarnApplicationClasspath(Configuration config) {
+ String[] cp = config.getTrimmedStrings(
+ YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+ YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH);
+ return cp != null ? Arrays.asList(cp) : new ArrayList<String>(0);
+
+ }
+
+
+ @Override
+ public String toString() {
+ return buildClasspath();
+ }
+
+ public String buildClasspath() {
+ return SliderUtils.join(pathElements,
+ CLASS_PATH_SEPARATOR,
+ false);
+ }
+
+ /**
+ * Get a copy of the path list
+ * @return the JARs
+ */
+ public List<String> getPathElements() {
+ return Collections.unmodifiableList(pathElements);
+ }
+
+ /**
+ * Append an entry
+ * @param path path
+ */
+ public void append(String path) {
+ pathElements.add(path);
+ }
+
+ /**
+ * Insert a path at the front of the list. This places it ahead of
+ * the standard YARN artifacts
+ * @param path path to the JAR. Absolute or relative -on the target
+ * system
+ */
+ public void insert(String path) {
+ pathElements.add(0, path);
+ }
+
+ public void appendAll(Collection<String> paths) {
+ pathElements.addAll(paths);
+ }
+
+ public void insertAll(Collection<String> paths) {
+ pathElements.addAll(0, paths);
+ }
+
+
+ public void addLibDir(String pathToLibDir) {
+ append(buildLibDir(pathToLibDir));
+ }
+
+ public void insertLibDir(String pathToLibDir) {
+ insert(buildLibDir(pathToLibDir));
+ }
+
+ public void addClassDirectory(String pathToDir) {
+ append(appendDirectoryTerminator(pathToDir));
+ }
+
+ public void insertClassDirectory(String pathToDir) {
+ insert(buildLibDir(appendDirectoryTerminator(pathToDir)));
+ }
+
+
+ public void addRemoteClasspathEnvVar() {
+ append(ApplicationConstants.Environment.CLASSPATH.$$());
+ }
+
+
+ public void insertRemoteClasspathEnvVar() {
+ append(ApplicationConstants.Environment.CLASSPATH.$$());
+ }
+
+
+ /**
+ * Build a lib dir path
+ * @param pathToLibDir path to the directory; may or may not end with a
+ * trailing space
+ * @return a path to a lib dir that is compatible with the java classpath
+ */
+ public String buildLibDir(String pathToLibDir) {
+ String dir = appendDirectoryTerminator(pathToLibDir);
+ dir += "*";
+ return dir;
+ }
+
+ private String appendDirectoryTerminator(String pathToLibDir) {
+ String dir = pathToLibDir.trim();
+ if (!dir.endsWith("/")) {
+ dir += "/";
+ }
+ return dir;
+ }
+
+ /**
+ * Split a classpath. This uses the local path separator so MUST NOT
+ * be used to work with remote classpaths
+ * @param localpath local path
+ * @return a splite
+ */
+ public Collection<String> splitClasspath(String localpath) {
+ String separator = System.getProperty("path.separator");
+ return StringUtils.getStringCollection(localpath, separator);
+ }
+
+ /**
+ * Get the local JVM classpath split up
+ * @return the list of entries on the JVM classpath env var
+ */
+ public Collection<String> localJVMClasspath() {
+ return splitClasspath(System.getProperty("java.class.path"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
new file mode 100644
index 0000000..7baa284
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Build a single command line to include in the container commands;
+ * Special support for JVM command buildup.
+ */
+public class CommandLineBuilder {
+ protected final List<String> argumentList = new ArrayList<>(20);
+
+ /**
+ * Add an entry to the command list
+ * @param args arguments -these will be converted strings
+ */
+ public void add(Object... args) {
+ for (Object arg : args) {
+ argumentList.add(arg.toString());
+ }
+ }
+
+ // Get the number of arguments
+ public int size() {
+ return argumentList.size();
+ }
+
+ /**
+ * Append the output and error files to the tail of the command
+ * @param stdout out
+ * @param stderr error. Set this to null to append into stdout
+ */
+ public void addOutAndErrFiles(String stdout, String stderr) {
+ Preconditions.checkNotNull(stdout, "Null output file");
+ Preconditions.checkState(!stdout.isEmpty(), "output filename invalid");
+ // write out the path output
+ argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
+ stdout);
+ if (stderr != null) {
+ argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
+ stderr);
+ } else {
+ argumentList.add("2>&1");
+ }
+ }
+
+ /**
+ * This just returns the command line
+ * @see #build()
+ * @return the command line
+ */
+ @Override
+ public String toString() {
+ return build();
+ }
+
+ /**
+ * Build the command line
+ * @return the command line
+ */
+ public String build() {
+ return SliderUtils.join(argumentList, " ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
new file mode 100644
index 0000000..fcbb69b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class ContainerLaunchService extends AbstractService{
+
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(ContainerLaunchService.class);
+
+ private ExecutorService executorService;
+ private SliderFileSystem fs;
+
+ public ContainerLaunchService(SliderFileSystem fs) {
+ super(ContainerLaunchService.class.getName());
+ this.fs = fs;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ executorService = Executors.newCachedThreadPool();
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (executorService != null) {
+ executorService.shutdownNow();
+ }
+ super.serviceStop();
+ }
+
+ public void launchCompInstance(Application application,
+ ComponentInstance instance, Container container) {
+ ContainerLauncher launcher =
+ new ContainerLauncher(application, instance, container);
+ executorService.execute(launcher);
+ }
+
+ private class ContainerLauncher implements Runnable {
+ public final Container container;
+ public final Application application;
+ public ComponentInstance instance;
+
+ public ContainerLauncher(
+ Application application,
+ ComponentInstance instance, Container container) {
+ this.container = container;
+ this.application = application;
+ this.instance = instance;
+ }
+
+ @Override public void run() {
+ Component compSpec = instance.getCompSpec();
+ ProviderService provider = ProviderFactory.getProviderService(
+ compSpec.getArtifact());
+ AbstractLauncher launcher = new AbstractLauncher(fs, null);
+ try {
+ provider.buildContainerLaunchContext(launcher, application,
+ instance, fs, getConfig());
+ instance.getComponent().getScheduler().getNmClient()
+ .startContainerAsync(container,
+ launcher.completeContainerLaunch());
+ } catch (Exception e) {
+ LOG.error(instance.getCompInstanceId()
+ + ": Failed to launch container. ", e);
+
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
new file mode 100644
index 0000000..fce58e5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.text.DateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
+
+/**
+ * Utils to work with credentials and tokens.
+ *
+ * Designed to be movable to Hadoop core
+ */
+public final class CredentialUtils {
+
+ private CredentialUtils() {
+ }
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CredentialUtils.class);
+
+ /**
+ * Save credentials to a byte buffer. Returns null if there were no
+ * credentials to save
+ * @param credentials credential set
+ * @return a byte buffer of serialized tokens
+ * @throws IOException if the credentials could not be written to the stream
+ */
+ public static ByteBuffer marshallCredentials(Credentials credentials) throws IOException {
+ ByteBuffer buffer = null;
+ if (!credentials.getAllTokens().isEmpty()) {
+ DataOutputBuffer dob = new DataOutputBuffer();
+ try {
+ credentials.writeTokenStorageToStream(dob);
+ } finally {
+ dob.close();
+ }
+ buffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ }
+ return buffer;
+ }
+
+ /**
+ * Save credentials to a file
+ * @param file file to save to (will be overwritten)
+ * @param credentials credentials to write
+ * @throws IOException
+ */
+ public static void saveTokens(File file,
+ Credentials credentials) throws IOException {
+ try(DataOutputStream daos = new DataOutputStream(
+ new FileOutputStream(file))) {
+ credentials.writeTokenStorageToStream(daos);
+ }
+ }
+
+ /**
+ * Look up and return the resource manager's principal. This method
+ * automatically does the <code>_HOST</code> replacement in the principal and
+ * correctly handles HA resource manager configurations.
+ *
+ * From: YARN-4629
+ * @param conf the {@link Configuration} file from which to read the
+ * principal
+ * @return the resource manager's principal string
+ * @throws IOException thrown if there's an error replacing the host name
+ */
+ public static String getRMPrincipal(Configuration conf) throws IOException {
+ String principal = conf.get(RM_PRINCIPAL, "");
+ String hostname;
+ Preconditions.checkState(!principal.isEmpty(), "Not set: " + RM_PRINCIPAL);
+
+ if (HAUtil.isHAEnabled(conf)) {
+ YarnConfiguration yarnConf = new YarnConfiguration(conf);
+ if (yarnConf.get(RM_HA_ID) == null) {
+ // If RM_HA_ID is not configured, use the first of RM_HA_IDS.
+ // Any valid RM HA ID should work.
+ String[] rmIds = yarnConf.getStrings(RM_HA_IDS);
+ Preconditions.checkState((rmIds != null) && (rmIds.length > 0),
+ "Not set " + RM_HA_IDS);
+ yarnConf.set(RM_HA_ID, rmIds[0]);
+ }
+
+ hostname = yarnConf.getSocketAddr(
+ RM_ADDRESS,
+ DEFAULT_RM_ADDRESS,
+ DEFAULT_RM_PORT).getHostName();
+ } else {
+ hostname = conf.getSocketAddr(
+ RM_ADDRESS,
+ DEFAULT_RM_ADDRESS,
+ DEFAULT_RM_PORT).getHostName();
+ }
+ return SecurityUtil.getServerPrincipal(principal, hostname);
+ }
+
+ /**
+ * Create and add any filesystem delegation tokens with
+ * the RM(s) configured to be able to renew them. Returns null
+ * on an insecure cluster (i.e. harmless)
+ * @param conf configuration
+ * @param fs filesystem
+ * @param credentials credentials to update
+ * @return a list of all added tokens.
+ * @throws IOException
+ */
+ public static Token<?>[] addRMRenewableFSDelegationTokens(Configuration conf,
+ FileSystem fs,
+ Credentials credentials) throws IOException {
+ Preconditions.checkArgument(conf != null);
+ Preconditions.checkArgument(credentials != null);
+ if (UserGroupInformation.isSecurityEnabled()) {
+ return fs.addDelegationTokens(CredentialUtils.getRMPrincipal(conf),
+ credentials);
+ }
+ return null;
+ }
+
+ /**
+ * Add an FS delegation token which can be renewed by the current user
+ * @param fs filesystem
+ * @param credentials credentials to update
+ * @throws IOException problems.
+ */
+ public static void addSelfRenewableFSDelegationTokens(
+ FileSystem fs,
+ Credentials credentials) throws IOException {
+ Preconditions.checkArgument(fs != null);
+ Preconditions.checkArgument(credentials != null);
+ fs.addDelegationTokens(
+ getSelfRenewer(),
+ credentials);
+ }
+
+ public static String getSelfRenewer() throws IOException {
+ return UserGroupInformation.getLoginUser().getShortUserName();
+ }
+
+ /**
+ * Create and add an RM delegation token to the credentials
+ * @param yarnClient Yarn Client
+ * @param credentials to add token to
+ * @return the token which was added
+ * @throws IOException
+ * @throws YarnException
+ */
+ public static Token<TokenIdentifier> addRMDelegationToken(YarnClient yarnClient,
+ Credentials credentials)
+ throws IOException, YarnException {
+ Configuration conf = yarnClient.getConfig();
+ Text rmPrincipal = new Text(CredentialUtils.getRMPrincipal(conf));
+ Text rmDTService = ClientRMProxy.getRMDelegationTokenService(conf);
+ Token<TokenIdentifier> rmDelegationToken =
+ ConverterUtils.convertFromYarn(
+ yarnClient.getRMDelegationToken(rmPrincipal),
+ rmDTService);
+ credentials.addToken(rmDelegationToken.getService(), rmDelegationToken);
+ return rmDelegationToken;
+ }
+
+ public static Token<TimelineDelegationTokenIdentifier> maybeAddTimelineToken(
+ Configuration conf,
+ Credentials credentials)
+ throws IOException, YarnException {
+ if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
+ LOG.debug("Timeline service enabled -fetching token");
+
+ try(TimelineClient timelineClient = TimelineClient.createTimelineClient()) {
+ timelineClient.init(conf);
+ timelineClient.start();
+ Token<TimelineDelegationTokenIdentifier> token =
+ timelineClient.getDelegationToken(
+ CredentialUtils.getRMPrincipal(conf));
+ credentials.addToken(token.getService(), token);
+ return token;
+ }
+ } else {
+ LOG.debug("Timeline service is disabled");
+ return null;
+ }
+ }
+
+ /**
+ * Filter a list of tokens from a set of credentials
+ * @param credentials credential source (a new credential set os re
+ * @param filter List of tokens to strip out
+ * @return a new, filtered, set of credentials
+ */
+ public static Credentials filterTokens(Credentials credentials,
+ List<Text> filter) {
+ Credentials result = new Credentials(credentials);
+ Iterator<Token<? extends TokenIdentifier>> iter =
+ result.getAllTokens().iterator();
+ while (iter.hasNext()) {
+ Token<? extends TokenIdentifier> token = iter.next();
+ LOG.debug("Token {}", token.getKind());
+ if (filter.contains(token.getKind())) {
+ LOG.debug("Filtering token {}", token.getKind());
+ iter.remove();
+ }
+ }
+ return result;
+ }
+
+ public static String dumpTokens(Credentials credentials, String separator) {
+ ArrayList<Token<? extends TokenIdentifier>> sorted =
+ new ArrayList<>(credentials.getAllTokens());
+ Collections.sort(sorted, new TokenComparator());
+ StringBuilder buffer = new StringBuilder(sorted.size()* 128);
+ for (Token<? extends TokenIdentifier> token : sorted) {
+ buffer.append(tokenToString(token)).append(separator);
+ }
+ return buffer.toString();
+ }
+
+ /**
+ * Create a string for people to look at
+ * @param token token to convert to a string form
+ * @return a printable view of the token
+ */
+ public static String tokenToString(Token<? extends TokenIdentifier> token) {
+ DateFormat df = DateFormat.getDateTimeInstance(
+ DateFormat.SHORT, DateFormat.SHORT);
+ StringBuilder buffer = new StringBuilder(128);
+ buffer.append(token.toString());
+ try {
+ TokenIdentifier ti = token.decodeIdentifier();
+ buffer.append("; ").append(ti);
+ if (ti instanceof AbstractDelegationTokenIdentifier) {
+ // details in human readable form, and compensate for information HDFS DT omits
+ AbstractDelegationTokenIdentifier dt = (AbstractDelegationTokenIdentifier) ti;
+ buffer.append("; Renewer: ").append(dt.getRenewer());
+ buffer.append("; Issued: ")
+ .append(df.format(new Date(dt.getIssueDate())));
+ buffer.append("; Max Date: ")
+ .append(df.format(new Date(dt.getMaxDate())));
+ }
+ } catch (IOException e) {
+ //marshall problem; not ours
+ LOG.debug("Failed to decode {}: {}", token, e, e);
+ }
+ return buffer.toString();
+ }
+
+ /**
+ * Get the expiry time of a token.
+ * @param token token to examine
+ * @return the time in milliseconds after which the token is invalid.
+ * @throws IOException
+ */
+ public static long getTokenExpiryTime(Token token) throws IOException {
+ TokenIdentifier identifier = token.decodeIdentifier();
+ Preconditions.checkState(identifier instanceof AbstractDelegationTokenIdentifier,
+ "Token %s of type: %s has an identifier which cannot be examined: %s",
+ token, token.getClass(), identifier);
+ AbstractDelegationTokenIdentifier id =
+ (AbstractDelegationTokenIdentifier) identifier;
+ return id.getMaxDate();
+ }
+
+ private static class TokenComparator
+ implements Comparator<Token<? extends TokenIdentifier>>, Serializable {
+ @Override
+ public int compare(Token<? extends TokenIdentifier> left,
+ Token<? extends TokenIdentifier> right) {
+ return left.getKind().toString().compareTo(right.getKind().toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
new file mode 100644
index 0000000..cbcb0d6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+
+import java.util.Map;
+
+/**
+ * Command line builder purely for the Java CLI.
+ * Some of the <code>define</code> methods are designed to work with Hadoop tool and
+ * Slider launcher applications.
+ */
+public class JavaCommandLineBuilder extends CommandLineBuilder {
+
+ public JavaCommandLineBuilder() {
+ add(getJavaBinary());
+ }
+
+ /**
+ * Get the java binary. This is called in the constructor so don't try and
+ * do anything other than return a constant.
+ * @return the path to the Java binary
+ */
+ protected String getJavaBinary() {
+ return ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java";
+ }
+
+ /**
+ * Set the size of the heap if a non-empty heap is passed in.
+ * @param heap empty string or something like "128M" ,"1G" etc. The value is
+ * trimmed.
+ */
+ public void setJVMHeap(String heap) {
+ if (SliderUtils.isSet(heap)) {
+ add("-Xmx" + heap.trim());
+ }
+ }
+
+ /**
+ * Turn Java assertions on
+ */
+ public void enableJavaAssertions() {
+ add("-ea");
+ add("-esa");
+ }
+
+ /**
+ * Add a system property definition -must be used before setting the main entry point
+ * @param property
+ * @param value
+ */
+ public void sysprop(String property, String value) {
+ Preconditions.checkArgument(property != null, "null property name");
+ Preconditions.checkArgument(value != null, "null value");
+ add("-D" + property + "=" + value);
+ }
+
+ public JavaCommandLineBuilder forceIPv4() {
+ sysprop("java.net.preferIPv4Stack", "true");
+ return this;
+ }
+
+ public JavaCommandLineBuilder headless() {
+ sysprop("java.awt.headless", "true");
+ return this;
+ }
+
+ public boolean addConfOption(Configuration conf, String key) {
+ return defineIfSet(key, conf.get(key));
+ }
+
+ /**
+ * Add a varargs list of configuration parameters —if they are present
+ * @param conf configuration source
+ * @param keys keys
+ */
+ public void addConfOptions(Configuration conf, String... keys) {
+ for (String key : keys) {
+ addConfOption(conf, key);
+ }
+ }
+
+ /**
+ * Add all configuration options which match the prefix
+ * @param conf configuration
+ * @param prefix prefix, e.g {@code "slider."}
+ * @return the number of entries copied
+ */
+ public int addPrefixedConfOptions(Configuration conf, String prefix) {
+ int copied = 0;
+ for (Map.Entry<String, String> entry : conf) {
+ if (entry.getKey().startsWith(prefix)) {
+ define(entry.getKey(), entry.getValue());
+ copied++;
+ }
+ }
+ return copied;
+ }
+
+ /**
+ * Ass a configuration option to the command line of the application
+ * @param conf configuration
+ * @param key key
+ * @param defVal default value
+ * @return the resolved configuration option
+ * @throws IllegalArgumentException if key is null or the looked up value
+ * is null (that is: the argument is missing and devVal was null.
+ */
+ public String addConfOptionToCLI(Configuration conf,
+ String key,
+ String defVal) {
+ Preconditions.checkArgument(key != null, "null key");
+ String val = conf.get(key, defVal);
+ define(key, val);
+ return val;
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI. This is very Hadoop API
+ * @param key key
+ * @param val value
+ * @throws IllegalArgumentException if either argument is null
+ */
+ public void define(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ Preconditions.checkArgument(val != null, "null value");
+ add("-D", key + "=" + val);
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
+ * is not null
+ * @param key key
+ * @param val value
+ */
+ public boolean defineIfSet(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ if (val != null) {
+ define(key, val);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Add a mandatory config option
+ * @param conf configuration
+ * @param key key
+ * @throws BadConfigException if the key is missing
+ */
+ public void addMandatoryConfOption(Configuration conf,
+ String key) throws BadConfigException {
+ if (!addConfOption(conf, key)) {
+ throw new BadConfigException("Missing configuration option: " + key);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
new file mode 100644
index 0000000..db9de7a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+
+/**
+ * The system is in a bad state
+ */
+public class BadClusterStateException extends SliderException {
+ public BadClusterStateException(String message,
+ Object... args) {
+ super(EXIT_BAD_STATE, message, args);
+ }
+
+ public BadClusterStateException(Throwable throwable,
+ String message, Object... args) {
+ super(EXIT_BAD_STATE, throwable, message, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
new file mode 100644
index 0000000..41e3251
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+public class BadCommandArgumentsException extends SliderException {
+ public BadCommandArgumentsException(String s, Object... args) {
+ super(EXIT_COMMAND_ARGUMENT_ERROR, s, args);
+ }
+
+ public BadCommandArgumentsException(Throwable throwable, String message,
+ Object... args) {
+ super(EXIT_COMMAND_ARGUMENT_ERROR, throwable, message, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
new file mode 100644
index 0000000..8199c3c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+/**
+ * An exception to raise on a bad configuration
+ */
+public class BadConfigException extends SliderException {
+
+ public BadConfigException(String s) {
+ super(EXIT_BAD_CONFIGURATION, s);
+ }
+
+ public BadConfigException(String message, Object... args) {
+ super(EXIT_BAD_CONFIGURATION, message, args);
+ }
+
+ public BadConfigException(
+ Throwable throwable,
+ String message, Object... args) {
+ super(EXIT_BAD_CONFIGURATION, throwable, message, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
new file mode 100644
index 0000000..3577b59
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+public interface ErrorStrings {
+ String E_UNSTABLE_CLUSTER = "Unstable Application Instance :";
+ String E_CLUSTER_RUNNING = "Application Instance running";
+ String E_ALREADY_EXISTS = "already exists";
+ String PRINTF_E_INSTANCE_ALREADY_EXISTS = "Application Instance \"%s\" already exists and is defined in %s";
+ String PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS = "Application Instance dir already exists: %s";
+ String E_MISSING_PATH = "Missing path ";
+ String E_INCOMPLETE_CLUSTER_SPEC =
+ "Cluster specification is marked as incomplete: ";
+ String E_UNKNOWN_INSTANCE = "Unknown application instance ";
+ String E_DESTROY_CREATE_RACE_CONDITION =
+ "created while it was being destroyed";
+ String E_UNKNOWN_ROLE = "Unknown role ";
+ /**
+ * ERROR Strings
+ */
+ String ERROR_NO_ACTION = "No action specified";
+ String ERROR_UNKNOWN_ACTION = "Unknown command: ";
+ String ERROR_NOT_ENOUGH_ARGUMENTS =
+ "Not enough arguments for action: ";
+ String ERROR_PARSE_FAILURE =
+ "Failed to parse ";
+ /**
+ * All the remaining values after argument processing
+ */
+ String ERROR_TOO_MANY_ARGUMENTS =
+ "Too many arguments";
+ String ERROR_DUPLICATE_ENTRY = "Duplicate entry for ";
+ String E_APPLICATION_NOT_RUNNING = "Application not running";
+ String E_FINISHED_APPLICATION = E_APPLICATION_NOT_RUNNING + ": %s state=%s ";
+ String E_NO_IMAGE_OR_HOME_DIR_SPECIFIED =
+ "Neither an image path nor binary home directory were specified";
+ String E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED =
+ "Both application image path and home dir have been provided";
+ String E_CONFIGURATION_DIRECTORY_NOT_FOUND =
+ "Configuration directory \"%s\" not found";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
new file mode 100644
index 0000000..d66b860
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+/**
+ * Get the exit code of an exception. Making it an interface allows
+ * us to retrofit exit codes onto existing classes
+ */
+public interface ExitCodeProvider {
+
+ /**
+ * Method to get the exit code
+ * @return the exit code
+ */
+ int getExitCode();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
new file mode 100644
index 0000000..9657536
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+/*
+ * Common Exit codes
+ * <p>
+ * Exit codes from 64 up are application specific.
+ * <p>
+ * Many of the exit codes are designed to resemble HTTP error codes,
+ * squashed into a single byte. e.g 44 , "not found" is the equivalent
+ * of 404
+ * <pre>
+ * 0-10: general command issues
+ * 30-39: equivalent to the 3XX responses, where those responses are
+ * considered errors by the application.
+ * 40-49: request-related errors
+ * 50-59: server-side problems. These may be triggered by the request.
+ * 64- : application specific error codes
+ * </pre>
+ */
+public interface LauncherExitCodes {
+
+ /**
+ * 0: success
+ */
+ int EXIT_SUCCESS = 0;
+
+ /**
+ * -1: generic "false" response. The operation worked but
+ * the result was not true
+ */
+ int EXIT_FALSE = -1;
+
+ /**
+ * Exit code when a client requested service termination: {@value}
+ */
+ int EXIT_CLIENT_INITIATED_SHUTDOWN = 1;
+
+ /**
+ * Exit code when targets could not be launched: {@value}
+ */
+ int EXIT_TASK_LAUNCH_FAILURE = 2;
+
+ /**
+ * Exit code when a control-C, kill -3, signal was picked up: {@value}
+ */
+ int EXIT_INTERRUPTED = 3;
+
+ /**
+ * Exit code when a usage message was printed: {@value}
+ */
+ int EXIT_USAGE = 4;
+
+ /**
+ * Exit code when something happened but we can't be specific: {@value}
+ */
+ int EXIT_OTHER_FAILURE = 5;
+
+ /**
+ * Exit code on connectivity problems: {@value}
+ */
+ int EXIT_MOVED = 31;
+
+ /**
+ * found: {@value}.
+ * <p>
+ * This is low value as in HTTP it is normally a success/redirect;
+ * whereas on the command line 0 is the sole success code.
+ * <p>
+ * <code>302 Found</code>
+ */
+ int EXIT_FOUND = 32;
+
+ /**
+ * Exit code on a request where the destination has not changed
+ * and (somehow) the command specified that this is an error.
+ * That is, this exit code is somehow different from a "success"
+ * : {@value}
+ * <p>
+ * <code>304 Not Modified </code>
+ */
+ int EXIT_NOT_MODIFIED = 34;
+
+ /**
+ * Exit code when the command line doesn't parse: {@value}, or
+ * when it is otherwise invalid.
+ * <p>
+ * <code>400 BAD REQUEST</code>
+ */
+ int EXIT_COMMAND_ARGUMENT_ERROR = 40;
+
+ /**
+ * The request requires user authentication: {@value}
+ * <p>
+ * <code>401 Unauthorized</code>
+ */
+ int EXIT_UNAUTHORIZED = 41;
+
+ /**
+ * Forbidden action: {@value}
+ * <p>
+ * <code>403: Forbidden</code>
+ */
+ int EXIT_FORBIDDEN = 43;
+
+ /**
+ * Something was not found: {@value}
+ * <p>
+ * <code>404: NOT FOUND</code>
+ */
+ int EXIT_NOT_FOUND = 44;
+
+ /**
+ * The operation is not allowed: {@value}
+ * <p>
+ * <code>405: NOT ALLOWED</code>
+ */
+ int EXIT_OPERATION_NOT_ALLOWED = 45;
+
+ /**
+ * The command is somehow not acceptable: {@value}
+ * <p>
+ * <code>406: NOT ACCEPTABLE</code>
+ */
+ int EXIT_NOT_ACCEPTABLE = 46;
+
+ /**
+ * Exit code on connectivity problems: {@value}
+ * <p>
+ * <code>408: Request Timeout</code>
+ */
+ int EXIT_CONNECTIVITY_PROBLEM = 48;
+
+ /**
+ * The request could not be completed due to a conflict with the current
+ * state of the resource. {@value}
+ * <p>
+ * <code>409: conflict</code>
+ */
+ int EXIT_CONFLICT = 49;
+
+ /**
+ * internal error: {@value}
+ * <p>
+ * <code>500 Internal Server Error</code>
+ */
+ int EXIT_INTERNAL_ERROR = 50;
+
+ /**
+ * Unimplemented feature: {@value}
+ * <p>
+ * <code>501: Not Implemented</code>
+ */
+ int EXIT_UNIMPLEMENTED = 51;
+
+ /**
+ * Service Unavailable; it may be available later: {@value}
+ * <p>
+ * <code>503 Service Unavailable</code>
+ */
+ int EXIT_SERVICE_UNAVAILABLE = 53;
+
+ /**
+ * The application does not support, or refuses to support this version: {@value}.
+ * If raised, this is expected to be raised server-side and likely due
+ * to client/server version incompatibilities.
+ * <p>
+ * <code> 505: Version Not Supported</code>
+ */
+ int EXIT_UNSUPPORTED_VERSION = 55;
+
+ /**
+ * Exit code when an exception was thrown from the service: {@value}
+ * <p>
+ * <code>5XX</code>
+ */
+ int EXIT_EXCEPTION_THROWN = 56;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
new file mode 100644
index 0000000..7be23f3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+public interface RestApiErrorMessages {
+ String ERROR_APPLICATION_NAME_INVALID =
+ "Application name is either empty or not provided";
+ String ERROR_APPLICATION_NAME_INVALID_FORMAT =
+ "Application name %s is not valid - only lower case letters, digits, " +
+ "underscore and hyphen are allowed, and the name must be no more " +
+ "than 63 characters";
+ String ERROR_COMPONENT_NAME_INVALID =
+ "Component name must be no more than %s characters: %s";
+ String ERROR_USER_NAME_INVALID =
+ "User name must be no more than 63 characters";
+
+ String ERROR_APPLICATION_NOT_RUNNING = "Application not running";
+ String ERROR_APPLICATION_DOES_NOT_EXIST = "Application not found";
+ String ERROR_APPLICATION_IN_USE = "Application already exists in started"
+ + " state";
+ String ERROR_APPLICATION_INSTANCE_EXISTS = "Application already exists in"
+ + " stopped/failed state (either restart with PUT or destroy with DELETE"
+ + " before creating a new one)";
+
+ String ERROR_SUFFIX_FOR_COMPONENT =
+ " for component %s (nor at the global level)";
+ String ERROR_ARTIFACT_INVALID = "Artifact is not provided";
+ String ERROR_ARTIFACT_FOR_COMP_INVALID =
+ ERROR_ARTIFACT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_ARTIFACT_ID_INVALID =
+ "Artifact id (like docker image name) is either empty or not provided";
+ String ERROR_ARTIFACT_ID_FOR_COMP_INVALID =
+ ERROR_ARTIFACT_ID_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+
+ String ERROR_RESOURCE_INVALID = "Resource is not provided";
+ String ERROR_RESOURCE_FOR_COMP_INVALID =
+ ERROR_RESOURCE_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_MEMORY_INVALID =
+ "Application resource or memory not provided";
+ String ERROR_RESOURCE_CPUS_INVALID =
+ "Application resource or cpus not provided";
+ String ERROR_RESOURCE_CPUS_INVALID_RANGE =
+ "Unacceptable no of cpus specified, either zero or negative";
+ String ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID =
+ ERROR_RESOURCE_MEMORY_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID =
+ ERROR_RESOURCE_CPUS_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE =
+ ERROR_RESOURCE_CPUS_INVALID_RANGE
+ + " for component %s (or at the global level)";
+ String ERROR_CONTAINERS_COUNT_INVALID =
+ "Invalid no of containers specified";
+ String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID =
+ ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_DEPENDENCY_INVALID = "Dependency %s for component %s is " +
+ "invalid, does not exist as a component";
+ String ERROR_DEPENDENCY_CYCLE = "Invalid dependencies, a cycle may " +
+ "exist: %s";
+
+ String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED =
+ "Cannot specify" + " cpus/memory along with profile";
+ String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED =
+ ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED
+ + " for component %s";
+ String ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET =
+ "Resource profile is not " + "supported yet. Please specify cpus/memory.";
+
+ String ERROR_NULL_ARTIFACT_ID =
+ "Artifact Id can not be null if artifact type is none";
+ String ERROR_ABSENT_NUM_OF_INSTANCE =
+ "Num of instances should appear either globally or per component";
+ String ERROR_ABSENT_LAUNCH_COMMAND =
+ "Launch_command is required when type is not DOCKER";
+
+ String ERROR_QUICKLINKS_FOR_COMP_INVALID = "Quicklinks specified at"
+ + " component level, needs corresponding values set at application level";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
new file mode 100644
index 0000000..e83ccbe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * A service launch exception that includes an exit code;
+ * when caught by the ServiceLauncher, it will convert that
+ * into a process exit code.
+ */
+public class ServiceLaunchException extends YarnException
+ implements ExitCodeProvider, LauncherExitCodes {
+
+ private final int exitCode;
+
+ /**
+ * Create an exception with the specific exit code
+ * @param exitCode exit code
+ * @param cause cause of the exception
+ */
+ public ServiceLaunchException(int exitCode, Throwable cause) {
+ super(cause);
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * Create an exception with the specific exit code and text
+ * @param exitCode exit code
+ * @param message message to use in exception
+ */
+ public ServiceLaunchException(int exitCode, String message) {
+ super(message);
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * Create an exception with the specific exit code, text and cause
+ * @param exitCode exit code
+ * @param message message to use in exception
+ * @param cause cause of the exception
+ */
+ public ServiceLaunchException(int exitCode, String message, Throwable cause) {
+ super(message, cause);
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * Get the exit code
+ * @return the exit code
+ */
+ @Override
+ public int getExitCode() {
+ return exitCode;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
new file mode 100644
index 0000000..5b74b80
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
+
+public class SliderException extends ServiceLaunchException implements
+ SliderExitCodes {
+ public SliderException() {
+ super(EXIT_EXCEPTION_THROWN, "SliderException");
+ }
+
+ public SliderException(int code, String message) {
+ super(code, message);
+ }
+
+ public SliderException(String s) {
+ super(EXIT_EXCEPTION_THROWN, s);
+ }
+
+ public SliderException(String s, Throwable throwable) {
+ super(EXIT_EXCEPTION_THROWN, s, throwable);
+ }
+
+ /**
+ * Format the exception as you create it
+ * @param code exit code
+ * @param message exception message -sprintf formatted
+ * @param args arguments for the formatting
+ */
+ public SliderException(int code, String message, Object... args) {
+ super(code, String.format(message, args));
+ }
+
+ /**
+ * Format the exception, include a throwable.
+ * The throwable comes before the message so that it is out of the varargs
+ * @param code exit code
+ * @param throwable thrown
+ * @param message message
+ * @param args arguments
+ */
+ public SliderException(int code,
+ Throwable throwable,
+ String message,
+ Object... args) {
+ super(code, String.format(message, args), throwable);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
new file mode 100644
index 0000000..3a9fa25
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+/**
+ * Used to raise a usage exception ... this has the exit code
+ * {@link #EXIT_USAGE}
+ */
+public class UsageException extends SliderException {
+ public UsageException(String s, Object... args) {
+ super(EXIT_USAGE, s, args);
+ }
+
+ public UsageException(Throwable throwable, String message,
+ Object... args) {
+ super(EXIT_USAGE, throwable, message, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
index 6c91a13..0d11be2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
@@ -19,18 +19,12 @@
package org.apache.hadoop.yarn.service.provider;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.SliderException;
-import org.codehaus.jettison.json.JSONObject;
-
-import java.io.File;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
import java.io.IOException;
import java.nio.file.Paths;
import java.util.HashSet;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
index 6f9f5175..504680d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -20,14 +20,14 @@ package org.apache.hadoop.yarn.service.provider;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.AbstractLauncher;
-import org.apache.slider.core.launch.CommandLineBuilder;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import org.apache.hadoop.yarn.service.ServiceContext;
import org.slf4j.Logger;
@@ -42,7 +42,7 @@ import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_RETR
import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
public abstract class AbstractProviderService implements ProviderService,
- SliderKeys {
+ YarnServiceConstants {
protected static final Logger log =
LoggerFactory.getLogger(AbstractProviderService.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
index b53652a..83c9961 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
@@ -19,8 +19,7 @@
package org.apache.hadoop.yarn.service.provider;
import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.core.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
index 306620d..9ef0176 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
@@ -19,10 +19,10 @@
package org.apache.hadoop.yarn.service.provider;
import org.apache.hadoop.conf.Configuration;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import java.io.IOException;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/86] [abbrv] hadoop git commit: YARN-5244. Documentation required
for DNS Server implementation. Contributed by Jon Maron
Posted by ji...@apache.org.
YARN-5244. Documentation required for DNS Server implementation. Contributed by Jon Maron
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/538c63ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/538c63ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/538c63ef
Branch: refs/heads/yarn-native-services
Commit: 538c63ef0634f197c622a4b284310768a7425ae5
Parents: 727e6d7
Author: Gour Saha <go...@apache.org>
Authored: Thu Aug 24 19:05:13 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 25 16:37:22 2017 -0700
----------------------------------------------------------------------
hadoop-project/src/site/site.xml | 7 +-
.../native-services/NativeServicesDiscovery.md | 127 +++++++++++++++++++
.../native-services/NativeServicesIntro.md | 0
.../src/site/resources/images/dns_overview.png | Bin 0 -> 41908 bytes
.../resources/images/dns_record_creation.jpeg | Bin 0 -> 51911 bytes
.../resources/images/dns_record_removal.jpeg | Bin 0 -> 58041 bytes
6 files changed, 133 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/538c63ef/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index a88f0e3..e6e9d86 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -151,7 +151,12 @@
<item name="Timeline Server" href="hadoop-yarn/hadoop-yarn-site/TimelineServer.html#Timeline_Server_REST_API_v1"/>
<item name="Timeline Service V.2" href="hadoop-yarn/hadoop-yarn-site/TimelineServiceV2.html#Timeline_Service_v.2_REST_API"/>
</menu>
-
+
+ <menu name="YARN Native Services" inherit="top">
+ <item name="Introduction" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesIntro.html"/>
+ <item name="Native Services Discovery" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesDiscovery.html"/>
+ </menu>
+
<menu name="Hadoop Compatible File Systems" inherit="top">
<item name="Aliyun OSS" href="hadoop-aliyun/tools/hadoop-aliyun/index.html"/>
<item name="Amazon S3" href="hadoop-aws/tools/hadoop-aws/index.html"/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/538c63ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
new file mode 100644
index 0000000..4a048af
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
@@ -0,0 +1,127 @@
+# YARN DNS Server
+## Introduction
+
+The YARN DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions:
+
+1. **Exposing existing service discovery information via DNS** - Information provided in
+the current YARN service registry’s records will be converted into DNS entries, thus
+allowing users to discover information about YARN applications using standard DNS
+client mechanisms (for e.g. a DNS SRV Record specifying the hostname and port
+number for services).
+2. **Enabling Container to IP mappings** - Enables discovery of the IPs of containers via
+standard DNS lookups. Given the availability of the records via DNS, container
+namebased communication will be facilitated (e.g. ‘curl
+http://myContainer.myDomain.com/endpoint’).
+
+## Service Properties
+
+The existing YARN Service Registry is leveraged as the source of information for the DNS Service.
+
+The following core functions are supported by the DNS Server:
+
+###Functional properties
+
+1. Supports creation of DNS records for endpoints of the deployed YARN applications
+2. Record names remain unchanged during restart of containers and/or applications
+3. Supports reverse lookups (name based on IP).
+4. Supports security using the standards defined by The Domain Name System Security
+Extensions (DNSSEC)
+5. Highly available
+6. Scalable - The service provides the responsiveness (e.g. low latency) required to
+respond to DNS queries (timeouts yield attempts to invoke other configured name
+servers).
+
+###Deployment properties
+
+1. Supports integration with existing DNS assets (e.g. a corporate DNS server) by acting as
+a DNS server for a Hadoop cluster zone/domain. The server is not intended to act as a
+primary DNS server and does not forward requests to other servers.
+2. The DNS Server exposes a port that can receive both TCP and UDP requests per
+DNS standards. The default port for DNS protocols is in a restricted, administrative port
+range (53), so the port is configurable for deployments in which the service may
+not be managed via an administrative account.
+
+##DNS Record Name Structure
+
+The DNS names of generated records are composed from the following elements (labels). Note that these elements must be compatible with DNS conventions (see “Preferred Name Syntax” in RFC 1035):
+
+* **domain** - the name of the cluster DNS domain. This name is provided as a
+configuration property. In addition, it is this name that is configured at a parent DNS
+server as the zone name for the defined yDNS zone (the zone for which the parent DNS
+server will forward requests to yDNS). E.g. yarncluster.com
+* **username** - the name of the application deployer. This name is the simple shortname (for
+e.g. the primary component of the Kerberos principal) associated with the user launching
+the application. As the username is one of the elements of DNS names, it is expected
+that this also confirms DNS name conventions (RFC 1035 linked above) so special translation is performed for names with special characters like hyphens and spaces.
+* **application name** - the name of the deployed YARN application. This name is inferred
+from the YARN registry path to the application's node. Application name, rather thn application id, was chosen as a way of making it easy for users to refer to human-readable DNS
+names. This obviously mandates certain uniqueness properties on application names.
+* **container id** - the YARN assigned ID to a container (e.g.
+container_e3741_1454001598828_01_000004)
+* **component name** - the name assigned to the deployed component (for e.g. a master
+component). A component is a distributed element of an application or service that is
+launched in a YARN container (e.g. an HBase master). One can imagine multiple
+components within an application. A component name is not yet a first class concept in
+YARN, but is a very useful one that we are introducing here for the sake of yDNS
+entries. Many frameworks like MapReduce, Slider already have component names
+(though, as mentioned, they are not yet supported in YARN in a first class fashion).
+* **api** - the api designation for the exposed endpoint
+
+###Notes about DNS Names
+
+* In most instances, the DNS names can be easily distinguished by the number of
+elements/labels that compose the name. The cluster’s domain name is always the last
+element. After that element is parsed out, reading from right to left, the first element
+maps to the application user and so on. Wherever it is not easily distinguishable, naming conventions are used to disambiguate the name using a prefix such as
+“container” or suffix such as “api”. For example, an endpoint published as a
+management endpoint will be referenced with the name *management-api.griduser.yarncluster.com*.
+* Unique application name (per user) is not currently supported/guaranteed by YARN, but
+it is supported by frameworks such as Apache Slider. The yDNS service currently
+leverages the last element of the ZK path entry for the application as an
+application name. These application names have to be unique for a given user.
+
+##DNS Server Functionality
+
+The primary functions of the DNS service are illustrated in the following diagram:
+
+
+
+
+###DNS record creation
+The following figure illustrates at slightly greater detail the DNS record creation and registration sequence (NOTE: service record updates would follow a similar sequence of steps,
+distinguished only by the different event type):
+
+
+
+###DNS record removal
+Similarly, record removal follows a similar sequence
+
+
+
+(NOTE: The DNS Zone requires a record as an argument for the deletion method, thus
+requiring similar parsing logic to identify the specific records that should be removed).
+
+###DNS Service initialization
+* The DNS service initializes both UDP and TCP listeners on a configured port. As
+noted above, the default port of 53 is in a restricted range that is only accessible to an
+account with administrative privileges.
+* Subsequently, the DNS service listens for inbound DNS requests. Those requests are
+standard DNS requests from users or other DNS servers (for example, DNS servers that have the
+YARN DNS service configured as a forwarder).
+
+## Configuration
+The YARN DNS server reads its configuration properties from the yarnsite.xml file. The following are the DNS associated configuration properties:
+
+| Name | Description |
+| ------------ | ------------- |
+| hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. |
+| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. |
+| hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. |
+| hadoop.registry.dns.bind-port | The port number for the DNS listener. The default port is 53. However, since that port falls in a administratoronly range, typical deployments may need to specify an alternate port. |
+| hadoop.registry.dns.dnssec.enabled | Indicates whether the DNSSEC support is enabled. Default is false. |
+| hadoop.registry.dns.public-key | The base64 representation of the server’s public key. Leveraged for creating the DNSKEY Record provided for DNSSEC client requests. |
+| hadoop.registry.dns.private-key-file | The path to the standard DNSSEC private key file. Must only be readable by the DNS launching identity. See [dnssec-keygen](https://ftp.isc.org/isc/bind/cur/9.9/doc/arm/man.dnssec-keygen.html) documentation. |
+| hadoop.registry.dns-ttl | The default TTL value to associate with DNS records. The default value is set to 1 (a value of 0 has undefined behavior). A typical value should be approximate to the time it takes YARN to restart a failed container. |
+| hadoop.registry.dns.zone-subnet | An indicator of the IP range associated with the cluster containers. The setting is utilized for the generation of the reverse zone name. |
+| hadoop.registry.dns.zone-mask | The network mask associated with the zone IP range. If specified, it is utilized to ascertain the IP range possible and come up with an appropriate reverse zone name. |
+| hadoop.registry.dns.zones-dir | A directory containing zone configuration files to read during zone initialization. This directory can contain zone master files named *zone-name.zone*. See [here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file documentation.|
http://git-wip-us.apache.org/repos/asf/hadoop/blob/538c63ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/hadoop/blob/538c63ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/dns_overview.png
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/dns_overview.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/dns_overview.png
new file mode 100644
index 0000000..b9e80b9
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/dns_overview.png differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/538c63ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/dns_record_creation.jpeg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/dns_record_creation.jpeg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/dns_record_creation.jpeg
new file mode 100644
index 0000000..63b2599
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/dns_record_creation.jpeg differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/538c63ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/dns_record_removal.jpeg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/dns_record_removal.jpeg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/dns_record_removal.jpeg
new file mode 100644
index 0000000..40d870c
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/dns_record_removal.jpeg differ
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
new file mode 100644
index 0000000..c0712c3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
@@ -0,0 +1,680 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SaslPropertiesResolver;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.crypto.Cipher;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.lang.reflect.InvocationTargetException;
+import java.net.InetAddress;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.security.UserGroupInformation.*;
+import static org.apache.hadoop.security.authentication.util.KerberosUtil.*;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
+
+/**
+ * Kerberos diagnostics
+ * At some point this may move to hadoop core, so please keep use of slider
+ * methods and classes to ~0.
+ *
+ * This operation expands some of the diagnostic output of the security code,
+ * but not all. For completeness
+ *
+ * Set the environment variable {@code HADOOP_JAAS_DEBUG=true}
+ * Set the log level for {@code org.apache.hadoop.security=DEBUG}
+ */
+public class KerberosDiags implements Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KerberosDiags.class);
+ public static final String KRB5_CCNAME = "KRB5CCNAME";
+ public static final String JAVA_SECURITY_KRB5_CONF
+ = "java.security.krb5.conf";
+ public static final String JAVA_SECURITY_KRB5_REALM
+ = "java.security.krb5.realm";
+ public static final String SUN_SECURITY_KRB5_DEBUG
+ = "sun.security.krb5.debug";
+ public static final String SUN_SECURITY_SPNEGO_DEBUG
+ = "sun.security.spnego.debug";
+ public static final String SUN_SECURITY_JAAS_FILE
+ = "java.security.auth.login.config";
+ public static final String KERBEROS_KINIT_COMMAND
+ = "hadoop.kerberos.kinit.command";
+ public static final String HADOOP_AUTHENTICATION_IS_DISABLED
+ = "Hadoop authentication is disabled";
+ public static final String UNSET = "(unset)";
+ public static final String NO_DEFAULT_REALM = "Cannot locate default realm";
+
+ private final Configuration conf;
+ private final List<String> services;
+ private final PrintStream out;
+ private final File keytab;
+ private final String principal;
+ private final long minKeyLength;
+ private final boolean securityRequired;
+
+ public static final String CAT_JVM = "JVM";
+ public static final String CAT_JAAS = "JAAS";
+ public static final String CAT_CONFIG = "CONFIG";
+ public static final String CAT_LOGIN = "LOGIN";
+ public static final String CAT_KERBEROS = "KERBEROS";
+ public static final String CAT_SASL = "SASL";
+
+ @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
+ public KerberosDiags(Configuration conf,
+ PrintStream out,
+ List<String> services,
+ File keytab,
+ String principal,
+ long minKeyLength,
+ boolean securityRequired) {
+ this.conf = conf;
+ this.services = services;
+ this.keytab = keytab;
+ this.principal = principal;
+ this.out = out;
+ this.minKeyLength = minKeyLength;
+ this.securityRequired = securityRequired;
+ }
+
+ @Override
+ public void close() throws IOException {
+ flush();
+ }
+
+ /**
+ * Execute diagnostics.
+ * <p>
+ * Things it would be nice if UGI made accessible
+ * <ol>
+ * <li>A way to enable JAAS debug programatically</li>
+ * <li>Access to the TGT</li>
+ * </ol>
+ * @return true if security was enabled and all probes were successful
+ * @throws KerberosDiagsFailure explicitly raised failure
+ * @throws Exception other security problems
+ */
+ @SuppressWarnings("deprecation")
+ public boolean execute() throws Exception {
+
+ title("Kerberos Diagnostics scan at %s",
+ new Date(System.currentTimeMillis()));
+
+ // check that the machine has a name
+ println("Hostname: %s",
+ InetAddress.getLocalHost().getCanonicalHostName());
+
+ // Fail fast on a JVM without JCE installed.
+ validateKeyLength();
+
+ // look at realm
+ println("JVM Kerberos Login Module = %s", getKrb5LoginModuleName());
+ printDefaultRealm();
+
+ title("System Properties");
+ for (String prop : new String[]{
+ JAVA_SECURITY_KRB5_CONF,
+ JAVA_SECURITY_KRB5_REALM,
+ SUN_SECURITY_KRB5_DEBUG,
+ SUN_SECURITY_SPNEGO_DEBUG,
+ SUN_SECURITY_JAAS_FILE
+ }) {
+ printSysprop(prop);
+ }
+
+ title("Environment Variables");
+ for (String env : new String[]{
+ "HADOOP_JAAS_DEBUG",
+ KRB5_CCNAME,
+ "HADOOP_USER_NAME",
+ "HADOOP_PROXY_USER",
+ HADOOP_TOKEN_FILE_LOCATION,
+ }) {
+ printEnv(env);
+ }
+
+ for (String prop : new String[]{
+ KERBEROS_KINIT_COMMAND,
+ HADOOP_SECURITY_AUTHENTICATION,
+ HADOOP_SECURITY_AUTHORIZATION,
+ "hadoop.kerberos.min.seconds.before.relogin", // not in 2.6
+ "hadoop.security.dns.interface", // not in 2.6
+ "hadoop.security.dns.nameserver", // not in 2.6
+ HADOOP_RPC_PROTECTION,
+ HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
+ HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX,
+ HADOOP_SECURITY_GROUP_MAPPING,
+ "hadoop.security.impersonation.provider.class", // not in 2.6
+ "dfs.data.transfer.protection" // HDFS
+ }) {
+ printConfOpt(prop);
+ }
+
+ // check that authentication is enabled
+ if (SecurityUtil.getAuthenticationMethod(conf)
+ .equals(AuthenticationMethod.SIMPLE)) {
+ println(HADOOP_AUTHENTICATION_IS_DISABLED);
+ failif(securityRequired, CAT_CONFIG, HADOOP_AUTHENTICATION_IS_DISABLED);
+ // no security, skip rest of test
+ return false;
+ }
+
+ validateKrb5File();
+ validateSasl(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS);
+ validateSasl("dfs.data.transfer.saslproperties.resolver.class");
+ validateKinitExecutable();
+ validateJAAS();
+ // now the big test: login, then try again
+ boolean krb5Debug = getAndSet(SUN_SECURITY_KRB5_DEBUG);
+ boolean spnegoDebug = getAndSet(SUN_SECURITY_SPNEGO_DEBUG);
+ try {
+ title("Logging in");
+
+ if (keytab != null) {
+ dumpKeytab(keytab);
+ loginFromKeytab();
+ } else {
+ UserGroupInformation loginUser = getLoginUser();
+ dumpUGI("Log in user", loginUser);
+ validateUGI("Login user", loginUser);
+ println("Ticket based login: %b", isLoginTicketBased());
+ println("Keytab based login: %b", isLoginKeytabBased());
+ }
+
+ return true;
+ } finally {
+ // restore original system properties
+ System.setProperty(SUN_SECURITY_KRB5_DEBUG,
+ Boolean.toString(krb5Debug));
+ System.setProperty(SUN_SECURITY_SPNEGO_DEBUG,
+ Boolean.toString(spnegoDebug));
+ }
+ }
+
+ /**
+ * Fail fast on a JVM without JCE installed.
+ *
+ * This is a recurrent problem
+ * (that is: it keeps creeping back with JVM updates);
+ * a fast failure is the best tactic
+ * @throws NoSuchAlgorithmException
+ */
+
+ protected void validateKeyLength() throws NoSuchAlgorithmException {
+ int aesLen = Cipher.getMaxAllowedKeyLength("AES");
+ println("Maximum AES encryption key length %d bits", aesLen);
+ failif (aesLen < minKeyLength,
+ CAT_JVM,
+ "Java Cryptography Extensions are not installed on this JVM."
+ +" Maximum supported key length %s - minimum required %d",
+ aesLen, minKeyLength);
+ }
+
+ /**
+ * Get the default realm.
+ * <p>
+ * Not having a default realm may be harmless, so is noted at info.
+ * All other invocation failures are downgraded to warn, as
+ * follow-on actions may still work.
+ * failure to invoke the method via introspection is rejected,
+ * as it's a sign of JVM compatibility issues that may have other
+ * consequences
+ */
+ protected void printDefaultRealm() {
+ try {
+ println("Default Realm = %s",
+ getDefaultRealm());
+ } catch (ClassNotFoundException
+ | IllegalAccessException
+ | NoSuchMethodException e) {
+
+ throw new KerberosDiagsFailure(CAT_JVM, e,
+ "Failed to invoke krb5.Config.getDefaultRealm: %s", e);
+ } catch (InvocationTargetException e) {
+ Throwable cause = e.getCause() != null ? e.getCause() : e;
+ if (cause.toString().contains(NO_DEFAULT_REALM)) {
+ // exception raised if there is no default realm. This is not
+ // always a problem, so downgrade to a message.
+ println("Host has no default realm");
+ LOG.debug(cause.toString(), cause);
+ } else {
+ println("Kerberos.getDefaultRealm() failed: %s\n%s",
+ cause,
+ org.apache.hadoop.util.StringUtils.stringifyException(cause));
+ }
+ }
+ }
+
+ /**
+ * Locate the krb5.conf file and dump it.
+ * No-op on windows.
+ * @throws IOException
+ */
+ private void validateKrb5File() throws IOException {
+ if (!Shell.WINDOWS) {
+ title("Locating Kerberos configuration file");
+ String krbPath = "/etc/krb5.conf";
+ String jvmKrbPath = System.getProperty(JAVA_SECURITY_KRB5_CONF);
+ if (jvmKrbPath != null) {
+ println("Setting kerberos path from sysprop %s: %s",
+ JAVA_SECURITY_KRB5_CONF, jvmKrbPath);
+ krbPath = jvmKrbPath;
+ }
+
+ String krb5name = System.getenv(KRB5_CCNAME);
+ if (krb5name != null) {
+ println("Setting kerberos path from environment variable %s: %s",
+ KRB5_CCNAME, krb5name);
+ krbPath = krb5name;
+ if (jvmKrbPath != null) {
+ println("Warning - both %s and %s were set - %s takes priority",
+ JAVA_SECURITY_KRB5_CONF, KRB5_CCNAME, KRB5_CCNAME);
+ }
+ }
+
+ File krbFile = new File(krbPath);
+ println("Kerberos configuration file = %s", krbFile);
+ failif(!krbFile.exists(),
+ CAT_KERBEROS,
+ "Kerberos configuration file %s not found", krbFile);
+ dump(krbFile);
+ }
+ }
+
+ /**
+ * Dump a keytab: list all principals.
+ * @param keytabFile the keytab file
+ * @throws IOException IO problems
+ */
+ public void dumpKeytab(File keytabFile) throws IOException {
+ title("Examining keytab %s", keytabFile);
+ File kt = keytabFile.getCanonicalFile();
+ failif(!kt.exists(), CAT_CONFIG, "Keytab not found: %s", kt);
+ failif(!kt.isFile(), CAT_CONFIG, "Keytab is not a valid file: %s", kt);
+
+ String[] names = getPrincipalNames(keytabFile.getCanonicalPath(),
+ Pattern.compile(".*"));
+ println("keytab entry count: %d", names.length);
+ for (String name : names) {
+ println(" %s", name);
+ }
+ println("-----");
+ }
+
+ /**
+ * Log in from a keytab, dump the UGI, validate it, then try and log in again.
+ * That second-time login catches JVM/Hadoop compatibility problems.
+ * @throws IOException
+ */
+ private void loginFromKeytab() throws IOException {
+ UserGroupInformation ugi;
+ String identity;
+ if (keytab != null) {
+ File kt = keytab.getCanonicalFile();
+ println("Using keytab %s principal %s", kt, principal);
+ identity = principal;
+
+ failif(StringUtils.isEmpty(principal), CAT_KERBEROS,
+ "No principal defined");
+ ugi = loginUserFromKeytabAndReturnUGI(principal, kt.getPath());
+ dumpUGI(identity, ugi);
+ validateUGI(principal, ugi);
+
+ title("Attempting to log in from keytab again");
+ // package scoped -hence the reason why this class must be in the
+ // hadoop.security package
+ setShouldRenewImmediatelyForTests(true);
+ // attempt a new login
+ ugi.reloginFromKeytab();
+ } else {
+ println("No keytab: logging is as current user");
+ }
+ }
+
+ /**
+ * Dump a UGI.
+ * @param title title of this section
+ * @param ugi UGI to dump
+ * @throws IOException
+ */
+ private void dumpUGI(String title, UserGroupInformation ugi)
+ throws IOException {
+ title(title);
+ println("UGI instance = %s", ugi);
+ println("Has kerberos credentials: %b", ugi.hasKerberosCredentials());
+ println("Authentication method: %s", ugi.getAuthenticationMethod());
+ println("Real Authentication method: %s",
+ ugi.getRealAuthenticationMethod());
+ title("Group names");
+ for (String name : ugi.getGroupNames()) {
+ println(name);
+ }
+ title("Credentials");
+ Credentials credentials = ugi.getCredentials();
+ List<Text> secretKeys = credentials.getAllSecretKeys();
+ title("Secret keys");
+ if (!secretKeys.isEmpty()) {
+ for (Text secret: secretKeys) {
+ println("%s", secret);
+ }
+ } else {
+ println("(none)");
+ }
+
+ dumpTokens(ugi);
+ }
+
+ /**
+ * Validate the UGI: verify it is kerberized.
+ * @param messagePrefix message in exceptions
+ * @param user user to validate
+ */
+ private void validateUGI(String messagePrefix, UserGroupInformation user) {
+ failif(!user.hasKerberosCredentials(),
+ CAT_LOGIN, "%s: No kerberos credentials for %s", messagePrefix, user);
+ failif(user.getAuthenticationMethod() == null,
+ CAT_LOGIN, "%s: Null AuthenticationMethod for %s", messagePrefix, user);
+ }
+
+ /**
+ * A cursory look at the {@code kinit} executable.
+ * If it is an absolute path: it must exist with a size > 0.
+ * If it is just a command, it has to be on the path. There's no check
+ * for that -but the PATH is printed out.
+ */
+ private void validateKinitExecutable() {
+ String kinit = conf.getTrimmed(KERBEROS_KINIT_COMMAND, "");
+ if (!kinit.isEmpty()) {
+ File kinitPath = new File(kinit);
+ println("%s = %s", KERBEROS_KINIT_COMMAND, kinitPath);
+ if (kinitPath.isAbsolute()) {
+ failif(!kinitPath.exists(), CAT_KERBEROS,
+ "%s executable does not exist: %s",
+ KERBEROS_KINIT_COMMAND, kinitPath);
+ failif(!kinitPath.isFile(), CAT_KERBEROS,
+ "%s path does not refer to a file: %s",
+ KERBEROS_KINIT_COMMAND, kinitPath);
+ failif(kinitPath.length() == 0, CAT_KERBEROS,
+ "%s file is empty: %s",
+ KERBEROS_KINIT_COMMAND, kinitPath);
+ } else {
+ println("Executable %s is relative -must be on the PATH", kinit);
+ printEnv("PATH");
+ }
+ }
+ }
+
+ /**
+ * Try to load the SASL resolver.
+ * @param saslPropsResolverKey key for the SASL resolver
+ */
+ private void validateSasl(String saslPropsResolverKey) {
+ title("Resolving SASL property %s", saslPropsResolverKey);
+ String saslPropsResolver = conf.getTrimmed(saslPropsResolverKey);
+ try {
+ Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass(
+ saslPropsResolverKey,
+ SaslPropertiesResolver.class, SaslPropertiesResolver.class);
+ println("Resolver is %s", resolverClass);
+ } catch (RuntimeException e) {
+ throw new KerberosDiagsFailure(CAT_SASL, e,
+ "Failed to load %s class %s",
+ saslPropsResolverKey, saslPropsResolver);
+ }
+ }
+
+ /**
+ * Validate any JAAS entry referenced in the {@link #SUN_SECURITY_JAAS_FILE}
+ * property.
+ */
+ private void validateJAAS() {
+ String jaasFilename = System.getProperty(SUN_SECURITY_JAAS_FILE);
+ if (jaasFilename != null) {
+ title("JAAS");
+ File jaasFile = new File(jaasFilename);
+ println("JAAS file is defined in %s: %s",
+ SUN_SECURITY_JAAS_FILE, jaasFile);
+ failif(!jaasFile.exists(), CAT_JAAS,
+ "JAAS file does not exist: %s", jaasFile);
+ failif(!jaasFile.isFile(), CAT_JAAS,
+ "Specified JAAS file is not a file: %s", jaasFile);
+ }
+ }
+
+ /**
+ * Dump all tokens of a user
+ * @param user user
+ */
+ public void dumpTokens(UserGroupInformation user) {
+ Collection<Token<? extends TokenIdentifier>> tokens
+ = user.getCredentials().getAllTokens();
+ title("Token Count: %d", tokens.size());
+ for (Token<? extends TokenIdentifier> token : tokens) {
+ println("Token %s", token.getKind());
+ }
+ }
+
+ /**
+ * Set the System property to true; return the old value for caching
+ * @param sysprop property
+ * @return the previous value
+ */
+ private boolean getAndSet(String sysprop) {
+ boolean old = Boolean.getBoolean(sysprop);
+ System.setProperty(sysprop, "true");
+ return old;
+ }
+
+ /**
+ * Flush all active output channels, including {@Code System.err},
+ * so as to stay in sync with any JRE log messages.
+ */
+ private void flush() {
+ if (out != null) {
+ out.flush();
+ } else {
+ System.out.flush();
+ }
+ System.err.flush();
+ }
+
+ /**
+ * Format and print a line of output.
+ * This goes to any output file, or
+ * is logged at info. The output is flushed before and after, to
+ * try and stay in sync with JRE logging.
+ * @param format format string
+ * @param args any arguments
+ */
+ @VisibleForTesting
+ public void println(String format, Object... args) {
+ println(format(format, args));
+ }
+
+ /**
+ * Print a line of output. This goes to any output file, or
+ * is logged at info. The output is flushed before and after, to
+ * try and stay in sync with JRE logging.
+ * @param msg message string
+ */
+ @VisibleForTesting
+ private void println(String msg) {
+ flush();
+ if (out != null) {
+ out.println(msg);
+ } else {
+ LOG.info(msg);
+ }
+ flush();
+ }
+
+ /**
+ * Print a title entry
+ * @param format format string
+ * @param args any arguments
+ */
+ private void title(String format, Object... args) {
+ println("");
+ println("");
+ String msg = "== " + format(format, args) + " ==";
+ println(msg);
+ println("");
+ }
+
+ /**
+ * Print a system property, or {@link #UNSET} if unset.
+ * @param property property to print
+ */
+ private void printSysprop(String property) {
+ println("%s = \"%s\"", property,
+ System.getProperty(property, UNSET));
+ }
+
+ /**
+ * Print a configuration option, or {@link #UNSET} if unset.
+ * @param option option to print
+ */
+ private void printConfOpt(String option) {
+ println("%s = \"%s\"", option, conf.get(option, UNSET));
+ }
+
+ /**
+ * Print an environment variable's name and value; printing
+ * {@link #UNSET} if it is not set
+ * @param variable environment variable
+ */
+ private void printEnv(String variable) {
+ String env = System.getenv(variable);
+ println("%s = \"%s\"", variable, env != null ? env : UNSET);
+ }
+
+ /**
+ * Dump any file to standard out; add a trailing newline
+ * @param file file to dump
+ * @throws IOException IO problems
+ */
+ public void dump(File file) throws IOException {
+ try (FileInputStream in = new FileInputStream(file)) {
+ for (String line : IOUtils.readLines(in)) {
+ println("%s", line);
+ }
+ }
+ println("");
+ }
+
+ /**
+ * Format and raise a failure
+ *
+ * @param category category for exception
+ * @param message string formatting message
+ * @param args any arguments for the formatting
+ * @throws KerberosDiagsFailure containing the formatted text
+ */
+ private void fail(String category, String message, Object... args)
+ throws KerberosDiagsFailure {
+ throw new KerberosDiagsFailure(category, message, args);
+ }
+
+ /**
+ * Conditional failure with string formatted arguments
+ * @param condition failure condition
+ * @param category category for exception
+ * @param message string formatting message
+ * @param args any arguments for the formatting
+ * @throws KerberosDiagsFailure containing the formatted text
+ * if the condition was met
+ */
+ private void failif(boolean condition,
+ String category,
+ String message,
+ Object... args)
+ throws KerberosDiagsFailure {
+ if (condition) {
+ fail(category, message, args);
+ }
+ }
+
+ /**
+ * Format a string, treating a call where there are no varags values
+ * as a string to pass through unformatted.
+ * @param message message, which is either a format string + args, or
+ * a general string
+ * @param args argument array
+ * @return a string for printing.
+ */
+ public static String format(String message, Object... args) {
+ if (args.length == 0) {
+ return message;
+ } else {
+ return String.format(message, args);
+ }
+ }
+
+ /**
+ * Diagnostics failures return the exit code 41, "unauthorized".
+ *
+ * They have a category, initially for testing: the category can be
+ * validated without having to match on the entire string.
+ */
+ public static class KerberosDiagsFailure extends ExitUtil.ExitException {
+ private final String category;
+
+ public KerberosDiagsFailure(String category, String message) {
+ super(41, category + ": " + message);
+ this.category = category;
+ }
+
+ public KerberosDiagsFailure(String category, String message, Object... args) {
+ this(category, format(message, args));
+ }
+
+ public KerberosDiagsFailure(String category, Throwable throwable,
+ String message, Object... args) {
+ this(category, message, args);
+ initCause(throwable);
+ }
+
+ public String getCategory() {
+ return category;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java
new file mode 100644
index 0000000..6efa880
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import java.util.regex.Pattern;
+
+/**
+ * Utility class to validate strings against a predefined pattern.
+ */
+public class PatternValidator {
+
+ public static final String E_INVALID_NAME =
+ "Invalid name %s does not match the pattern pattern %s ";
+ private final Pattern valid;
+ private final String pattern;
+
+ public PatternValidator(String pattern) {
+ this.pattern = pattern;
+ valid = Pattern.compile(pattern);
+ }
+
+ /**
+ * Validate the name -restricting it to the set defined in
+ * @param name name to validate
+ * @throws IllegalArgumentException if not a valid name
+ */
+ public void validate(String name) {
+ if (!matches(name)) {
+ throw new IllegalArgumentException(
+ String.format(E_INVALID_NAME, name, pattern));
+ }
+ }
+
+ /**
+ * Query to see if the pattern matches
+ * @param name name to validate
+ * @return true if the string matches the pattern
+ */
+ public boolean matches(String name) {
+ return valid.matcher(name).matches();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
new file mode 100644
index 0000000..2dbf37f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * a scanner which can take an input string for a range or scan the lot.
+ */
+public class PortScanner {
+ private static Pattern NUMBER_RANGE = Pattern.compile("^(\\d+)\\s*-\\s*(\\d+)$");
+ private static Pattern SINGLE_NUMBER = Pattern.compile("^\\d+$");
+
+ private List<Integer> remainingPortsToCheck;
+
+ public PortScanner() {
+ }
+
+ public void setPortRange(String input) throws BadConfigException {
+ // first split based on commas
+ Set<Integer> inputPorts= new TreeSet<Integer>();
+ String[] ranges = input.split(",");
+ for ( String range : ranges ) {
+ if (range.trim().isEmpty()) {
+ continue;
+ }
+ Matcher m = SINGLE_NUMBER.matcher(range.trim());
+ if (m.find()) {
+ inputPorts.add(Integer.parseInt(m.group()));
+ continue;
+ }
+ m = NUMBER_RANGE.matcher(range.trim());
+ if (m.find()) {
+ String[] boundaryValues = m.group(0).split("-");
+ int start = Integer.parseInt(boundaryValues[0].trim());
+ int end = Integer.parseInt(boundaryValues[1].trim());
+ if (end < start) {
+ throw new BadConfigException("End of port range is before start: "
+ + range + " in input: " + input);
+ }
+ for (int i = start; i < end + 1; i++) {
+ inputPorts.add(i);
+ }
+ continue;
+ }
+ throw new BadConfigException("Bad port range: " + range + " in input: "
+ + input);
+ }
+ if (inputPorts.size() == 0) {
+ throw new BadConfigException("No ports found in range: " + input);
+ }
+ this.remainingPortsToCheck = new ArrayList<Integer>(inputPorts);
+ }
+
+ public List<Integer> getRemainingPortsToCheck() {
+ return remainingPortsToCheck;
+ }
+
+ public int getAvailablePort() throws SliderException, IOException {
+ if (remainingPortsToCheck != null) {
+ return getAvailablePortViaPortArray();
+ } else {
+ return SliderUtils.getOpenPort();
+ }
+ }
+
+ private int getAvailablePortViaPortArray() throws SliderException {
+ boolean found = false;
+ int availablePort = -1;
+ Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator();
+ while (portsToCheck.hasNext() && !found) {
+ int portToCheck = portsToCheck.next();
+ found = SliderUtils.isPortAvailable(portToCheck);
+ if (found) {
+ availablePort = portToCheck;
+ portsToCheck.remove();
+ }
+ }
+
+ if (availablePort < 0) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "No available ports found in configured range {}",
+ remainingPortsToCheck);
+ }
+
+ return availablePort;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java
new file mode 100644
index 0000000..9d00b3c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * JSON-serializable description of a published key-val configuration.
+ *
+ * The values themselves are not serialized in the external view; they have
+ * to be served up by the far end
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedConfiguration {
+
+ public String description;
+ public long updated;
+
+ public String updatedTime;
+
+ public Map<String, String> entries = new HashMap<>();
+
+ public PublishedConfiguration() {
+ }
+
+ /**
+ * build an empty published configuration
+ * @param description configuration description
+ */
+ public PublishedConfiguration(String description) {
+ this.description = description;
+ }
+
+ /**
+ * Build a configuration from the entries
+ * @param description configuration description
+ * @param entries entries to put
+ */
+ public PublishedConfiguration(String description,
+ Iterable<Map.Entry<String, String>> entries) {
+ this.description = description;
+ putValues(entries);
+ }
+
+ /**
+ * Build a published configuration, using the keys from keysource,
+ * but resolving the values from the value source, via Configuration.get()
+ * @param description configuration description
+ * @param keysource source of keys
+ * @param valuesource source of values
+ */
+ public PublishedConfiguration(String description,
+ Iterable<Map.Entry<String, String>> keysource,
+ Configuration valuesource) {
+ this.description = description;
+ putValues(ConfigHelper.resolveConfiguration(keysource, valuesource));
+ }
+
+
+ /**
+ * Is the configuration empty. This means either that it has not
+ * been given any values, or it is stripped down copy set down over the
+ * wire.
+ * @return true if it is empty
+ */
+ public boolean isEmpty() {
+ return entries.isEmpty();
+ }
+
+
+ public void setUpdated(long updated) {
+ this.updated = updated;
+ this.updatedTime = new Date(updated).toString();
+ }
+
+ public long getUpdated() {
+ return updated;
+ }
+
+ /**
+ * Set the values from an iterable (this includes a Hadoop Configuration
+ * and Java properties object).
+ * Any existing value set is discarded
+ * @param entries entries to put
+ */
+ public void putValues(Iterable<Map.Entry<String, String>> entries) {
+ this.entries = new HashMap<String, String>();
+ for (Map.Entry<String, String> entry : entries) {
+ this.entries.put(entry.getKey(), entry.getValue());
+ }
+
+ }
+
+ /**
+ * Convert to Hadoop XML
+ * @return the configuration as a Hadoop Configuratin
+ */
+ public Configuration asConfiguration() {
+ Configuration conf = new Configuration(false);
+ try {
+ ConfigHelper.addConfigMap(conf, entries, "");
+ } catch (BadConfigException e) {
+ // triggered on a null value; switch to a runtime (and discard the stack)
+ throw new RuntimeException(e.toString());
+ }
+ return conf;
+ }
+
+ public String asConfigurationXML() throws IOException {
+ return ConfigHelper.toXml(asConfiguration());
+ }
+
+ /**
+ * Convert values to properties
+ * @return a property file
+ */
+ public Properties asProperties() {
+ Properties props = new Properties();
+ props.putAll(entries);
+ return props;
+ }
+
+ /**
+ * Return the values as json string
+ * @return the JSON representation
+ * @throws IOException marshalling failure
+ */
+ public String asJson() throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
+ String json = mapper.writeValueAsString(entries);
+ return json;
+ }
+
+
+ /**
+ * This makes a copy without the nested content -so is suitable
+ * for returning as part of the list of a parent's values
+ * @return the copy
+ */
+ public PublishedConfiguration shallowCopy() {
+ PublishedConfiguration that = new PublishedConfiguration();
+ that.description = this.description;
+ that.updated = this.updated;
+ that.updatedTime = this.updatedTime;
+ return that;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("PublishedConfiguration{");
+ sb.append("description='").append(description).append('\'');
+ sb.append(" entries = ").append(entries.size());
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Create an outputter for a given format
+ * @param format format to use
+ * @return an instance of output
+ */
+ public PublishedConfigurationOutputter createOutputter(ConfigFormat format) {
+ return PublishedConfigurationOutputter.createOutputter(format, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
new file mode 100644
index 0000000..88ecf2c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.DumperOptions.FlowStyle;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.StringWriter;
+import java.util.Properties;
+
+/**
+ * Output a published configuration
+ */
+public abstract class PublishedConfigurationOutputter {
+
+ private static final String COMMENTS = "Generated by Apache Slider";
+
+ protected final PublishedConfiguration owner;
+
+ protected PublishedConfigurationOutputter(PublishedConfiguration owner) {
+ this.owner = owner;
+ }
+
+ /**
+ * Save the config to a destination file, in the format of this outputter
+ * @param dest destination file
+ * @throws IOException
+ */
+/* JDK7
+ public void save(File dest) throws IOException {
+ try(FileOutputStream out = new FileOutputStream(dest)) {
+ save(out);
+ out.close();
+ }
+ }
+*/
+ public void save(File dest) throws IOException {
+ FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
+ }
+
+ /**
+ * Save the content. The default saves the asString() value
+ * to the output stream
+ * @param out output stream
+ * @throws IOException
+ */
+ public void save(OutputStream out) throws IOException {
+ IOUtils.write(asString(), out, Charsets.UTF_8);
+ }
+ /**
+ * Convert to a string
+ * @return the string form
+ * @throws IOException
+ */
+ public abstract String asString() throws IOException;
+
+ /**
+ * Create an outputter for the chosen format
+ * @param format format enumeration
+ * @param owner owning config
+ * @return the outputter
+ */
+
+ public static PublishedConfigurationOutputter createOutputter(ConfigFormat format,
+ PublishedConfiguration owner) {
+ Preconditions.checkNotNull(owner);
+ switch (format) {
+ case XML:
+ case HADOOP_XML:
+ return new XmlOutputter(owner);
+ case PROPERTIES:
+ return new PropertiesOutputter(owner);
+ case JSON:
+ return new JsonOutputter(owner);
+ case ENV:
+ return new EnvOutputter(owner);
+ case TEMPLATE:
+ return new TemplateOutputter(owner);
+ case YAML:
+ return new YamlOutputter(owner);
+ default:
+ throw new RuntimeException("Unsupported format :" + format);
+ }
+ }
+
+ public static class XmlOutputter extends PublishedConfigurationOutputter {
+
+
+ private final Configuration configuration;
+
+ public XmlOutputter(PublishedConfiguration owner) {
+ super(owner);
+ configuration = owner.asConfiguration();
+ }
+
+ @Override
+ public void save(OutputStream out) throws IOException {
+ configuration.writeXml(out);
+ }
+
+ @Override
+ public String asString() throws IOException {
+ return ConfigHelper.toXml(configuration);
+ }
+
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+ }
+
+ public static class PropertiesOutputter extends PublishedConfigurationOutputter {
+
+ private final Properties properties;
+
+ public PropertiesOutputter(PublishedConfiguration owner) {
+ super(owner);
+ properties = owner.asProperties();
+ }
+
+ @Override
+ public void save(OutputStream out) throws IOException {
+ properties.store(out, COMMENTS);
+ }
+
+
+ public String asString() throws IOException {
+ StringWriter sw = new StringWriter();
+ properties.store(sw, COMMENTS);
+ return sw.toString();
+ }
+ }
+
+
+ public static class JsonOutputter extends PublishedConfigurationOutputter {
+
+ public JsonOutputter(PublishedConfiguration owner) {
+ super(owner);
+ }
+
+ @Override
+ public String asString() throws IOException {
+ return owner.asJson();
+ }
+ }
+
+
+ public static class EnvOutputter extends PublishedConfigurationOutputter {
+
+ public EnvOutputter(PublishedConfiguration owner) {
+ super(owner);
+ }
+
+ @Override
+ public String asString() throws IOException {
+ if (!owner.entries.containsKey("content")) {
+ throw new IOException("Configuration has no content field and cannot " +
+ "be retrieved as type 'env'");
+ }
+ String content = owner.entries.get("content");
+ return ConfigUtils.replaceProps(owner.entries, content);
+ }
+ }
+
+ public static class TemplateOutputter extends EnvOutputter {
+ public TemplateOutputter(PublishedConfiguration owner) {
+ super(owner);
+ }
+ }
+
+ public static class YamlOutputter extends PublishedConfigurationOutputter {
+
+ private final Yaml yaml;
+
+ public YamlOutputter(PublishedConfiguration owner) {
+ super(owner);
+ DumperOptions options = new DumperOptions();
+ options.setDefaultFlowStyle(FlowStyle.BLOCK);
+ yaml = new Yaml(options);
+ }
+
+ public String asString() throws IOException {
+ return yaml.dump(owner.entries);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
new file mode 100644
index 0000000..405f690
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.service.utils.ApplicationReportSerDeser;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+
+/**
+ * Serialized form of an application report which can be persisted
+ * and then parsed. It can not be converted back into a
+ * real YARN application report
+ *
+ * Useful for testing
+ */
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+
+public class SerializedApplicationReport {
+
+ public String applicationId;
+ public String applicationAttemptId;
+ public String name;
+ public String applicationType;
+ public String user;
+ public String queue;
+ public String host;
+ public Integer rpcPort;
+ public String state;
+ public String diagnostics;
+ public String url;
+ /**
+ * This value is non-null only when a report is generated from a submission context.
+ * The YARN {@link ApplicationReport} structure does not propagate this value
+ * from the RM.
+ */
+ public Long submitTime;
+ public Long startTime;
+ public Long finishTime;
+ public String finalStatus;
+ public String origTrackingUrl;
+ public Float progress;
+
+ public SerializedApplicationReport() {
+ }
+
+ public SerializedApplicationReport(ApplicationReport report) {
+ this.applicationId = report.getApplicationId().toString();
+ ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId();
+ this.applicationAttemptId = attemptId != null ? attemptId.toString() : "N/A";
+ this.name = report.getName();
+ this.applicationType = report.getApplicationType();
+ this.user = report.getUser();
+ this.queue = report.getQueue();
+ this.host = report.getHost();
+ this.rpcPort = report.getRpcPort();
+ this.state = report.getYarnApplicationState().toString();
+ this.diagnostics = report.getDiagnostics();
+ this.startTime = report.getStartTime();
+ this.finishTime = report.getFinishTime();
+ FinalApplicationStatus appStatus = report.getFinalApplicationStatus();
+ this.finalStatus = appStatus == null ? "" : appStatus.toString();
+ this.progress = report.getProgress();
+ this.url = report.getTrackingUrl();
+ this.origTrackingUrl= report.getOriginalTrackingUrl();
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return ApplicationReportSerDeser.toString(this);
+ } catch (IOException e) {
+ return super.toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index c87c3b4..21cb049 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -25,20 +25,16 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.Configuration;
-import org.apache.slider.api.resource.Resource;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.persist.JsonSerDeser;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Resource;
import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-import org.apache.slider.server.servicemonitor.MonitorUtils;
-import org.apache.slider.server.services.utility.PatternValidator;
-import org.apache.slider.util.RestApiConstants;
-import org.apache.slider.util.RestApiErrorMessages;
+import org.apache.hadoop.yarn.service.servicemonitor.probe.MonitorUtils;
+import org.apache.hadoop.yarn.service.conf.RestApiConstants;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
import org.codehaus.jackson.map.PropertyNamingStrategy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -127,9 +123,7 @@ public class ServiceApiUtil {
List<Component> componentsToAdd = new ArrayList<>();
for (Component comp : application.getComponents()) {
int maxCompLength = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
- if (comp.getUniqueComponentSupport()) {
- maxCompLength = maxCompLength - Long.toString(Long.MAX_VALUE).length();
- }
+ maxCompLength = maxCompLength - Long.toString(Long.MAX_VALUE).length();
if (dnsEnabled && comp.getName().length() > maxCompLength) {
throw new IllegalArgumentException(String.format(RestApiErrorMessages
.ERROR_COMPONENT_NAME_INVALID, maxCompLength, comp.getName()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java
new file mode 100644
index 0000000..7440b11
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+
+
+public class ServiceRegistryUtils {
+
+ /**
+ * Base path for services
+ */
+ public static final String ZK_SERVICES = "services";
+
+ /**
+ * Base path for all Slider references
+ */
+ public static final String ZK_SLIDER = "slider";
+ public static final String ZK_USERS = "users";
+ public static final String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
+ public static final String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
+
+ /**
+ * Get the registry path for an instance under the user's home node
+ * @param instanceName application instance
+ * @return a path to the registry location for this application instance.
+ */
+ public static String registryPathForInstance(String instanceName) {
+ return RegistryUtils.servicePath(
+ RegistryUtils.currentUser(), YarnServiceConstants.APP_TYPE, instanceName
+ );
+ }
+
+ /**
+ * Build the path to a cluster; exists once the cluster has come up.
+ * Even before that, a ZK watcher could wait for it.
+ * @param username user
+ * @param clustername name of the cluster
+ * @return a strin
+ */
+ public static String mkClusterPath(String username, String clustername) {
+ return mkSliderUserPath(username) + "/" + clustername;
+ }
+
+ /**
+ * Build the path to a cluster; exists once the cluster has come up.
+ * Even before that, a ZK watcher could wait for it.
+ * @param username user
+ * @return a string
+ */
+ public static String mkSliderUserPath(String username) {
+ return SVC_SLIDER_USERS + "/" + username;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
new file mode 100644
index 0000000..d6d664e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+
+/**
+ * Extends Core Filesystem with operations to manipulate ClusterDescription
+ * persistent state
+ */
+public class SliderFileSystem extends CoreFileSystem {
+
+ Path appDir = null;
+
+ public SliderFileSystem(FileSystem fileSystem,
+ Configuration configuration) {
+ super(fileSystem, configuration);
+ }
+
+ public SliderFileSystem(Configuration configuration) throws IOException {
+ super(configuration);
+ }
+
+ public void setAppDir(Path appDir) {
+ this.appDir = appDir;
+ }
+
+ public Path getAppDir() {
+ return this.appDir;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
deleted file mode 100644
index 333859e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
+++ /dev/null
@@ -1,68 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This is the log4j configuration for Slider Application Master
-
-# Log rotation based on size (100KB) with a max of 10 backup files
-log4j.rootLogger=INFO, amlog
-log4j.threshhold=ALL
-log4j.appender.amlog=org.apache.log4j.RollingFileAppender
-log4j.appender.amlog.layout=org.apache.log4j.PatternLayout
-log4j.appender.amlog.File=${LOG_DIR}/slider.log
-log4j.appender.amlog.MaxFileSize=1MB
-log4j.appender.amlog.MaxBackupIndex=10
-
-# log layout skips stack-trace creation operations by avoiding line numbers and method
-log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
-
-# debug edition is much more expensive
-#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
-
-# configure stderr
-# set the conversion pattern of stderr
-# Print the date in ISO 8601 format
-log4j.appender.stderr=org.apache.log4j.ConsoleAppender
-log4j.appender.stderr.Target=System.err
-log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
-log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
-
-log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
-log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
-log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
-#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
-
-# for debugging Slider
-#log4j.logger.org.apache.slider=DEBUG
-
-# uncomment to debug service lifecycle issues
-#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
-#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
-
-# uncomment for YARN operations
-#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
-
-# uncomment this to debug security problems
-#log4j.logger.org.apache.hadoop.security=DEBUG
-
-#crank back on some noise
-log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-log4j.logger.org.apache.hadoop.hdfs=WARN
-log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
-
-log4j.logger.org.apache.zookeeper=WARN
-log4j.logger.org.apache.curator.framework.state=ERROR
-log4j.logger.org.apache.curator.framework.imps=WARN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
new file mode 100644
index 0000000..58c8e27
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is the log4j configuration for Slider Application Master
+
+# Log rotation based on size (256MB) with a max of 20 backup files
+log4j.rootLogger=INFO, amlog
+log4j.threshhold=ALL
+log4j.appender.amlog=org.apache.log4j.RollingFileAppender
+log4j.appender.amlog.layout=org.apache.log4j.PatternLayout
+log4j.appender.amlog.File=${LOG_DIR}/serviceam.log
+log4j.appender.amlog.MaxFileSize=256MB
+log4j.appender.amlog.MaxBackupIndex=20
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# configure stderr
+# set the conversion pattern of stderr
+# Print the date in ISO 8601 format
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+
+# for debugging yarn-service framework
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.curator.framework.state=ERROR
+log4j.logger.org.apache.curator.framework.imps=WARN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 02317e5..c8de037 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -59,8 +59,6 @@
<source>
<directory>${basedir}/src/main/proto</directory>
<includes>
- <include>SliderClusterMessages.proto</include>
- <include>SliderClusterProtocol.proto</include>
<include>ClientAMProtocol.proto</include>
</includes>
</source>
@@ -92,32 +90,11 @@
</plugin>
<plugin>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-maven-plugin</artifactId>
- <version>${avro.version}</version>
- <executions>
- <execution>
- <phase>generate-sources</phase>
- <goals>
- <goal>schema</goal>
- </goals>
- <configuration>
- <sourceDirectory>${project.basedir}/src/main/avro/
- </sourceDirectory>
- <outputDirectory>${project.build.directory}/generated-sources/java
- </outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/*.json</exclude>
- <exclude>src/main/resources/webapps/slideram/.keep</exclude>
</excludes>
</configuration>
</plugin>
@@ -218,11 +195,6 @@
</dependency>
<dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </dependency>
-
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</dependency>
@@ -422,7 +394,7 @@
<attach>false</attach>
<finalName>${project.artifactId}-${project.version}</finalName>
<descriptorRefs>
- <descriptorRef>hadoop-yarn-slider-dist</descriptorRef>
+ <descriptorRef>hadoop-yarn-services-dist</descriptorRef>
</descriptorRefs>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
deleted file mode 100644
index 3667c01..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
+++ /dev/null
@@ -1,114 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-[
-
- {
- "type": "record",
- "name": "NodeEntryRecord",
- "namespace": "org.apache.slider.server.avro",
- "fields": [
- {
- "name": "host",
- "type": "string"
- },
- {
- "name": "role",
- "type": "int"
- },
- {
- "name": "active",
- "type": "boolean"
- },
- {
- "name": "last_used",
- "type": "long"
- }
- ]
- },
-
- {
- "type": "record",
- "name": "RoleHistoryHeader",
- "namespace": "org.apache.slider.server.avro",
- "fields": [
- {
- "name": "version",
- "type": "int"
- },
- {
- "name": "saved",
- "type": "long"
- },
- {
- "name": "savedx",
- "type": "string"
- },
- {
- "name": "savedate",
- "type": "string",
- "default": ""
- },
- {
- "name": "roles",
- "type": "int"
- }
- ]
- },
- {
- "type": "record",
- "name": "RoleHistoryMapping",
- "namespace": "org.apache.slider.server.avro",
- "fields": [
- {
- "name": "rolemap",
- "type": {
- "type": "map",
- "values": "int"
- }
- }
- ]
- },
- {
- "type": "record",
- "name": "RoleHistoryFooter",
- "namespace": "org.apache.slider.server.avro",
- "fields": [
- {
- "name": "count",
- "type": "long"
- }
- ]
- },
-
- {
- "type": "record",
- "name": "RoleHistoryRecord",
- "namespace": "org.apache.slider.server.avro",
- "fields": [
- {
- "name": "entry",
- "type": [
- "org.apache.slider.server.avro.NodeEntryRecord",
- "org.apache.slider.server.avro.RoleHistoryHeader",
- "org.apache.slider.server.avro.RoleHistoryFooter",
- "org.apache.slider.server.avro.RoleHistoryMapping"
- ]
- }
- ]
- }
-
-]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
deleted file mode 100644
index 905d4b1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
+++ /dev/null
@@ -1,677 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.security;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.Shell;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.crypto.Cipher;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.io.PrintWriter;
-import java.lang.reflect.InvocationTargetException;
-import java.net.InetAddress;
-import java.security.NoSuchAlgorithmException;
-import java.util.Collection;
-import java.util.Date;
-import java.util.List;
-import java.util.regex.Pattern;
-
-import static org.apache.hadoop.security.UserGroupInformation.*;
-import static org.apache.hadoop.security.authentication.util.KerberosUtil.*;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
-
-/**
- * Kerberos diagnostics
- * At some point this may move to hadoop core, so please keep use of slider
- * methods and classes to ~0.
- *
- * This operation expands some of the diagnostic output of the security code,
- * but not all. For completeness
- *
- * Set the environment variable {@code HADOOP_JAAS_DEBUG=true}
- * Set the log level for {@code org.apache.hadoop.security=DEBUG}
- */
-public class KerberosDiags implements Closeable {
-
- private static final Logger LOG = LoggerFactory.getLogger(KerberosDiags.class);
- public static final String KRB5_CCNAME = "KRB5CCNAME";
- public static final String JAVA_SECURITY_KRB5_CONF
- = "java.security.krb5.conf";
- public static final String JAVA_SECURITY_KRB5_REALM
- = "java.security.krb5.realm";
- public static final String SUN_SECURITY_KRB5_DEBUG
- = "sun.security.krb5.debug";
- public static final String SUN_SECURITY_SPNEGO_DEBUG
- = "sun.security.spnego.debug";
- public static final String SUN_SECURITY_JAAS_FILE
- = "java.security.auth.login.config";
- public static final String KERBEROS_KINIT_COMMAND
- = "hadoop.kerberos.kinit.command";
- public static final String HADOOP_AUTHENTICATION_IS_DISABLED
- = "Hadoop authentication is disabled";
- public static final String UNSET = "(unset)";
- public static final String NO_DEFAULT_REALM = "Cannot locate default realm";
-
- private final Configuration conf;
- private final List<String> services;
- private final PrintStream out;
- private final File keytab;
- private final String principal;
- private final long minKeyLength;
- private final boolean securityRequired;
-
- public static final String CAT_JVM = "JVM";
- public static final String CAT_JAAS = "JAAS";
- public static final String CAT_CONFIG = "CONFIG";
- public static final String CAT_LOGIN = "LOGIN";
- public static final String CAT_KERBEROS = "KERBEROS";
- public static final String CAT_SASL = "SASL";
-
- @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
- public KerberosDiags(Configuration conf,
- PrintStream out,
- List<String> services,
- File keytab,
- String principal,
- long minKeyLength,
- boolean securityRequired) {
- this.conf = conf;
- this.services = services;
- this.keytab = keytab;
- this.principal = principal;
- this.out = out;
- this.minKeyLength = minKeyLength;
- this.securityRequired = securityRequired;
- }
-
- @Override
- public void close() throws IOException {
- flush();
- }
-
- /**
- * Execute diagnostics.
- * <p>
- * Things it would be nice if UGI made accessible
- * <ol>
- * <li>A way to enable JAAS debug programatically</li>
- * <li>Access to the TGT</li>
- * </ol>
- * @return true if security was enabled and all probes were successful
- * @throws KerberosDiagsFailure explicitly raised failure
- * @throws Exception other security problems
- */
- @SuppressWarnings("deprecation")
- public boolean execute() throws Exception {
-
- title("Kerberos Diagnostics scan at %s",
- new Date(System.currentTimeMillis()));
-
- // check that the machine has a name
- println("Hostname: %s",
- InetAddress.getLocalHost().getCanonicalHostName());
-
- // Fail fast on a JVM without JCE installed.
- validateKeyLength();
-
- // look at realm
- println("JVM Kerberos Login Module = %s", getKrb5LoginModuleName());
- printDefaultRealm();
-
- title("System Properties");
- for (String prop : new String[]{
- JAVA_SECURITY_KRB5_CONF,
- JAVA_SECURITY_KRB5_REALM,
- SUN_SECURITY_KRB5_DEBUG,
- SUN_SECURITY_SPNEGO_DEBUG,
- SUN_SECURITY_JAAS_FILE
- }) {
- printSysprop(prop);
- }
-
- title("Environment Variables");
- for (String env : new String[]{
- "HADOOP_JAAS_DEBUG",
- KRB5_CCNAME,
- HADOOP_USER_NAME,
- HADOOP_PROXY_USER,
- HADOOP_TOKEN_FILE_LOCATION,
- }) {
- printEnv(env);
- }
-
- for (String prop : new String[]{
- KERBEROS_KINIT_COMMAND,
- HADOOP_SECURITY_AUTHENTICATION,
- HADOOP_SECURITY_AUTHORIZATION,
- "hadoop.kerberos.min.seconds.before.relogin", // not in 2.6
- "hadoop.security.dns.interface", // not in 2.6
- "hadoop.security.dns.nameserver", // not in 2.6
- HADOOP_RPC_PROTECTION,
- HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
- HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX,
- HADOOP_SECURITY_GROUP_MAPPING,
- "hadoop.security.impersonation.provider.class", // not in 2.6
- "dfs.data.transfer.protection" // HDFS
- }) {
- printConfOpt(prop);
- }
-
- // check that authentication is enabled
- if (SecurityUtil.getAuthenticationMethod(conf)
- .equals(AuthenticationMethod.SIMPLE)) {
- println(HADOOP_AUTHENTICATION_IS_DISABLED);
- failif(securityRequired, CAT_CONFIG, HADOOP_AUTHENTICATION_IS_DISABLED);
- // no security, skip rest of test
- return false;
- }
-
- validateKrb5File();
- validateSasl(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS);
- validateSasl("dfs.data.transfer.saslproperties.resolver.class");
- validateKinitExecutable();
- validateJAAS();
- // now the big test: login, then try again
- boolean krb5Debug = getAndSet(SUN_SECURITY_KRB5_DEBUG);
- boolean spnegoDebug = getAndSet(SUN_SECURITY_SPNEGO_DEBUG);
- try {
- title("Logging in");
-
- if (keytab != null) {
- dumpKeytab(keytab);
- loginFromKeytab();
- } else {
- UserGroupInformation loginUser = getLoginUser();
- dumpUGI("Log in user", loginUser);
- validateUGI("Login user", loginUser);
- println("Ticket based login: %b", isLoginTicketBased());
- println("Keytab based login: %b", isLoginKeytabBased());
- }
-
- return true;
- } finally {
- // restore original system properties
- System.setProperty(SUN_SECURITY_KRB5_DEBUG,
- Boolean.toString(krb5Debug));
- System.setProperty(SUN_SECURITY_SPNEGO_DEBUG,
- Boolean.toString(spnegoDebug));
- }
- }
-
- /**
- * Fail fast on a JVM without JCE installed.
- *
- * This is a recurrent problem
- * (that is: it keeps creeping back with JVM updates);
- * a fast failure is the best tactic
- * @throws NoSuchAlgorithmException
- */
-
- protected void validateKeyLength() throws NoSuchAlgorithmException {
- int aesLen = Cipher.getMaxAllowedKeyLength("AES");
- println("Maximum AES encryption key length %d bits", aesLen);
- failif (aesLen < minKeyLength,
- CAT_JVM,
- "Java Cryptography Extensions are not installed on this JVM."
- +" Maximum supported key length %s - minimum required %d",
- aesLen, minKeyLength);
- }
-
- /**
- * Get the default realm.
- * <p>
- * Not having a default realm may be harmless, so is noted at info.
- * All other invocation failures are downgraded to warn, as
- * follow-on actions may still work.
- * failure to invoke the method via introspection is rejected,
- * as it's a sign of JVM compatibility issues that may have other
- * consequences
- */
- protected void printDefaultRealm() {
- try {
- println("Default Realm = %s",
- getDefaultRealm());
- } catch (ClassNotFoundException
- | IllegalAccessException
- | NoSuchMethodException e) {
-
- throw new KerberosDiagsFailure(CAT_JVM, e,
- "Failed to invoke krb5.Config.getDefaultRealm: %s", e);
- } catch (InvocationTargetException e) {
- Throwable cause = e.getCause() != null ? e.getCause() : e;
- if (cause.toString().contains(NO_DEFAULT_REALM)) {
- // exception raised if there is no default realm. This is not
- // always a problem, so downgrade to a message.
- println("Host has no default realm");
- LOG.debug(cause.toString(), cause);
- } else {
- println("Kerberos.getDefaultRealm() failed: %s\n%s",
- cause,
- org.apache.hadoop.util.StringUtils.stringifyException(cause));
- }
- }
- }
-
- /**
- * Locate the krb5.conf file and dump it.
- * No-op on windows.
- * @throws IOException
- */
- private void validateKrb5File() throws IOException {
- if (!Shell.WINDOWS) {
- title("Locating Kerberos configuration file");
- String krbPath = "/etc/krb5.conf";
- String jvmKrbPath = System.getProperty(JAVA_SECURITY_KRB5_CONF);
- if (jvmKrbPath != null) {
- println("Setting kerberos path from sysprop %s: %s",
- JAVA_SECURITY_KRB5_CONF, jvmKrbPath);
- krbPath = jvmKrbPath;
- }
-
- String krb5name = System.getenv(KRB5_CCNAME);
- if (krb5name != null) {
- println("Setting kerberos path from environment variable %s: %s",
- KRB5_CCNAME, krb5name);
- krbPath = krb5name;
- if (jvmKrbPath != null) {
- println("Warning - both %s and %s were set - %s takes priority",
- JAVA_SECURITY_KRB5_CONF, KRB5_CCNAME, KRB5_CCNAME);
- }
- }
-
- File krbFile = new File(krbPath);
- println("Kerberos configuration file = %s", krbFile);
- failif(!krbFile.exists(),
- CAT_KERBEROS,
- "Kerberos configuration file %s not found", krbFile);
- dump(krbFile);
- }
- }
-
- /**
- * Dump a keytab: list all principals.
- * @param keytabFile the keytab file
- * @throws IOException IO problems
- */
- public void dumpKeytab(File keytabFile) throws IOException {
- title("Examining keytab %s", keytabFile);
- File kt = keytabFile.getCanonicalFile();
- failif(!kt.exists(), CAT_CONFIG, "Keytab not found: %s", kt);
- failif(!kt.isFile(), CAT_CONFIG, "Keytab is not a valid file: %s", kt);
-
- String[] names = getPrincipalNames(keytabFile.getCanonicalPath(),
- Pattern.compile(".*"));
- println("keytab entry count: %d", names.length);
- for (String name : names) {
- println(" %s", name);
- }
- println("-----");
- }
-
- /**
- * Log in from a keytab, dump the UGI, validate it, then try and log in again.
- * That second-time login catches JVM/Hadoop compatibility problems.
- * @throws IOException
- */
- private void loginFromKeytab() throws IOException {
- UserGroupInformation ugi;
- String identity;
- if (keytab != null) {
- File kt = keytab.getCanonicalFile();
- println("Using keytab %s principal %s", kt, principal);
- identity = principal;
-
- failif(StringUtils.isEmpty(principal), CAT_KERBEROS,
- "No principal defined");
- ugi = loginUserFromKeytabAndReturnUGI(principal, kt.getPath());
- dumpUGI(identity, ugi);
- validateUGI(principal, ugi);
-
- title("Attempting to log in from keytab again");
- // package scoped -hence the reason why this class must be in the
- // hadoop.security package
- setShouldRenewImmediatelyForTests(true);
- // attempt a new login
- ugi.reloginFromKeytab();
- } else {
- println("No keytab: logging is as current user");
- }
- }
-
- /**
- * Dump a UGI.
- * @param title title of this section
- * @param ugi UGI to dump
- * @throws IOException
- */
- private void dumpUGI(String title, UserGroupInformation ugi)
- throws IOException {
- title(title);
- println("UGI instance = %s", ugi);
- println("Has kerberos credentials: %b", ugi.hasKerberosCredentials());
- println("Authentication method: %s", ugi.getAuthenticationMethod());
- println("Real Authentication method: %s",
- ugi.getRealAuthenticationMethod());
- title("Group names");
- for (String name : ugi.getGroupNames()) {
- println(name);
- }
- title("Credentials");
- Credentials credentials = ugi.getCredentials();
- List<Text> secretKeys = credentials.getAllSecretKeys();
- title("Secret keys");
- if (!secretKeys.isEmpty()) {
- for (Text secret: secretKeys) {
- println("%s", secret);
- }
- } else {
- println("(none)");
- }
-
- dumpTokens(ugi);
- }
-
- /**
- * Validate the UGI: verify it is kerberized.
- * @param messagePrefix message in exceptions
- * @param user user to validate
- */
- private void validateUGI(String messagePrefix, UserGroupInformation user) {
- failif(!user.hasKerberosCredentials(),
- CAT_LOGIN, "%s: No kerberos credentials for %s", messagePrefix, user);
- failif(user.getAuthenticationMethod() == null,
- CAT_LOGIN, "%s: Null AuthenticationMethod for %s", messagePrefix, user);
- }
-
- /**
- * A cursory look at the {@code kinit} executable.
- * If it is an absolute path: it must exist with a size > 0.
- * If it is just a command, it has to be on the path. There's no check
- * for that -but the PATH is printed out.
- */
- private void validateKinitExecutable() {
- String kinit = conf.getTrimmed(KERBEROS_KINIT_COMMAND, "");
- if (!kinit.isEmpty()) {
- File kinitPath = new File(kinit);
- println("%s = %s", KERBEROS_KINIT_COMMAND, kinitPath);
- if (kinitPath.isAbsolute()) {
- failif(!kinitPath.exists(), CAT_KERBEROS,
- "%s executable does not exist: %s",
- KERBEROS_KINIT_COMMAND, kinitPath);
- failif(!kinitPath.isFile(), CAT_KERBEROS,
- "%s path does not refer to a file: %s",
- KERBEROS_KINIT_COMMAND, kinitPath);
- failif(kinitPath.length() == 0, CAT_KERBEROS,
- "%s file is empty: %s",
- KERBEROS_KINIT_COMMAND, kinitPath);
- } else {
- println("Executable %s is relative -must be on the PATH", kinit);
- printEnv("PATH");
- }
- }
- }
-
- /**
- * Try to load the SASL resolver.
- * @param saslPropsResolverKey key for the SASL resolver
- */
- private void validateSasl(String saslPropsResolverKey) {
- title("Resolving SASL property %s", saslPropsResolverKey);
- String saslPropsResolver = conf.getTrimmed(saslPropsResolverKey);
- try {
- Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass(
- saslPropsResolverKey,
- SaslPropertiesResolver.class, SaslPropertiesResolver.class);
- println("Resolver is %s", resolverClass);
- } catch (RuntimeException e) {
- throw new KerberosDiagsFailure(CAT_SASL, e,
- "Failed to load %s class %s",
- saslPropsResolverKey, saslPropsResolver);
- }
- }
-
- /**
- * Validate any JAAS entry referenced in the {@link #SUN_SECURITY_JAAS_FILE}
- * property.
- */
- private void validateJAAS() {
- String jaasFilename = System.getProperty(SUN_SECURITY_JAAS_FILE);
- if (jaasFilename != null) {
- title("JAAS");
- File jaasFile = new File(jaasFilename);
- println("JAAS file is defined in %s: %s",
- SUN_SECURITY_JAAS_FILE, jaasFile);
- failif(!jaasFile.exists(), CAT_JAAS,
- "JAAS file does not exist: %s", jaasFile);
- failif(!jaasFile.isFile(), CAT_JAAS,
- "Specified JAAS file is not a file: %s", jaasFile);
- }
- }
-
- /**
- * Dump all tokens of a user
- * @param user user
- */
- public void dumpTokens(UserGroupInformation user) {
- Collection<Token<? extends TokenIdentifier>> tokens
- = user.getCredentials().getAllTokens();
- title("Token Count: %d", tokens.size());
- for (Token<? extends TokenIdentifier> token : tokens) {
- println("Token %s", token.getKind());
- }
- }
-
- /**
- * Set the System property to true; return the old value for caching
- * @param sysprop property
- * @return the previous value
- */
- private boolean getAndSet(String sysprop) {
- boolean old = Boolean.getBoolean(sysprop);
- System.setProperty(sysprop, "true");
- return old;
- }
-
- /**
- * Flush all active output channels, including {@Code System.err},
- * so as to stay in sync with any JRE log messages.
- */
- private void flush() {
- if (out != null) {
- out.flush();
- } else {
- System.out.flush();
- }
- System.err.flush();
- }
-
- /**
- * Format and print a line of output.
- * This goes to any output file, or
- * is logged at info. The output is flushed before and after, to
- * try and stay in sync with JRE logging.
- * @param format format string
- * @param args any arguments
- */
- @VisibleForTesting
- public void println(String format, Object... args) {
- println(format(format, args));
- }
-
- /**
- * Print a line of output. This goes to any output file, or
- * is logged at info. The output is flushed before and after, to
- * try and stay in sync with JRE logging.
- * @param msg message string
- */
- @VisibleForTesting
- private void println(String msg) {
- flush();
- if (out != null) {
- out.println(msg);
- } else {
- LOG.info(msg);
- }
- flush();
- }
-
- /**
- * Print a title entry
- * @param format format string
- * @param args any arguments
- */
- private void title(String format, Object... args) {
- println("");
- println("");
- String msg = "== " + format(format, args) + " ==";
- println(msg);
- println("");
- }
-
- /**
- * Print a system property, or {@link #UNSET} if unset.
- * @param property property to print
- */
- private void printSysprop(String property) {
- println("%s = \"%s\"", property,
- System.getProperty(property, UNSET));
- }
-
- /**
- * Print a configuration option, or {@link #UNSET} if unset.
- * @param option option to print
- */
- private void printConfOpt(String option) {
- println("%s = \"%s\"", option, conf.get(option, UNSET));
- }
-
- /**
- * Print an environment variable's name and value; printing
- * {@link #UNSET} if it is not set
- * @param variable environment variable
- */
- private void printEnv(String variable) {
- String env = System.getenv(variable);
- println("%s = \"%s\"", variable, env != null ? env : UNSET);
- }
-
- /**
- * Dump any file to standard out; add a trailing newline
- * @param file file to dump
- * @throws IOException IO problems
- */
- public void dump(File file) throws IOException {
- try (FileInputStream in = new FileInputStream(file)) {
- for (String line : IOUtils.readLines(in)) {
- println("%s", line);
- }
- }
- println("");
- }
-
- /**
- * Format and raise a failure
- *
- * @param category category for exception
- * @param message string formatting message
- * @param args any arguments for the formatting
- * @throws KerberosDiagsFailure containing the formatted text
- */
- private void fail(String category, String message, Object... args)
- throws KerberosDiagsFailure {
- throw new KerberosDiagsFailure(category, message, args);
- }
-
- /**
- * Conditional failure with string formatted arguments
- * @param condition failure condition
- * @param category category for exception
- * @param message string formatting message
- * @param args any arguments for the formatting
- * @throws KerberosDiagsFailure containing the formatted text
- * if the condition was met
- */
- private void failif(boolean condition,
- String category,
- String message,
- Object... args)
- throws KerberosDiagsFailure {
- if (condition) {
- fail(category, message, args);
- }
- }
-
- /**
- * Format a string, treating a call where there are no varags values
- * as a string to pass through unformatted.
- * @param message message, which is either a format string + args, or
- * a general string
- * @param args argument array
- * @return a string for printing.
- */
- public static String format(String message, Object... args) {
- if (args.length == 0) {
- return message;
- } else {
- return String.format(message, args);
- }
- }
-
- /**
- * Diagnostics failures return the exit code 41, "unauthorized".
- *
- * They have a category, initially for testing: the category can be
- * validated without having to match on the entire string.
- */
- public static class KerberosDiagsFailure extends ExitUtil.ExitException {
- private final String category;
-
- public KerberosDiagsFailure(String category, String message) {
- super(41, category + ": " + message);
- this.category = category;
- }
-
- public KerberosDiagsFailure(String category, String message, Object... args) {
- this(category, format(message, args));
- }
-
- public KerberosDiagsFailure(String category, Throwable throwable,
- String message, Object... args) {
- this(category, message, args);
- initCause(throwable);
- }
-
- public String getCategory() {
- return category;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
index bbb4c44..4743f28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
@@ -29,7 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import static org.apache.slider.api.ResourceKeys.NODE_FAILURE_THRESHOLD;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.NODE_BLACKLIST_THRESHOLD;
/**
* This tracks the container failures per node. If the failure counter exceeds
@@ -52,7 +52,7 @@ public class ContainerFailureTracker {
this.context = context;
this.component = component;
maxFailurePerNode = component.getComponentSpec().getConfiguration()
- .getPropertyInt(NODE_FAILURE_THRESHOLD, 3);
+ .getPropertyInt(NODE_BLACKLIST_THRESHOLD, 3);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java
deleted file mode 100644
index 2037a3b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.provider.ProviderService;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.launch.AbstractLauncher;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-public class ContainerLaunchService extends AbstractService{
-
- protected static final Logger LOG =
- LoggerFactory.getLogger(ContainerLaunchService.class);
-
- private ExecutorService executorService;
- private SliderFileSystem fs;
-
- public ContainerLaunchService(SliderFileSystem fs) {
- super(ContainerLaunchService.class.getName());
- this.fs = fs;
- }
-
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- executorService = Executors.newCachedThreadPool();
- super.serviceInit(conf);
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (executorService != null) {
- executorService.shutdownNow();
- }
- super.serviceStop();
- }
-
- public void launchCompInstance(Application application,
- ComponentInstance instance, Container container) {
- ContainerLauncher launcher =
- new ContainerLauncher(application, instance, container);
- executorService.execute(launcher);
- }
-
- private class ContainerLauncher implements Runnable {
- public final Container container;
- public final Application application;
- public ComponentInstance instance;
-
- public ContainerLauncher(
- Application application,
- ComponentInstance instance, Container container) {
- this.container = container;
- this.application = application;
- this.instance = instance;
- }
-
- @Override public void run() {
- org.apache.slider.api.resource.Component compSpec = instance.getCompSpec();
- ProviderService provider = ProviderFactory.getProviderService(
- compSpec.getArtifact());
- AbstractLauncher launcher = new AbstractLauncher(fs, null);
- try {
- provider.buildContainerLaunchContext(launcher, application,
- instance, fs, getConfig());
- instance.getComponent().getScheduler().getNmClient()
- .startContainerAsync(container,
- launcher.completeContainerLaunch());
- } catch (Exception e) {
- LOG.error(instance.getCompInstanceId()
- + ": Failed to launch container. ", e);
-
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
index 80668a0..c7616af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
@@ -21,9 +21,9 @@ package org.apache.hadoop.yarn.service;
import com.google.common.cache.LoadingCache;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
public class ServiceContext {
public Application application = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
index c22dec4..d099f8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -35,10 +35,12 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.service.client.params.SliderAMArgs;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.servicemonitor.ServiceMonitor;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,6 +70,7 @@ public class ServiceMaster extends CompositeService {
context = new ServiceContext();
Path appDir = getAppDir();
+ context.serviceHdfsDir = appDir.toString();
SliderFileSystem fs = new SliderFileSystem(conf);
context.fs = fs;
fs.setAppDir(appDir);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java
deleted file mode 100644
index bc37614..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.slider.api.InternalKeys;
-import org.apache.hadoop.yarn.service.component.ComponentEvent;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
-import org.apache.hadoop.yarn.service.component.ComponentState;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.server.servicemonitor.ProbeStatus;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.RUNNING_BUT_UNREADY;
-import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.BECOME_NOT_READY;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.BECOME_READY;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.READY;
-
-public class ServiceMonitor extends AbstractService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceMonitor.class);
-
- public ScheduledExecutorService executorService;
- private Map<ContainerId, ComponentInstance> liveInstances = null;
- private ServiceContext context;
- private Configuration conf;
-
- public ServiceMonitor(String name, ServiceContext context) {
- super(name);
- liveInstances = context.scheduler.getLiveInstances();
- this.context = context;
- }
-
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- executorService = Executors.newScheduledThreadPool(1);
- this.conf = conf;
- super.serviceInit(conf);
- }
-
- @Override
- public void serviceStart() throws Exception {
- long readinessCheckInterval =
- YarnServiceConf.getLong(InternalKeys.MONITOR_INTERVAL,
- InternalKeys.DEFAULT_MONITOR_INTERVAL,
- context.application.getConfiguration(), conf);
-
- executorService
- .scheduleAtFixedRate(new ReadinessChecker(), readinessCheckInterval,
- readinessCheckInterval, TimeUnit.SECONDS);
-
- long failureResetInterval = SliderUtils
- .getTimeRange(context.application.getConfiguration(),
- ResourceKeys.CONTAINER_FAILURE_WINDOW,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES, 0);
-
- executorService
- .scheduleAtFixedRate(new ContainerFailureReset(), failureResetInterval,
- failureResetInterval, TimeUnit.SECONDS);
- }
-
- @Override
- public void serviceStop() throws Exception {
- if (executorService != null) {
- executorService.shutdownNow();
- }
- }
-
- private class ReadinessChecker implements Runnable {
-
- @Override
- public void run() {
-
- // check if the comp instance are ready
- for (Map.Entry<ContainerId, ComponentInstance> entry : liveInstances
- .entrySet()) {
- ComponentInstance instance = entry.getValue();
-
- ProbeStatus status = instance.ping();
- if (status.isSuccess()) {
- if (instance.getState() == RUNNING_BUT_UNREADY) {
- // synchronously update the state.
- instance.handle(
- new ComponentInstanceEvent(entry.getKey(), BECOME_READY));
- }
- } else {
- if (instance.getState() == READY) {
- instance.handle(
- new ComponentInstanceEvent(entry.getKey(), BECOME_NOT_READY));
- }
- }
- }
-
- for (Component component : context.scheduler.getAllComponents()
- .values()) {
- // If comp hasn't started yet and its dependencies are satisfied
- if (component.getState() == ComponentState.INIT && component
- .areDependenciesReady()) {
- LOG.info("[COMPONENT {}]: Dependencies satisfied, ramping up.",
- component.getName());
- ComponentEvent event = new ComponentEvent(component.getName(), FLEX)
- .setDesired(component.getComponentSpec().getNumberOfContainers());
- component.handle(event);
- }
- }
- }
- }
-
- private class ContainerFailureReset implements Runnable {
- @Override
- public void run() {
- for (Component component : context.scheduler.getAllComponents().values()) {
- component.resetCompFailureCount();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 590655f..8c968dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -54,26 +54,25 @@ import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType;
import org.apache.hadoop.yarn.service.component.Component;
import org.apache.hadoop.yarn.service.component.ComponentEvent;
import org.apache.hadoop.yarn.service.component.ComponentEventType;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
import org.apache.hadoop.yarn.service.provider.ProviderUtils;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
import org.apache.hadoop.yarn.util.BoundedAppender;
-import org.apache.slider.api.RoleKeys;
-import org.apache.slider.api.ServiceApiConstants;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.core.registry.info.CustomRegistryConstants;
-import org.apache.slider.core.zk.ZKIntegration;
-import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -93,8 +92,8 @@ import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
+import static org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants.*;
import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
-import static org.apache.slider.api.ServiceApiConstants.*;
/**
*
@@ -203,7 +202,7 @@ public class ServiceScheduler extends CompositeService {
protected YarnRegistryViewForProviders createYarnRegistryOperations(
ServiceContext context, RegistryOperations registryClient) {
return new YarnRegistryViewForProviders(registryClient,
- RegistryUtils.currentUser(), SliderKeys.APP_TYPE, app.getName(),
+ RegistryUtils.currentUser(), YarnServiceConstants.APP_TYPE, app.getName(),
context.attemptId);
}
@@ -236,7 +235,8 @@ public class ServiceScheduler extends CompositeService {
DefaultMetricsSystem.shutdown();
if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
- serviceTimelinePublisher.serviceAttemptUnregistered(context);
+ serviceTimelinePublisher
+ .serviceAttemptUnregistered(context, diagnostics.toString());
}
// Cleanup each component instance. no need to release containers as
// they will be automatically released by RM
@@ -285,7 +285,7 @@ public class ServiceScheduler extends CompositeService {
LOG.error("Failed to get user.", e);
}
globalTokens
- .put(SERVICE_ZK_PATH, ZKIntegration.mkClusterPath(user, app.getName()));
+ .put(SERVICE_ZK_PATH, ServiceRegistryUtils.mkClusterPath(user, app.getName()));
globalTokens.put(ServiceApiConstants.USER, user);
String dnsDomain = getConfig().getTrimmed(KEY_DNS_DOMAIN);
@@ -336,7 +336,7 @@ public class ServiceScheduler extends CompositeService {
context.configCache = configFileCache;
}
- protected void registerServiceInstance(ApplicationAttemptId attemptId,
+ private void registerServiceInstance(ApplicationAttemptId attemptId,
Application application) throws IOException {
LOG.info("Registering " + attemptId + ", " + application.getName()
+ " into registry");
@@ -345,11 +345,11 @@ public class ServiceScheduler extends CompositeService {
attemptId.getApplicationId().toString());
serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
PersistencePolicies.APPLICATION);
- serviceRecord.description = "Slider Application Master";
+ serviceRecord.description = "Yarn Service Master";
serviceRecord.addExternalEndpoint(RegistryTypeUtils
- .ipcEndpoint(CustomRegistryConstants.AM_IPC_PROTOCOL,
- new InetSocketAddress(5000))); // FIXME
+ .ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc",
+ context.clientAMService.getBindAddress()));
// set any provided attributes
setUserProvidedServiceRecordAttributes(application.getConfiguration(),
@@ -376,13 +376,13 @@ public class ServiceScheduler extends CompositeService {
}
});
if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
- serviceTimelinePublisher.serviceAttemptRegistered(app);
+ serviceTimelinePublisher.serviceAttemptRegistered(app, getConfig());
}
}
private void setUserProvidedServiceRecordAttributes(
- org.apache.slider.api.resource.Configuration conf, ServiceRecord record) {
- String prefix = RoleKeys.SERVICE_RECORD_ATTRIBUTE_PREFIX;
+ org.apache.hadoop.yarn.service.api.records.Configuration conf, ServiceRecord record) {
+ String prefix = "service.record.attribute";
for (Map.Entry<String, String> entry : conf.getProperties().entrySet()) {
if (entry.getKey().startsWith(prefix)) {
String key = entry.getKey().substring(prefix.length() + 1);
@@ -395,10 +395,10 @@ public class ServiceScheduler extends CompositeService {
long allocateId = 0;
// sort components by dependencies
- Collection<org.apache.slider.api.resource.Component> sortedComponents =
+ Collection<org.apache.hadoop.yarn.service.api.records.Component> sortedComponents =
ServiceApiUtil.sortByDependencies(app.getComponents());
- for (org.apache.slider.api.resource.Component compSpec : sortedComponents) {
+ for (org.apache.hadoop.yarn.service.api.records.Component compSpec : sortedComponents) {
Component component = new Component(compSpec, allocateId, context);
componentsById.put(allocateId, component);
componentsByName.put(component.getName(), component);
@@ -517,7 +517,7 @@ public class ServiceScheduler extends CompositeService {
@Override public float getProgress() {
// get running containers over desired containers
long total = 0;
- for (org.apache.slider.api.resource.Component component : app
+ for (org.apache.hadoop.yarn.service.api.records.Component component : app
.getComponents()) {
total += component.getNumberOfContainers();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java
new file mode 100644
index 0000000..cf9e31f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.constants;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
+
+/**
+ * This class defines constants that can be used in input spec for
+ * variable substitutions
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public interface ServiceApiConstants {
+
+ // Constants for service
+ String SERVICE_NAME = $("SERVICE_NAME");
+
+ String SERVICE_NAME_LC = $("SERVICE_NAME.lc");
+
+ String USER = $("USER");
+
+ String DOMAIN = $("DOMAIN");
+
+ // Constants for component
+ String COMPONENT_NAME = $("COMPONENT_NAME");
+
+ String COMPONENT_NAME_LC = $("COMPONENT_NAME.lc");
+
+ String COMPONENT_INSTANCE_NAME = $("COMPONENT_INSTANCE_NAME");
+
+ // Constants for component instance
+ String COMPONENT_ID = $("COMPONENT_ID");
+
+ String CONTAINER_ID = $("CONTAINER_ID");
+
+ // Constants for default cluster ZK
+ String CLUSTER_ZK_QUORUM = $("CLUSTER_ZK_QUORUM");
+
+ // URI for the default cluster fs
+ String CLUSTER_FS_URI = $("CLUSTER_FS_URI");
+
+ // the host component of the cluster fs UI
+ String CLUSTER_FS_HOST = $("CLUSTER_FS_HOST");
+
+ // Path in zookeeper for a specific service
+ String SERVICE_ZK_PATH = $("SERVICE_ZK_PATH");
+
+ // Constants for service specific hdfs dir
+ String SERVICE_HDFS_DIR = $("SERVICE_HDFS_DIR");
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java
new file mode 100644
index 0000000..f9e5154
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * An Application resource has the following attributes.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "An Application resource has the following attributes.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({ "name", "state", "resource", "number_of_containers",
+ "lifetime", "containers" })
+public class Application extends BaseResource {
+ private static final long serialVersionUID = -4491694636566094885L;
+
+ private String name = null;
+ private String id = null;
+ private Artifact artifact = null;
+ private Resource resource = null;
+ private String launchCommand = null;
+ private Date launchTime = null;
+ private Long numberOfContainers = null;
+ private Long numberOfRunningContainers = null;
+ private Long lifetime = null;
+ private PlacementPolicy placementPolicy = null;
+ private List<Component> components = new ArrayList<>();
+ private Configuration configuration = new Configuration();
+ private List<Container> containers = new ArrayList<>();
+ private ApplicationState state = null;
+ private Map<String, String> quicklinks = new HashMap<>();
+ private String queue = null;
+
+ /**
+ * A unique application name.
+ **/
+ public Application name(String name) {
+ this.name = name;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", required = true, value = "A unique application name.")
+ @JsonProperty("name")
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * A unique application id.
+ **/
+ public Application id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A unique application id.")
+ @JsonProperty("id")
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
+ * Artifact of single-component applications. Mandatory if components
+ * attribute is not specified.
+ **/
+ public Application artifact(Artifact artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact of single-component applications. Mandatory if components attribute is not specified.")
+ @JsonProperty("artifact")
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void setArtifact(Artifact artifact) {
+ this.artifact = artifact;
+ }
+
+ /**
+ * Resource of single-component applications or the global default for
+ * multi-component applications. Mandatory if it is a single-component
+ * application and if cpus and memory are not specified at the Application
+ * level.
+ **/
+ public Application resource(Resource resource) {
+ this.resource = resource;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Resource of single-component applications or the global default for multi-component applications. Mandatory if it is a single-component application and if cpus and memory are not specified at the Application level.")
+ @JsonProperty("resource")
+ public Resource getResource() {
+ return resource;
+ }
+
+ public void setResource(Resource resource) {
+ this.resource = resource;
+ }
+
+ /**
+ * The custom launch command of an application component (optional). If not
+ * specified for applications with docker images say, it will default to the
+ * default start command of the image. If there is a single component in this
+ * application, you can specify this without the need to have a 'components'
+ * section.
+ **/
+ public Application launchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The custom launch command of an application component (optional). If not specified for applications with docker images say, it will default to the default start command of the image. If there is a single component in this application, you can specify this without the need to have a 'components' section.")
+ @JsonProperty("launch_command")
+ public String getLaunchCommand() {
+ return launchCommand;
+ }
+
+ @XmlElement(name = "launch_command")
+ public void setLaunchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ }
+
+ /**
+ * The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.
+ **/
+ public Application launchTime(Date launchTime) {
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.")
+ @JsonProperty("launch_time")
+ public Date getLaunchTime() {
+ return launchTime == null ? null : (Date) launchTime.clone();
+ }
+
+ @XmlElement(name = "launch_time")
+ public void setLaunchTime(Date launchTime) {
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
+ }
+
+ /**
+ * Number of containers for each app-component in the application. Each
+ * app-component can further override this app-level global default.
+ **/
+ public Application numberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Number of containers for each app-component in the application. Each app-component can further override this app-level global default.")
+ @JsonProperty("number_of_containers")
+ public Long getNumberOfContainers() {
+ return numberOfContainers;
+ }
+
+ @XmlElement(name = "number_of_containers")
+ public void setNumberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ }
+
+ /**
+ * In get response this provides the total number of running containers for
+ * this application (across all components) at the time of request. Note, a
+ * subsequent request can return a different number as and when more
+ * containers get allocated until it reaches the total number of containers or
+ * if a flex request has been made between the two requests.
+ **/
+ public Application numberOfRunningContainers(Long numberOfRunningContainers) {
+ this.numberOfRunningContainers = numberOfRunningContainers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "In get response this provides the total number of running containers for this application (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.")
+ @JsonProperty("number_of_running_containers")
+ public Long getNumberOfRunningContainers() {
+ return numberOfRunningContainers;
+ }
+
+ @XmlElement(name = "number_of_running_containers")
+ public void setNumberOfRunningContainers(Long numberOfRunningContainers) {
+ this.numberOfRunningContainers = numberOfRunningContainers;
+ }
+
+ /**
+ * Life time (in seconds) of the application from the time it reaches the
+ * RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For
+ * unlimited lifetime do not set a lifetime value.
+ **/
+ public Application lifetime(Long lifetime) {
+ this.lifetime = lifetime;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Life time (in seconds) of the application from the time it reaches the RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.")
+ @JsonProperty("lifetime")
+ public Long getLifetime() {
+ return lifetime;
+ }
+
+ public void setLifetime(Long lifetime) {
+ this.lifetime = lifetime;
+ }
+
+ /**
+ * Advanced scheduling and placement policies (optional). If not specified, it
+ * defaults to the default placement policy of the app owner. The design of
+ * placement policies are in the works. It is not very clear at this point,
+ * how policies in conjunction with labels be exposed to application owners.
+ * This is a placeholder for now. The advanced structure of this attribute
+ * will be determined by YARN-4902.
+ **/
+ public Application placementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the app owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to application owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.")
+ @JsonProperty("placement_policy")
+ public PlacementPolicy getPlacementPolicy() {
+ return placementPolicy;
+ }
+
+ @XmlElement(name = "placement_policy")
+ public void setPlacementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ }
+
+ /**
+ * Components of an application.
+ **/
+ public Application components(List<Component> components) {
+ this.components = components;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Components of an application.")
+ @JsonProperty("components")
+ public List<Component> getComponents() {
+ return components;
+ }
+
+ public void setComponents(List<Component> components) {
+ this.components = components;
+ }
+
+ public void addComponent(Component component) {
+ components.add(component);
+ }
+
+ public Component getComponent(String name) {
+ for (Component component : components) {
+ if (component.getName().equals(name)) {
+ return component;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Config properties of an application. Configurations provided at the
+ * application/global level are available to all the components. Specific
+ * properties can be overridden at the component level.
+ **/
+ public Application configuration(Configuration configuration) {
+ this.configuration = configuration;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Config properties of an application. Configurations provided at the application/global level are available to all the components. Specific properties can be overridden at the component level.")
+ @JsonProperty("configuration")
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public void setConfiguration(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ /**
+ * Containers of a started application. Specifying a value for this attribute
+ * for the POST payload raises a validation error. This blob is available only
+ * in the GET response of a started application.
+ **/
+ public Application containers(List<Container> containers) {
+ this.containers = containers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Containers of a started application. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.")
+ @JsonProperty("containers")
+ public List<Container> getContainers() {
+ return containers;
+ }
+
+ public void setContainers(List<Container> containers) {
+ this.containers = containers;
+ }
+
+ public void addContainer(Container container) {
+ this.containers.add(container);
+ }
+
+ /**
+ * State of the application. Specifying a value for this attribute for the
+ * POST payload raises a validation error. This attribute is available only in
+ * the GET response of a started application.
+ **/
+ public Application state(ApplicationState state) {
+ this.state = state;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "State of the application. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started application.")
+ @JsonProperty("state")
+ public ApplicationState getState() {
+ return state;
+ }
+
+ public void setState(ApplicationState state) {
+ this.state = state;
+ }
+
+ /**
+ * A blob of key-value pairs of quicklinks to be exported for an application.
+ **/
+ public Application quicklinks(Map<String, String> quicklinks) {
+ this.quicklinks = quicklinks;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key-value pairs of quicklinks to be exported for an application.")
+ @JsonProperty("quicklinks")
+ public Map<String, String> getQuicklinks() {
+ return quicklinks;
+ }
+
+ public void setQuicklinks(Map<String, String> quicklinks) {
+ this.quicklinks = quicklinks;
+ }
+
+ /**
+ * The YARN queue that this application should be submitted to.
+ **/
+ public Application queue(String queue) {
+ this.queue = queue;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The YARN queue that this application should be submitted to.")
+ @JsonProperty("queue")
+ public String getQueue() {
+ return queue;
+ }
+
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Application application = (Application) o;
+ return Objects.equals(this.name, application.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Application {\n");
+
+ sb.append(" name: ").append(toIndentedString(name)).append("\n");
+ sb.append(" id: ").append(toIndentedString(id)).append("\n");
+ sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
+ sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
+ sb.append(" launchCommand: ").append(toIndentedString(launchCommand))
+ .append("\n");
+ sb.append(" launchTime: ").append(toIndentedString(launchTime))
+ .append("\n");
+ sb.append(" numberOfContainers: ")
+ .append(toIndentedString(numberOfContainers)).append("\n");
+ sb.append(" numberOfRunningContainers: ")
+ .append(toIndentedString(numberOfRunningContainers)).append("\n");
+ sb.append(" lifetime: ").append(toIndentedString(lifetime)).append("\n");
+ sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy))
+ .append("\n");
+ sb.append(" components: ").append(toIndentedString(components))
+ .append("\n");
+ sb.append(" configuration: ").append(toIndentedString(configuration))
+ .append("\n");
+ sb.append(" containers: ").append(toIndentedString(containers))
+ .append("\n");
+ sb.append(" state: ").append(toIndentedString(state)).append("\n");
+ sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
+ .append("\n");
+ sb.append(" queue: ").append(toIndentedString(queue)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java
new file mode 100644
index 0000000..acef562
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * The current state of an application.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "The current state of an application.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public enum ApplicationState {
+ ACCEPTED, STARTED, READY, STOPPED, FAILED;
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[75/86] [abbrv] hadoop git commit: Rebase onto trunk - fix some
issues. Contributed by Rohith Sharma K S
Posted by ji...@apache.org.
Rebase onto trunk - fix some issues. Contributed by Rohith Sharma K S
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7164b079
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7164b079
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7164b079
Branch: refs/heads/yarn-native-services
Commit: 7164b079b0f77eab6cd1fccf9fa95ee2f24688c0
Parents: 54d2e6b
Author: Jian He <ji...@apache.org>
Authored: Mon Sep 4 22:07:24 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 25 16:37:24 2017 -0700
----------------------------------------------------------------------
.../hadoop-yarn-services-api/pom.xml | 2 +-
.../hadoop-yarn-services-core/pom.xml | 2 +-
.../ServiceTimelinePublisher.java | 32 ++++++++++++--------
.../TestServiceTimelinePublisher.java | 2 --
.../hadoop-yarn-services/pom.xml | 2 +-
.../src/main/webapp/app/routes/yarn-services.js | 2 +-
6 files changed, 23 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7164b079/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index 1077ccd..74d9681 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-applications</artifactId>
- <version>3.0.0-beta1-SNAPSHOT</version>
+ <version>3.1.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-yarn-services-api</artifactId>
<name>Apache Hadoop YARN Services API</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7164b079/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index d9b7adb..fb07edc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-services</artifactId>
- <version>3.0.0-beta1-SNAPSHOT</version>
+ <version>3.1.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-yarn-services-core</artifactId>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7164b079/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
index 5e65ad9..9e7d8e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
@@ -66,6 +66,7 @@ public class ServiceTimelinePublisher extends CompositeService {
protected void serviceInit(org.apache.hadoop.conf.Configuration configuration)
throws Exception {
addService(timelineClient);
+ super.serviceInit(configuration);
}
@@ -95,8 +96,8 @@ public class ServiceTimelinePublisher extends CompositeService {
// create info keys
Map<String, Object> entityInfos = new HashMap<String, Object>();
entityInfos.put(ServiceTimelineMetricsConstants.NAME, service.getName());
- entityInfos.put(ServiceTimelineMetricsConstants.STATE,
- service.getState().toString());
+// entityInfos.put(ServiceTimelineMetricsConstants.STATE,
+// service.getState().toString());
entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME,
currentTimeMillis);
entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS,
@@ -232,18 +233,23 @@ public class ServiceTimelinePublisher extends CompositeService {
// create info keys
Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_ID,
- component.getArtifact().getId());
- entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_TYPE,
- component.getArtifact().getType().toString());
- if (component.getResource().getProfile() != null) {
- entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_PROFILE,
- component.getResource().getProfile());
+ if (component.getArtifact() != null) {
+ entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_ID,
+ component.getArtifact().getId());
+ entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_TYPE,
+ component.getArtifact().getType().toString());
+ }
+
+ if (component.getResource() != null) {
+ entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_CPU,
+ component.getResource().getCpus());
+ entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_MEMORY,
+ component.getResource().getMemory());
+ if (component.getResource().getProfile() != null) {
+ entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_PROFILE,
+ component.getResource().getProfile());
+ }
}
- entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_CPU,
- component.getResource().getCpus());
- entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_MEMORY,
- component.getResource().getMemory());
if (component.getLaunchCommand() != null) {
entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_COMMAND,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7164b079/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
index b742553..e7c7600 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
@@ -82,9 +82,7 @@ public class TestServiceTimelinePublisher {
timelineClient =
new DummyTimelineClient(ApplicationId.fromString(SERVICEID));
serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
- timelineClient.init(config);
serviceTimelinePublisher.init(config);
- timelineClient.start();
serviceTimelinePublisher.start();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7164b079/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
index 1233804..716fdb7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-beta1-SNAPSHOT</version>
+ <version>3.1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-services</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7164b079/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-services.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-services.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-services.js
index 0f74f2f..a6535ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-services.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-services.js
@@ -23,7 +23,7 @@ export default AbstractRoute.extend({
model() {
return Ember.RSVP.hash({
apps: this.store.query('yarn-app', {
- applicationTypes: "yarn-native-service"
+ applicationTypes: "yarn-service"
}),
});
},
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[84/86] [abbrv] hadoop git commit: Rebased onto trunk - fix conflicts
Posted by ji...@apache.org.
Rebased onto trunk - fix conflicts
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b69afb7a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b69afb7a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b69afb7a
Branch: refs/heads/yarn-native-services
Commit: b69afb7af79a60abb6985326f10b93fc275a7c76
Parents: 803eb06
Author: Jian He <ji...@apache.org>
Authored: Mon Sep 11 17:34:30 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 25 16:37:25 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/service/ServiceScheduler.java | 11 +++++++++++
1 file changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b69afb7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index fb2fd16..7b809b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -574,6 +574,11 @@ public class ServiceScheduler extends CompositeService {
}
+ @Override public void onContainerResourceUpdated(ContainerId containerId,
+ Resource resource) {
+
+ }
+
@Override public void onGetContainerStatusError(ContainerId containerId,
Throwable t) {
@@ -586,6 +591,12 @@ public class ServiceScheduler extends CompositeService {
}
@Override
+ public void onUpdateContainerResourceError(ContainerId containerId,
+ Throwable t) {
+
+ }
+
+ @Override
public void onStopContainerError(ContainerId containerId, Throwable t) {
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java
deleted file mode 100644
index da122da..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.servicemonitor;
-
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.util.Map;
-
-/**
- * Probe for a port being open.
- */
-public class PortProbe extends Probe {
- protected static final Logger log = LoggerFactory.getLogger(PortProbe.class);
- private final int port;
- private final int timeout;
-
- public PortProbe(int port, int timeout) {
- super("Port probe of " + port + " for " + timeout + "ms", null);
- this.port = port;
- this.timeout = timeout;
- }
-
- public static PortProbe create(Map<String, String> props)
- throws IOException {
- int port = getPropertyInt(props, PORT_PROBE_PORT, null);
-
- if (port >= 65536) {
- throw new IOException(PORT_PROBE_PORT + " " + port + " is out of " +
- "range");
- }
-
- int timeout = getPropertyInt(props, PORT_PROBE_CONNECT_TIMEOUT,
- PORT_PROBE_CONNECT_TIMEOUT_DEFAULT);
-
- return new PortProbe(port, timeout);
- }
-
- /**
- * Try to connect to the (host,port); a failure to connect within
- * the specified timeout is a failure.
- * @param instance role instance
- * @return the outcome
- */
- @Override
- public ProbeStatus ping(ComponentInstance instance) {
- ProbeStatus status = new ProbeStatus();
-
- if (instance.getContainerStatus() == null || SliderUtils
- .isEmpty(instance.getContainerStatus().getIPs())) {
- status.fail(this, new IOException(
- instance.getCompInstanceName() + ": IP is not available yet"));
- return status;
- }
-
- String ip = instance.getContainerStatus().getIPs().get(0);
- InetSocketAddress sockAddr = new InetSocketAddress(ip, port);
- Socket socket = new Socket();
- try {
- if (log.isDebugEnabled()) {
- log.debug(instance.getCompInstanceName() + ": Connecting " + sockAddr
- .toString() + ", timeout=" + MonitorUtils
- .millisToHumanTime(timeout));
- }
- socket.connect(sockAddr, timeout);
- status.succeed(this);
- } catch (Throwable e) {
- String error =
- instance.getCompInstanceName() + ": Probe " + sockAddr + " failed";
- log.debug(error, e);
- status.fail(this, new IOException(error, e));
- } finally {
- IOUtils.closeSocket(socket);
- }
- return status;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/Probe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/Probe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/Probe.java
deleted file mode 100644
index 4809b45..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/Probe.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.servicemonitor;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * Base class of all probes.
- */
-public abstract class Probe implements MonitorKeys {
-
- protected final Configuration conf;
- private String name;
-
- /**
- * Create a probe of a specific name
- *
- * @param name probe name
- * @param conf configuration being stored.
- */
- public Probe(String name, Configuration conf) {
- this.name = name;
- this.conf = conf;
- }
-
-
- protected void setName(String name) {
- this.name = name;
- }
-
- public String getName() {
- return name;
- }
-
-
- @Override
- public String toString() {
- return getName();
- }
-
- public static String getProperty(Map<String, String> props, String name,
- String defaultValue) throws IOException {
- String value = props.get(name);
- if (StringUtils.isEmpty(value)) {
- if (defaultValue == null) {
- throw new IOException(name + " not specified");
- }
- return defaultValue;
- }
- return value;
- }
-
- public static int getPropertyInt(Map<String, String> props, String name,
- Integer defaultValue) throws IOException {
- String value = props.get(name);
- if (StringUtils.isEmpty(value)) {
- if (defaultValue == null) {
- throw new IOException(name + " not specified");
- }
- return defaultValue;
- }
- return Integer.parseInt(value);
- }
-
- /**
- * perform any prelaunch initialization
- */
- public void init() throws IOException {
-
- }
-
- /**
- * Ping the endpoint. All exceptions must be caught and included in the
- * (failure) status.
- *
- * @param instance instance to ping
- * @return the status
- */
- public abstract ProbeStatus ping(ComponentInstance instance);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeStatus.java
deleted file mode 100644
index 24668bd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeStatus.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.servicemonitor;
-
-import java.io.Serializable;
-import java.util.Date;
-
-/**
- * Status message of a probe. This is designed to be sent over the wire, though the exception
- * Had better be unserializable at the far end if that is to work.
- */
-public final class ProbeStatus implements Serializable {
- private static final long serialVersionUID = 165468L;
-
- private long timestamp;
- private String timestampText;
- private boolean success;
- private boolean realOutcome;
- private String message;
- private Throwable thrown;
- private transient Probe originator;
-
- public ProbeStatus() {
- }
-
- public ProbeStatus(long timestamp, String message, Throwable thrown) {
- this.success = false;
- this.message = message;
- this.thrown = thrown;
- setTimestamp(timestamp);
- }
-
- public ProbeStatus(long timestamp, String message) {
- this.success = true;
- setTimestamp(timestamp);
- this.message = message;
- this.thrown = null;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public void setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- timestampText = new Date(timestamp).toString();
- }
-
- public boolean isSuccess() {
- return success;
- }
-
- /**
- * Set both the success and the real outcome bits to the same value
- * @param success the new value
- */
- public void setSuccess(boolean success) {
- this.success = success;
- realOutcome = success;
- }
-
- public String getTimestampText() {
- return timestampText;
- }
-
- public boolean getRealOutcome() {
- return realOutcome;
- }
-
- public String getMessage() {
- return message;
- }
-
- public void setMessage(String message) {
- this.message = message;
- }
-
- public Throwable getThrown() {
- return thrown;
- }
-
- public void setThrown(Throwable thrown) {
- this.thrown = thrown;
- }
-
- /**
- * Get the probe that generated this result. May be null
- * @return a possibly null reference to a probe
- */
- public Probe getOriginator() {
- return originator;
- }
-
- /**
- * The probe has succeeded -capture the current timestamp, set
- * success to true, and record any other data needed.
- * @param probe probe
- */
- public void succeed(Probe probe) {
- finish(probe, true, probe.getName(), null);
- }
-
- /**
- * A probe has failed either because the test returned false, or an exception
- * was thrown. The {@link #success} field is set to false, any exception
- * thrown is recorded.
- * @param probe probe that failed
- * @param thrown an exception that was thrown.
- */
- public void fail(Probe probe, Throwable thrown) {
- finish(probe, false, "Failure in " + probe, thrown);
- }
-
- public void finish(Probe probe, boolean succeeded, String text, Throwable thrown) {
- setTimestamp(System.currentTimeMillis());
- setSuccess(succeeded);
- originator = probe;
- message = text;
- this.thrown = thrown;
- }
-
- @Override
- public String toString() {
- LogEntryBuilder builder = new LogEntryBuilder("Probe Status");
- builder.elt("time", timestampText)
- .elt("outcome", (success ? "success" : "failure"));
-
- if (success != realOutcome) {
- builder.elt("originaloutcome", (realOutcome ? "success" : "failure"));
- }
- builder.elt("message", message);
- if (thrown != null) {
- builder.elt("exception", thrown);
- }
-
- return builder.toString();
- }
-
- /**
- * Flip the success bit on while the real outcome bit is kept false
- */
- public void markAsSuccessful() {
- success = true;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
deleted file mode 100644
index 43f0e4e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.utility;
-
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.common.tools.ConfigHelper;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.zk.ZookeeperUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Base service for the standard slider client/server services
- */
-public abstract class AbstractSliderLaunchedService extends
- LaunchedWorkflowCompositeService {
- private static final Logger log =
- LoggerFactory.getLogger(AbstractSliderLaunchedService.class);
-
- protected AbstractSliderLaunchedService(String name) {
- super(name);
- // make sure all the yarn configs get loaded
- ConfigHelper.registerDeprecatedConfigItems();
- }
-
- /**
- * look up the registry quorum from the config
- * @return the quorum string
- * @throws BadConfigException if it is not there or invalid
- */
- public String lookupZKQuorum() throws BadConfigException {
-
- String registryQuorum = getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
-
- // though if neither is set: trouble
- if (SliderUtils.isUnset(registryQuorum)) {
- throw new BadConfigException(
- "No Zookeeper quorum provided in the"
- + " configuration property " + RegistryConstants.KEY_REGISTRY_ZK_QUORUM
- );
- }
- ZookeeperUtils.splitToHostsAndPortsStrictly(registryQuorum);
- return registryQuorum;
- }
-
- /**
- * Create, adopt ,and start the YARN registration service
- * @return the registry operations service, already deployed as a child
- * of the AbstractSliderLaunchedService instance.
- */
- public RegistryOperations startRegistryOperationsService()
- throws BadConfigException {
-
- // push back the slider registry entry if needed
- RegistryOperations registryWriterService =
- createRegistryOperationsInstance();
- deployChildService(registryWriterService);
- return registryWriterService;
- }
-
- /**
- * Create the registry operations instance. This is to allow
- * subclasses to instantiate a subclass service
- * @return an instance to match to the lifecycle of this service
- */
- protected RegistryOperations createRegistryOperationsInstance() {
- return RegistryOperationsFactory.createInstance("YarnRegistry", getConfig());
- }
-
- /**
- * Utility method to require an argument to be set (non null, non-empty)
- * @param argname argument name
- * @param value value
- * @throws BadCommandArgumentsException if the condition is not met
- */
- protected static void requireArgumentSet(String argname, String value)
- throws BadCommandArgumentsException {
- require(isSet(value), "Required argument %s missing", argname );
- }
-
- /**
- * Require a condition to hold; throw {@link BadCommandArgumentsException} if not.
- * The exception text is the formatted message.
- * @param condition condition
- * @param message string to format
- * @param args list of arguments to format.
- * @throws BadCommandArgumentsException
- */
- protected static void require(boolean condition, String message,
- Object... args)
- throws BadCommandArgumentsException {
- if (!condition) {
- throw new BadCommandArgumentsException(message, args);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
deleted file mode 100644
index 40ceab8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.utility;
-
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.service.ServiceStateChangeListener;
-
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Wait for a service to stop.
- *
- * WARNING: the notification may come in as soon as the service enters
- * the stopped state: it may take some time for the actual stop operation
- * to complete.
- */
-public class EndOfServiceWaiter implements ServiceStateChangeListener {
-
- private final AtomicBoolean finished = new AtomicBoolean(false);
- private final String name;
- private Service service;
-
- /**
- * Wait for a service; use the service name as this instance's name
- * @param service service
- */
- public EndOfServiceWaiter(Service service) {
- this(service.getName(), service);
- }
-
-
- /**
- * Wait for a service
- * @param name name for messages
- * @param service service
- */
- public EndOfServiceWaiter(String name, Service service) {
- this.name = name;
- this.service = service;
- service.registerServiceListener(this);
- }
-
- public synchronized void waitForServiceToStop(long timeout) throws
- InterruptedException, TimeoutException {
- service.waitForServiceToStop(timeout);
- if (!finished.get()) {
- wait(timeout);
- if (!finished.get()) {
- throw new TimeoutException(name
- + " did not finish after " + timeout +
- " milliseconds");
- }
- }
- }
-
- /**
- * Wait for service state change callbacks; notify self if the service has
- * now stopped
- * @param service service
- */
- @Override
- public synchronized void stateChanged(Service service) {
- if (service.isInState(Service.STATE.STOPPED)) {
- finished.set(true);
- notify();
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
deleted file mode 100644
index bcd1969..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.utility;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.Service;
-import org.apache.slider.core.main.LauncherExitCodes;
-import org.apache.slider.core.main.RunService;
-import org.apache.slider.server.services.workflow.WorkflowCompositeService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This is a workflow compositoe service which can be launched from the CLI
- * ... catches the arguments and implements a stub runService operation.
- */
-public class LaunchedWorkflowCompositeService extends WorkflowCompositeService
- implements RunService {
- private static final Logger log = LoggerFactory.getLogger(
- LaunchedWorkflowCompositeService.class);
- private String[] argv;
-
- public LaunchedWorkflowCompositeService(String name) {
- super(name);
- }
-
- public LaunchedWorkflowCompositeService(String name, Service... children) {
- super(name, children);
- }
-
- /**
- * Implementation of set-ness, groovy definition of true/false for a string
- * @param s
- * @return true iff the string is non-null and non-empty
- */
- protected static boolean isUnset(String s) {
- return StringUtils.isEmpty(s);
- }
-
- protected static boolean isSet(String s) {
- return StringUtils.isNotEmpty(s);
- }
-
- protected String[] getArgv() {
- return argv;
- }
-
- /**
- * Pre-init argument binding
- * @param config the initial configuration build up by the
- * service launcher.
- * @param args argument list list of arguments passed to the command line
- * after any launcher-specific commands have been stripped.
- * @return the configuration
- * @throws Exception
- */
- @Override
- public Configuration bindArgs(Configuration config, String... args) throws
- Exception {
- this.argv = args;
- if (log.isDebugEnabled()) {
- log.debug("Binding {} Arguments:", args.length);
-
- StringBuilder builder = new StringBuilder();
- for (String arg : args) {
- builder.append('"').append(arg).append("\" ");
- }
- log.debug(builder.toString());
- }
- return config;
- }
-
- @Override
- public int runService() throws Throwable {
- return LauncherExitCodes.EXIT_SUCCESS;
- }
-
- @Override
- public synchronized void addService(Service service) {
- Preconditions.checkArgument(service != null, "null service argument");
- super.addService(service);
- }
-
- /**
- * Run a child service -initing and starting it if this
- * service has already passed those parts of its own lifecycle
- * @param service the service to start
- */
- protected boolean deployChildService(Service service) {
- service.init(getConfig());
- addService(service);
- if (isInState(STATE.STARTED)) {
- service.start();
- return true;
- }
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java
deleted file mode 100644
index 6ab9de6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.utility;
-
-import org.apache.slider.server.appmaster.web.rest.RestPaths;
-
-import java.util.regex.Pattern;
-
-/**
- * Utility class to validate strings against a predefined pattern.
- */
-public class PatternValidator {
-
- public static final String E_INVALID_NAME =
- "Invalid name %s does not match the pattern pattern %s ";
- private final Pattern valid;
- private final String pattern;
-
- public PatternValidator(String pattern) {
- this.pattern = pattern;
- valid = Pattern.compile(pattern);
- }
-
- /**
- * Validate the name -restricting it to the set defined in
- * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP}
- * @param name name to validate
- * @throws IllegalArgumentException if not a valid name
- */
- public void validate(String name) {
- if (!matches(name)) {
- throw new IllegalArgumentException(
- String.format(E_INVALID_NAME, name, pattern));
- }
- }
-
- /**
- * Query to see if the pattern matches
- * @param name name to validate
- * @return true if the string matches the pattern
- */
- public boolean matches(String name) {
- return valid.matcher(name).matches();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java
deleted file mode 100644
index ebfcb99..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.utility;
-
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.webapp.WebApp;
-
-/**
- * Contains a webapp reference and stops it in teardown if non-null
- * <p>
- * It does not start the application.
- * Access to the field is not synchronized across threads; it is the
- * responsibility of the caller.
- */
-public class WebAppService<T extends WebApp> extends AbstractService {
-
- private volatile T webApp;
-
- public WebAppService(String name) {
- super(name);
- }
-
- public WebAppService(String name, T app) {
- super(name);
- webApp = app;
- }
-
- public T getWebApp() {
- return webApp;
- }
-
- public void setWebApp(T webApp) {
- this.webApp = webApp;
- }
-
-
- @Override
- protected void serviceStart() throws Exception {
-
- }
-
- /**
- * Stop operation stops the webapp; sets the reference to null
- * @throws Exception
- */
- @Override
- protected void serviceStop() throws Exception {
- if (webApp != null) {
- webApp.stop();
- webApp = null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
deleted file mode 100644
index 8b711aa..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.service.AbstractService;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-/**
- * Service that closes the closeable supplied during shutdown, if not null.
- *
- * As the Service interface itself extends Closeable, this service
- * can be used to shut down other services if desired.
- */
-public class ClosingService<C extends Closeable> extends AbstractService {
-
- private C closeable;
-
- public ClosingService(String name) {
- super(name);
- }
-
- /**
- * Construct an instance of the service
- * @param name service name
- * @param closeable closeable to close (may be null)
- */
- public ClosingService(String name,
- C closeable) {
- super(name);
- this.closeable = closeable;
- }
-
- /**
- * Construct an instance of the service, using the default name
- * @param closeable closeable to close (may be null)
- */
- public ClosingService(C closeable) {
- this("ClosingService", closeable);
- }
-
-
- /**
- * Get the closeable
- * @return the closeable
- */
- public synchronized C getCloseable() {
- return closeable;
- }
-
- /**
- * Set or update the closeable.
- * @param closeable
- */
- public synchronized void setCloseable(C closeable) {
- this.closeable = closeable;
- }
-
- /**
- * Stop routine will close the closeable -if not null - and set the
- * reference to null afterwards
- * This operation does raise any exception on the close, though it does
- * record it
- */
- @Override
- protected void serviceStop() {
- C target = getCloseable();
- if (target != null) {
- try {
- target.close();
- } catch (IOException ioe) {
- noteFailure(ioe);
- }
- setCloseable(null);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
deleted file mode 100644
index 352be49..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
+++ /dev/null
@@ -1,301 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.service.ServiceStateException;
-import org.apache.slider.core.main.ServiceLaunchException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Service wrapper for an external program that is launched and can/will terminate.
- * This service is notified when the subprocess terminates, and stops itself
- * and converts a non-zero exit code into a failure exception.
- *
- * <p>
- * Key Features:
- * <ol>
- * <li>The property {@link #executionTimeout} can be set to set a limit
- * on the duration of a process</li>
- * <li>Output is streamed to the output logger provided</li>.
- * <li>The most recent lines of output are saved to a linked list</li>.
- * <li>A synchronous callback, {@link LongLivedProcessLifecycleEvent}, is raised on the start
- * and finish of a process.</li>
- * </ol>
- *
- * Usage:
- * <p></p>
- * The service can be built in the constructor, {@link #ForkedProcessService(String, Map, List)},
- * or have its simple constructor used to instantiate the service, then the
- * {@link #build(Map, List)} command used to define the environment variables
- * and list of commands to execute. One of these two options MUST be exercised
- * before calling the services's {@link #start()} method.
- * <p></p>
- * The forked process is executed in the service's {@link #serviceStart()} method;
- * if still running when the service is stopped, {@link #serviceStop()} will
- * attempt to stop it.
- * <p></p>
- *
- * The service delegates process execution to {@link LongLivedProcess},
- * receiving callbacks via the {@link LongLivedProcessLifecycleEvent}.
- * When the service receives a callback notifying that the process has completed,
- * it calls its {@link #stop()} method. If the error code was non-zero,
- * the service is logged as having failed.
- */
-public class ForkedProcessService
- extends WorkflowExecutorService<ExecutorService>
- implements LongLivedProcessLifecycleEvent, Runnable {
-
- /**
- * Log for the forked master process
- */
- private static final Logger LOG =
- LoggerFactory.getLogger(ForkedProcessService.class);
-
- private final AtomicBoolean processTerminated = new AtomicBoolean(false);
- private boolean processStarted = false;
- private LongLivedProcess process;
- private int executionTimeout = -1;
- private int timeoutCode = 1;
- /**
- log to log to; defaults to this service log
- */
- private Logger processLog = LOG;
-
- /**
- * Exit code set when the spawned process exits
- */
- private AtomicInteger exitCode = new AtomicInteger(0);
-
- /**
- * Create an instance of the service
- * @param name a name
- */
- public ForkedProcessService(String name) {
- super(name);
- }
-
- /**
- * Create an instance of the service, set up the process
- * @param name a name
- * @param commandList list of commands is inserted on the front
- * @param env environment variables above those generated by
- * @throws IOException IO problems
- */
- public ForkedProcessService(String name,
- Map<String, String> env,
- List<String> commandList) throws IOException {
- super(name);
- build(env, commandList);
- }
-
- @Override //AbstractService
- protected void serviceStart() throws Exception {
- if (process == null) {
- throw new ServiceStateException("Process not yet configured");
- }
- //now spawn the process -expect updates via callbacks
- process.start();
- }
-
- @Override //AbstractService
- protected void serviceStop() throws Exception {
- completed();
- stopForkedProcess();
- }
-
- private void stopForkedProcess() {
- if (process != null) {
- process.stop();
- }
- }
-
- /**
- * Set the process log. This may be null for "do not log"
- * @param processLog process log
- */
- public void setProcessLog(Logger processLog) {
- this.processLog = processLog;
- process.setProcessLog(processLog);
- }
-
- /**
- * Set the timeout by which time a process must have finished -or -1 for forever
- * @param timeout timeout in milliseconds
- */
- public void setTimeout(int timeout, int code) {
- this.executionTimeout = timeout;
- this.timeoutCode = code;
- }
-
- /**
- * Build the process to execute when the service is started
- * @param commandList list of commands is inserted on the front
- * @param env environment variables above those generated by
- * @throws IOException IO problems
- */
- public void build(Map<String, String> env,
- List<String> commandList)
- throws IOException {
- assert process == null;
-
- process = new LongLivedProcess(getName(), processLog, commandList);
- process.setLifecycleCallback(this);
- //set the env variable mapping
- process.putEnvMap(env);
- }
-
- @Override // notification from executed process
- public synchronized void onProcessStarted(LongLivedProcess process) {
- LOG.debug("Process has started");
- processStarted = true;
- if (executionTimeout > 0) {
- setExecutor(ServiceThreadFactory.singleThreadExecutor(getName(), true));
- execute(this);
- }
- }
-
- @Override // notification from executed process
- public void onProcessExited(LongLivedProcess process,
- int uncorrected,
- int code) {
- try {
- synchronized (this) {
- completed();
- //note whether or not the service had already stopped
- LOG.debug("Process has exited with exit code {}", code);
- if (code != 0) {
- reportFailure(code, getName() + " failed with code " + code);
- }
- }
- } finally {
- stop();
- }
- }
-
- private void reportFailure(int code, String text) {
- //error
- ServiceLaunchException execEx = new ServiceLaunchException(code, text);
- LOG.debug("Noting failure", execEx);
- noteFailure(execEx);
- }
-
- /**
- * handle timeout response by escalating it to a failure
- */
- @Override
- public void run() {
- try {
- synchronized (processTerminated) {
- if (!processTerminated.get()) {
- processTerminated.wait(executionTimeout);
- }
- }
-
- } catch (InterruptedException e) {
- //assume signalled; exit
- }
- //check the status; if the marker isn't true, bail
- if (!processTerminated.getAndSet(true)) {
- LOG.info("process timeout: reporting error code {}", timeoutCode);
-
- //timeout
- if (isInState(STATE.STARTED)) {
- //trigger a failure
- stopForkedProcess();
- }
- reportFailure(timeoutCode, getName() + ": timeout after " + executionTimeout
- + " millis: exit code =" + timeoutCode);
- }
- }
-
- /**
- * Note the process as having completed.
- * The process marked as terminated
- * -and anything synchronized on <code>processTerminated</code>
- * is notified
- */
- protected void completed() {
- processTerminated.set(true);
- synchronized (processTerminated) {
- processTerminated.notify();
- }
- }
-
- public boolean isProcessTerminated() {
- return processTerminated.get();
- }
-
- public synchronized boolean isProcessStarted() {
- return processStarted;
- }
-
- /**
- * Is a process running: between started and terminated
- * @return true if the process is up.
- */
- public synchronized boolean isProcessRunning() {
- return processStarted && !isProcessTerminated();
- }
-
-
- public Integer getExitCode() {
- return process.getExitCode();
- }
-
- public int getExitCodeSignCorrected() {
- Integer exitCode = process.getExitCodeSignCorrected();
- if (exitCode == null) return -1;
- return exitCode;
- }
-
- /**
- * Get the recent output from the process, or [] if not defined
- * @return a possibly empty list
- */
- public List<String> getRecentOutput() {
- return process != null
- ? process.getRecentOutput()
- : new LinkedList<String>();
- }
-
- /**
- * Get the recent output from the process, or [] if not defined
- *
- * @param finalOutput flag to indicate "wait for the final output of the process"
- * @param duration the duration, in ms,
- * to wait for recent output to become non-empty
- * @return a possibly empty list
- */
- public List<String> getRecentOutput(boolean finalOutput, int duration) {
- if (process == null) {
- return new LinkedList<>();
- }
- return process.getRecentOutput(finalOutput, duration);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
deleted file mode 100644
index 90a8d40..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
+++ /dev/null
@@ -1,599 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Execute a long-lived process.
- *
- * <p>
- * Hadoop's {@link org.apache.hadoop.util.Shell} class assumes it is executing
- * a short lived application; this class allows for the process to run for the
- * life of the Java process that forked it.
- * It is designed to be embedded inside a YARN service, though this is not
- * the sole way that it can be used
- * <p>
- * Key Features:
- * <ol>
- * <li>Output is streamed to the output logger provided</li>.
- * <li>the input stream is closed as soon as the process starts.</li>
- * <li>The most recent lines of output are saved to a linked list</li>.
- * <li>A synchronous callback, {@link LongLivedProcessLifecycleEvent},
- * is raised on the start and finish of a process.</li>
- * </ol>
- *
- */
-public class LongLivedProcess implements Runnable {
- /**
- * Limit on number of lines to retain in the "recent" line list:{@value}
- */
- public static final int RECENT_LINE_LOG_LIMIT = 64;
-
- /**
- * Const defining the time in millis between polling for new text.
- */
- private static final int STREAM_READER_SLEEP_TIME = 200;
-
- /**
- * limit on the length of a stream before it triggers an automatic newline.
- */
- private static final int LINE_LENGTH = 256;
- private final ProcessBuilder processBuilder;
- private Process process;
- private Integer exitCode = null;
- private final String name;
- private final ExecutorService processExecutor;
- private final ExecutorService logExecutor;
-
- private ProcessStreamReader processStreamReader;
- //list of recent lines, recorded for extraction into reports
- private final List<String> recentLines = new LinkedList<>();
- private int recentLineLimit = RECENT_LINE_LOG_LIMIT;
- private LongLivedProcessLifecycleEvent lifecycleCallback;
- private final AtomicBoolean finalOutputProcessed = new AtomicBoolean(false);
-
- /**
- * Log supplied in the constructor for the spawned process -accessible
- * to inner classes
- */
- private Logger processLog;
-
- /**
- * Class log -accessible to inner classes
- */
- private static final Logger LOG = LoggerFactory.getLogger(LongLivedProcess.class);
-
- /**
- * flag to indicate that the process is done
- */
- private final AtomicBoolean finished = new AtomicBoolean(false);
-
- /**
- * Create an instance
- * @param name process name
- * @param processLog log for output (or null)
- * @param commands command list
- */
- public LongLivedProcess(String name,
- Logger processLog,
- List<String> commands) {
- Preconditions.checkArgument(commands != null, "commands");
-
- this.name = name;
- this.processLog = processLog;
- ServiceThreadFactory factory = new ServiceThreadFactory(name, true);
- processExecutor = Executors.newSingleThreadExecutor(factory);
- logExecutor = Executors.newSingleThreadExecutor(factory);
- processBuilder = new ProcessBuilder(commands);
- processBuilder.redirectErrorStream(false);
- }
-
- /**
- * Set the limit on recent lines to retain
- * @param recentLineLimit size of rolling list of recent lines.
- */
- public void setRecentLineLimit(int recentLineLimit) {
- this.recentLineLimit = recentLineLimit;
- }
-
- /**
- * Set an optional application exit callback
- * @param lifecycleCallback callback to notify on application exit
- */
- public void setLifecycleCallback(LongLivedProcessLifecycleEvent lifecycleCallback) {
- this.lifecycleCallback = lifecycleCallback;
- }
-
- /**
- * Add an entry to the environment
- * @param envVar envVar -must not be null
- * @param val value
- */
- public void setEnv(String envVar, String val) {
- Preconditions.checkArgument(envVar != null, "envVar");
- Preconditions.checkArgument(val != null, "val");
- processBuilder.environment().put(envVar, val);
- }
-
- /**
- * Bulk set the environment from a map. This does
- * not replace the existing environment, just extend it/overwrite single
- * entries.
- * @param map map to add
- */
- public void putEnvMap(Map<String, String> map) {
- for (Map.Entry<String, String> entry : map.entrySet()) {
- String val = entry.getValue();
- String key = entry.getKey();
- setEnv(key, val);
- }
- }
-
- /**
- * Get the process environment
- * @param variable environment variable
- * @return the value or null if there is no match
- */
- public String getEnv(String variable) {
- return processBuilder.environment().get(variable);
- }
-
- /**
- * Set the process log. Ignored once the process starts
- * @param processLog new log ... may be null
- */
- public void setProcessLog(Logger processLog) {
- this.processLog = processLog;
- }
-
- /**
- * Get the process reference
- * @return the process -null if the process is not started
- */
- public Process getProcess() {
- return process;
- }
-
- /**
- * Get the process builder -this can be manipulated
- * up to the start() operation. As there is no synchronization
- * around it, it must only be used in the same thread setting up the commmand.
- * @return the process builder
- */
- public ProcessBuilder getProcessBuilder() {
- return processBuilder;
- }
-
- /**
- * Get the command list
- * @return the comands
- */
- public List<String> getCommands() {
- return processBuilder.command();
- }
-
- public String getCommand() {
- return getCommands().get(0);
- }
-
- /**
- * probe to see if the process is running
- * @return true iff the process has been started and is not yet finished
- */
- public boolean isRunning() {
- return process != null && !finished.get();
- }
-
- /**
- * Get the exit code: null until the process has finished
- * @return the exit code or null
- */
- public Integer getExitCode() {
- return exitCode;
- }
-
- /**
- * Get the exit code sign corrected: null until the process has finished
- * @return the exit code or null
- */
- public Integer getExitCodeSignCorrected() {
- Integer result;
- if (exitCode != null) {
- result = (exitCode << 24) >> 24;
- } else {
- result = null;
- }
- return result;
- }
-
- /**
- * Stop the process if it is running.
- * This will trigger an application completion event with the given exit code
- */
- public void stop() {
- if (!isRunning()) {
- return;
- }
- process.destroy();
- }
-
- /**
- * Get a text description of the builder suitable for log output
- * @return a multiline string
- */
- protected String describeBuilder() {
- StringBuilder buffer = new StringBuilder();
- for (String arg : processBuilder.command()) {
- buffer.append('"').append(arg).append("\" ");
- }
- return buffer.toString();
- }
-
- /**
- * Dump the environment to a string builder
- * @param buffer the buffer to append to
- */
- public void dumpEnv(StringBuilder buffer) {
- buffer.append("\nEnvironment\n-----------");
- Map<String, String> env = processBuilder.environment();
- Set<String> keys = env.keySet();
- List<String> sortedKeys = new ArrayList<String>(keys);
- Collections.sort(sortedKeys);
- for (String key : sortedKeys) {
- buffer.append(key).append("=").append(env.get(key)).append('\n');
- }
- }
-
- /**
- * Exec the process
- * @return the process
- * @throws IOException on aany failure to start the process
- * @throws FileNotFoundException if the process could not be found
- */
- private Process spawnChildProcess() throws IOException {
- if (process != null) {
- throw new IOException("Process already started");
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Spawning process:\n " + describeBuilder());
- }
- try {
- process = processBuilder.start();
- } catch (IOException e) {
- // on windows, upconvert DOS error 2 from ::CreateProcess()
- // to its real meaning: FileNotFound
- if (e.toString().contains("CreateProcess error=2")) {
- FileNotFoundException fnfe =
- new FileNotFoundException(e.toString());
- fnfe.initCause(e);
- throw fnfe;
- } else {
- throw e;
- }
- }
- return process;
- }
-
- /**
- * Entry point for waiting for the program to finish
- */
- @Override // Runnable
- public void run() {
- Preconditions.checkNotNull(process, "null process");
- LOG.debug("Lifecycle callback thread running");
- //notify the callback that the process has started
- if (lifecycleCallback != null) {
- lifecycleCallback.onProcessStarted(this);
- }
- try {
- //close stdin for the process
- IOUtils.closeStream(process.getOutputStream());
- exitCode = process.waitFor();
- } catch (InterruptedException e) {
- LOG.debug("Process wait interrupted -exiting thread", e);
- } finally {
- //here the process has finished
- LOG.debug("process {} has finished", name);
- //tell the logger it has to finish too
- finished.set(true);
-
- // shut down the threads
- logExecutor.shutdown();
- try {
- logExecutor.awaitTermination(60, TimeUnit.SECONDS);
- } catch (InterruptedException ignored) {
- //ignored
- }
-
- //now call the callback if it is set
- if (lifecycleCallback != null) {
- lifecycleCallback.onProcessExited(this, exitCode,
- getExitCodeSignCorrected());
- }
- }
- }
-
- /**
- * Spawn the application
- * @throws IOException IO problems
- */
- public void start() throws IOException {
-
- spawnChildProcess();
- processStreamReader =
- new ProcessStreamReader(processLog, STREAM_READER_SLEEP_TIME);
- logExecutor.submit(processStreamReader);
- processExecutor.submit(this);
- }
-
- /**
- * Get the lines of recent output
- * @return the last few lines of output; an empty list if there are none
- * or the process is not actually running
- */
- public synchronized List<String> getRecentOutput() {
- return new ArrayList<String>(recentLines);
- }
-
- /**
- * @return whether lines of recent output are empty
- */
- public synchronized boolean isRecentOutputEmpty() {
- return recentLines.isEmpty();
- }
-
- /**
- * Query to see if the final output has been processed
- * @return
- */
- public boolean isFinalOutputProcessed() {
- return finalOutputProcessed.get();
- }
-
- /**
- * Get the recent output from the process, or [] if not defined
- *
- * @param finalOutput flag to indicate "wait for the final output of the process"
- * @param duration the duration, in ms,
- * ro wait for recent output to become non-empty
- * @return a possibly empty list
- */
- public List<String> getRecentOutput(boolean finalOutput, int duration) {
- long start = System.currentTimeMillis();
- while (System.currentTimeMillis() - start <= duration) {
- boolean finishedOutput;
- if (finalOutput) {
- // final flag means block until all data is done
- finishedOutput = isFinalOutputProcessed();
- } else {
- // there is some output
- finishedOutput = !isRecentOutputEmpty();
- }
- if (finishedOutput) {
- break;
- }
- try {
- Thread.sleep(100);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- break;
- }
- }
- return getRecentOutput();
- }
-
- /**
- * add the recent line to the list of recent lines; deleting
- * an earlier on if the limit is reached.
- *
- * Implementation note: yes, a circular array would be more
- * efficient, especially with some power of two as the modulo,
- * but is it worth the complexity and risk of errors for
- * something that is only called once per line of IO?
- * @param line line to record
- * @param isErrorStream is the line from the error stream
- * @param logger logger to log to - null for no logging
- */
- private synchronized void recordRecentLine(String line,
- boolean isErrorStream,
- Logger logger) {
- if (line == null) {
- return;
- }
- String entry = (isErrorStream ? "[ERR] " : "[OUT] ") + line;
- recentLines.add(entry);
- if (recentLines.size() > recentLineLimit) {
- recentLines.remove(0);
- }
- if (logger != null) {
- if (isErrorStream) {
- logger.warn(line);
- } else {
- logger.info(line);
- }
- }
- }
-
- /**
- * Class to read data from the two process streams, and, when run in a thread
- * to keep running until the <code>done</code> flag is set.
- * Lines are fetched from stdout and stderr and logged at info and error
- * respectively.
- */
-
- private class ProcessStreamReader implements Runnable {
- private final Logger streamLog;
- private final int sleepTime;
-
- /**
- * Create an instance
- * @param streamLog log -or null to disable logging (recent entries
- * will still be retained)
- * @param sleepTime time to sleep when stopping
- */
- private ProcessStreamReader(Logger streamLog, int sleepTime) {
- this.streamLog = streamLog;
- this.sleepTime = sleepTime;
- }
-
- /**
- * Return a character if there is one, -1 if nothing is ready yet
- * @param reader reader
- * @return the value from the reader, or -1 if it is not ready
- * @throws IOException IO problems
- */
- private int readCharNonBlocking(BufferedReader reader) throws IOException {
- if (reader.ready()) {
- return reader.read();
- } else {
- return -1;
- }
- }
-
- /**
- * Read in a line, or, if the limit has been reached, the buffer
- * so far
- * @param reader source of data
- * @param line line to build
- * @param limit limit of line length
- * @return true if the line can be printed
- * @throws IOException IO trouble
- */
- @SuppressWarnings("NestedAssignment")
- private boolean readAnyLine(BufferedReader reader,
- StringBuilder line,
- int limit)
- throws IOException {
- int next;
- while ((-1 != (next = readCharNonBlocking(reader)))) {
- if (next != '\n') {
- line.append((char) next);
- limit--;
- if (line.length() > limit) {
- //enough has been read in to print it any
- return true;
- }
- } else {
- //line end return flag to say so
- return true;
- }
- }
- //here the end of the stream is hit, or the limit
- return false;
- }
-
-
- @Override //Runnable
- @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
- public void run() {
- BufferedReader errReader = null;
- BufferedReader outReader = null;
- StringBuilder outLine = new StringBuilder(LINE_LENGTH);
- StringBuilder errorLine = new StringBuilder(LINE_LENGTH);
- try {
- errReader = new BufferedReader(
- new InputStreamReader(process.getErrorStream(), "UTF-8"));
- outReader = new BufferedReader(
- new InputStreamReader(process.getInputStream(), "UTF-8"));
- while (!finished.get()) {
- boolean processed = false;
- if (readAnyLine(errReader, errorLine, LINE_LENGTH)) {
- recordRecentLine(errorLine.toString(), true, streamLog);
- errorLine.setLength(0);
- processed = true;
- }
- if (readAnyLine(outReader, outLine, LINE_LENGTH)) {
- recordRecentLine(outLine.toString(), false, streamLog);
- outLine.setLength(0);
- processed |= true;
- }
- if (!processed && !finished.get()) {
- //nothing processed: wait a bit for data.
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException e) {
- //ignore this, rely on the done flag
- LOG.debug("Ignoring ", e);
- }
- }
- }
- // finished: cleanup
-
- //print the current error line then stream through the rest
- recordFinalOutput(errReader, errorLine, true, streamLog);
- //now do the info line
- recordFinalOutput(outReader, outLine, false, streamLog);
-
- } catch (Exception ignored) {
- LOG.warn("encountered {}", ignored, ignored);
- //process connection has been torn down
- } finally {
- // close streams
- IOUtils.closeStream(errReader);
- IOUtils.closeStream(outReader);
- //mark output as done
- finalOutputProcessed.set(true);
- }
- }
-
- /**
- * Record the final output of a process stream
- * @param reader reader of output
- * @param lineBuilder string builder into which line is built
- * @param isErrorStream flag to indicate whether or not this is the
- * is the line from the error stream
- * @param logger logger to log to
- * @throws IOException
- */
- protected void recordFinalOutput(BufferedReader reader,
- StringBuilder lineBuilder, boolean isErrorStream, Logger logger) throws
- IOException {
- String line = lineBuilder.toString();
- recordRecentLine(line, isErrorStream, logger);
- line = reader.readLine();
- while (line != null) {
- recordRecentLine(line, isErrorStream, logger);
- line = reader.readLine();
- if (Thread.interrupted()) {
- break;
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcessLifecycleEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcessLifecycleEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcessLifecycleEvent.java
deleted file mode 100644
index a13b508..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcessLifecycleEvent.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-/**
- * Callback when a long-lived application exits
- */
-public interface LongLivedProcessLifecycleEvent {
-
- /**
- * Callback when a process is started
- * @param process the process invoking the callback
- */
- void onProcessStarted(LongLivedProcess process);
-
- /**
- * Callback when a process has finished
- * @param process the process invoking the callback
- * @param exitCode exit code from the process
- * @param signCorrectedCode the code- as sign corrected
- */
- void onProcessExited(LongLivedProcess process,
- int exitCode,
- int signCorrectedCode);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceParent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceParent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceParent.java
deleted file mode 100644
index a123584..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceParent.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.service.Service;
-
-import java.util.List;
-
-/**
- * Interface for accessing services that contain one or more child
- * services.
- */
-public interface ServiceParent extends Service {
-
- /**
- * Add a child service. It must be in a consistent state with the
- * service to which it is being added.
- * @param service the service to add.
- */
- void addService(Service service);
-
- /**
- * Get an unmodifiable list of services
- * @return a list of child services at the time of invocation -
- * added services will not be picked up.
- */
- List<Service> getServices();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingCallable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingCallable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingCallable.java
deleted file mode 100644
index 5ebf77c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingCallable.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.service.Service;
-
-import java.util.concurrent.Callable;
-
-/**
- * A runnable which terminates its owner; it also catches any
- * exception raised and can serve it back.
- *
- */
-public class ServiceTerminatingCallable<V> implements Callable<V> {
-
- private final Service owner;
- private Exception exception;
- /**
- * This is the callback
- */
- private final Callable<V> callable;
-
-
- /**
- * Create an instance. If the owner is null, the owning service
- * is not terminated.
- * @param owner owning service -can be null
- * @param callable callback.
- */
- public ServiceTerminatingCallable(Service owner,
- Callable<V> callable) {
- Preconditions.checkArgument(callable != null, "null callable");
- this.owner = owner;
- this.callable = callable;
- }
-
-
- /**
- * Get the owning service
- * @return the service to receive notification when
- * the runnable completes.
- */
- public Service getOwner() {
- return owner;
- }
-
- /**
- * Any exception raised by inner <code>action's</code> run.
- * @return an exception or null.
- */
- public Exception getException() {
- return exception;
- }
-
- /**
- * Delegates the call to the callable supplied in the constructor,
- * then calls the stop() operation on its owner. Any exception
- * is caught, noted and rethrown
- * @return the outcome of the delegated call operation
- * @throws Exception if one was raised.
- */
- @Override
- public V call() throws Exception {
- try {
- return callable.call();
- } catch (Exception e) {
- exception = e;
- throw e;
- } finally {
- if (owner != null) {
- owner.stop();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingRunnable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingRunnable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingRunnable.java
deleted file mode 100644
index dc591df..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingRunnable.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.service.Service;
-
-/**
- * A runnable which terminates its after running; it also catches any
- * exception raised and can serve it back.
- */
-public class ServiceTerminatingRunnable implements Runnable {
-
- private final Service owner;
- private final Runnable action;
- private Exception exception;
-
- /**
- * Create an instance.
- * @param owner owning service
- * @param action action to execute before terminating the service
- */
- public ServiceTerminatingRunnable(Service owner, Runnable action) {
- Preconditions.checkArgument(owner != null, "null owner");
- Preconditions.checkArgument(action != null, "null action");
- this.owner = owner;
- this.action = action;
- }
-
- /**
- * Get the owning service.
- * @return the service to receive notification when
- * the runnable completes.
- */
- public Service getOwner() {
- return owner;
- }
-
- /**
- * Any exception raised by inner <code>action's</code> run.
- * @return an exception or null.
- */
- public Exception getException() {
- return exception;
- }
-
- @Override
- public void run() {
- try {
- action.run();
- } catch (Exception e) {
- exception = e;
- }
- owner.stop();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceThreadFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceThreadFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceThreadFactory.java
deleted file mode 100644
index 737197b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceThreadFactory.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import com.google.common.base.Preconditions;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * A thread factory that creates threads (possibly daemon threads)
- * using the name and naming policy supplied.
- * The thread counter starts at 1, increments atomically,
- * and is supplied as the second argument in the format string.
- *
- * A static method, {@link #singleThreadExecutor(String, boolean)},
- * exists to simplify the construction of an executor with a single well-named
- * threads.
- *
- * Example
- * <pre>
- * ExecutorService exec = ServiceThreadFactory.newSingleThreadExecutor("live", true)
- * </pre>
- */
-public class ServiceThreadFactory implements ThreadFactory {
-
- private static final AtomicInteger counter = new AtomicInteger(1);
-
- /**
- * Default format for thread names: {@value}.
- */
- public static final String DEFAULT_NAMING_FORMAT = "%s-%03d";
- private final String name;
- private final boolean daemons;
- private final String namingFormat;
-
- /**
- * Create an instance
- * @param name base thread name
- * @param daemons flag to indicate the threads should be marked as daemons
- * @param namingFormat format string to generate thread names from
- */
- public ServiceThreadFactory(String name,
- boolean daemons,
- String namingFormat) {
- Preconditions.checkArgument(name != null, "null name");
- Preconditions.checkArgument(namingFormat != null, "null naming format");
- this.name = name;
- this.daemons = daemons;
- this.namingFormat = namingFormat;
- }
-
- /**
- * Create an instance with the default naming format.
- * @param name base thread name
- * @param daemons flag to indicate the threads should be marked as daemons
- */
- public ServiceThreadFactory(String name,
- boolean daemons) {
- this(name, daemons, DEFAULT_NAMING_FORMAT);
- }
-
- @Override
- public Thread newThread(Runnable r) {
- Preconditions.checkArgument(r != null, "null runnable");
- String threadName =
- String.format(namingFormat, name, counter.getAndIncrement());
- Thread thread = new Thread(r, threadName);
- thread.setDaemon(daemons);
- return thread;
- }
-
- /**
- * Create a single thread executor using this naming policy.
- * @param name base thread name
- * @param daemons flag to indicate the threads should be marked as daemons
- * @return an executor
- */
- public static ExecutorService singleThreadExecutor(String name,
- boolean daemons) {
- return Executors.newSingleThreadExecutor(
- new ServiceThreadFactory(name, daemons));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
deleted file mode 100644
index 65d14b7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-
-/**
- * A service that calls the supplied callback when it is started -after the
- * given delay.
- *
- * It can be configured to stop itself after the callback has
- * completed, marking any exception raised as the exception of this service.
- * The notifications come in on a callback thread -a thread that is only
- * started in this service's <code>start()</code> operation.
- */
-public class WorkflowCallbackService<V> extends
- WorkflowScheduledExecutorService<ScheduledExecutorService> {
- protected static final Logger LOG =
- LoggerFactory.getLogger(WorkflowCallbackService.class);
-
- /**
- * This is the callback.
- */
- private final Callable<V> callback;
- private final int delay;
- private final ServiceTerminatingCallable<V> command;
-
- private ScheduledFuture<V> scheduledFuture;
-
- /**
- * Create an instance of the service
- * @param name service name
- * @param callback callback to invoke
- * @param delay delay -or 0 for no delay
- * @param terminate terminate this service after the callback?
- */
- public WorkflowCallbackService(String name,
- Callable<V> callback,
- int delay,
- boolean terminate) {
- super(name);
- Preconditions.checkNotNull(callback, "Null callback argument");
- this.callback = callback;
- this.delay = delay;
- command = new ServiceTerminatingCallable<V>(
- terminate ? this : null,
- callback);
- }
-
- public ScheduledFuture<V> getScheduledFuture() {
- return scheduledFuture;
- }
-
- @Override
- protected void serviceStart() throws Exception {
- LOG.debug("Notifying {} after a delay of {} millis", callback, delay);
- ScheduledExecutorService executorService =
- Executors.newSingleThreadScheduledExecutor(
- new ServiceThreadFactory(getName(), true));
- setExecutor(executorService);
- scheduledFuture =
- executorService.schedule(command, delay, TimeUnit.MILLISECONDS);
- }
-
- /**
- * Stop the service.
- * If there is any exception noted from any executed notification,
- * note the exception in this class
- * @throws Exception exception.
- */
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
- // propagate any failure
- if (getCallbackException() != null) {
- throw getCallbackException();
- }
- }
-
- /**
- * Get the exception raised by a callback. Will always be null if the
- * callback has not been executed; will only be non-null after any success.
- * @return a callback
- */
- public Exception getCallbackException() {
- return command.getException();
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index 647bfe9..a044838 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -24,30 +24,24 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.RoleKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.api.resource.Configuration;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.AbstractLauncher;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.utils.PublishedConfiguration;
+import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,23 +49,18 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.regex.Pattern;
-import static org.apache.slider.api.ServiceApiConstants.*;
-import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
+import static org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants.*;
/**
* This is a factoring out of methods handy for providers. It's bonded to a log
* at construction time.
*/
-public class ProviderUtils implements RoleKeys, SliderKeys {
+public class ProviderUtils implements YarnServiceConstants {
protected static final Logger log =
LoggerFactory.getLogger(ProviderUtils.class);
@@ -174,46 +163,22 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
}
/**
- * Get resource requirements from a String value. If value isn't specified,
- * use the default value. If value is greater than max, use the max value.
- * @param val string value
- * @param defVal default value
- * @param maxVal maximum value
- * @return int resource requirement
- */
- public int getRoleResourceRequirement(String val,
- int defVal,
- int maxVal) {
- if (val==null) {
- val = Integer.toString(defVal);
- }
- Integer intVal;
- if (ResourceKeys.YARN_RESOURCE_MAX.equals(val)) {
- intVal = maxVal;
- } else {
- intVal = Integer.decode(val);
- }
- return intVal;
- }
-
-
- /**
* Localize the service keytabs for the application.
* @param launcher container launcher
* @param fileSystem file system
* @throws IOException trouble uploading to HDFS
*/
- public void localizeServiceKeytabs(ContainerLauncher launcher,
+ public void localizeServiceKeytabs(AbstractLauncher launcher,
SliderFileSystem fileSystem, Application application) throws IOException {
Configuration conf = application.getConfiguration();
String keytabPathOnHost =
- conf.getProperty(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
+ conf.getProperty(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
if (SliderUtils.isUnset(keytabPathOnHost)) {
String amKeytabName =
- conf.getProperty(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+ conf.getProperty(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
String keytabDir =
- conf.getProperty(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
+ conf.getProperty(YarnServiceConf.KEY_HDFS_KEYTAB_DIR);
// we need to localize the keytab files in the directory
Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
application.getName());
@@ -434,38 +399,4 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
String.valueOf(instance.getCompInstanceId().getId()));
return tokens;
}
-
- /**
- * Add ROLE_HOST tokens for substitution into config values.
- * @param tokens existing tokens
- * @param amState access to AM state
- */
- public static void addComponentHostTokens(Map<String, String> tokens,
- StateAccessForProviders amState) {
- if (amState == null) {
- return;
- }
- for (Map.Entry<String, Map<String, ClusterNode>> entry :
- amState.getRoleClusterNodeMapping().entrySet()) {
- String tokenName = entry.getKey().toUpperCase(Locale.ENGLISH) + "_HOST";
- String hosts = StringUtils .join(",",
- getHostsList(entry.getValue().values(), true));
- tokens.put($(tokenName), hosts);
- }
- }
-
- /**
- * Return a list of hosts based on current ClusterNodes.
- * @param values cluster nodes
- * @param hostOnly whether host or host/server name will be added to list
- * @return list of hosts
- */
- public static Iterable<String> getHostsList(Collection<ClusterNode> values,
- boolean hostOnly) {
- List<String> hosts = new ArrayList<>();
- for (ClusterNode cn : values) {
- hosts.add(hostOnly ? cn.host : cn.host + "/" + cn.name);
- }
- return hosts;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
index 32cedb6..0920a9c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
@@ -19,8 +19,8 @@ package org.apache.hadoop.yarn.service.provider.defaultImpl;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.ConfigFile;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import java.io.IOException;
import java.nio.file.Paths;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
index a77214c..33f8278 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
@@ -19,9 +19,9 @@ package org.apache.hadoop.yarn.service.provider.defaultImpl;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
index c1f225c..d4a2254 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
@@ -19,16 +19,16 @@ package org.apache.hadoop.yarn.service.provider.docker;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.slider.util.RestApiErrorMessages;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
import java.io.IOException;
public class DockerClientProvider extends AbstractClientProvider
- implements SliderKeys {
+ implements YarnServiceConstants {
public DockerClientProvider() {
super();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
index c20eaad..236ddd9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
@@ -21,9 +21,9 @@ import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
import java.io.IOException;
import java.text.MessageFormat;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
index 2b54be9..01f7b20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
@@ -20,17 +20,17 @@ package org.apache.hadoop.yarn.service.provider.tarball;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.slider.util.RestApiErrorMessages;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
import java.io.IOException;
import java.nio.file.Paths;
public class TarballClientProvider extends AbstractClientProvider
- implements SliderKeys {
+ implements YarnServiceConstants {
public TarballClientProvider() {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
index 3c3d425..2403255 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
@@ -22,9 +22,9 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
new file mode 100644
index 0000000..56634f6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.registry;
+
+/**
+ * These are constants unique to the Slider AM
+ */
+public class CustomRegistryConstants {
+
+ public static final String MANAGEMENT_REST_API =
+ "classpath:org.apache.slider.management";
+
+ public static final String REGISTRY_REST_API =
+ "classpath:org.apache.slider.registry";
+
+ public static final String PUBLISHER_REST_API =
+ "classpath:org.apache.slider.publisher";
+
+ public static final String PUBLISHER_CONFIGURATIONS_API =
+ "classpath:org.apache.slider.publisher.configurations";
+
+ public static final String PUBLISHER_EXPORTS_API =
+ "classpath:org.apache.slider.publisher.exports";
+
+ public static final String PUBLISHER_DOCUMENTS_API =
+ "classpath:org.apache.slider.publisher.documents";
+
+ public static final String AGENT_SECURE_REST_API =
+ "classpath:org.apache.slider.agents.secure";
+
+ public static final String AGENT_ONEWAY_REST_API =
+ "classpath:org.apache.slider.agents.oneway";
+
+ public static final String AM_IPC_PROTOCOL =
+ "classpath:org.apache.slider.appmaster.ipc";
+
+ public static final String AM_REST_BASE =
+ "classpath:org.apache.slider.client.rest";
+
+ public static final String WEB_UI = "http://";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
new file mode 100644
index 0000000..ef5ed91
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.registry;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.registry.client.api.BindFlags;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceId;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join;
+
+/**
+ * Registry view for providers. This tracks where the service
+ * is registered, offers access to the record and other things.
+ */
+public class YarnRegistryViewForProviders {
+ private static final Log LOG =
+ LogFactory.getLog(YarnRegistryViewForProviders.class);
+
+ private final RegistryOperations registryOperations;
+ private final String user;
+ private final String sliderServiceClass;
+ private final String instanceName;
+ /**
+ * Record used where the service registered itself.
+ * Null until the service is registered
+ */
+ private ServiceRecord selfRegistration;
+
+ /**
+ * Path where record was registered
+ * Null until the service is registered
+ */
+ private String selfRegistrationPath;
+
+ public YarnRegistryViewForProviders(RegistryOperations registryOperations,
+ String user,
+ String sliderServiceClass,
+ String instanceName,
+ ApplicationAttemptId applicationAttemptId) {
+ Preconditions.checkArgument(registryOperations != null,
+ "null registry operations");
+ Preconditions.checkArgument(user != null, "null user");
+ Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass),
+ "unset service class");
+ Preconditions.checkArgument(SliderUtils.isSet(instanceName),
+ "instanceName");
+ Preconditions.checkArgument(applicationAttemptId != null,
+ "null applicationAttemptId");
+ this.registryOperations = registryOperations;
+ this.user = user;
+ this.sliderServiceClass = sliderServiceClass;
+ this.instanceName = instanceName;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+
+ private void setSelfRegistration(ServiceRecord selfRegistration) {
+ this.selfRegistration = selfRegistration;
+ }
+
+ /**
+ * Get the path to where the service has registered itself.
+ * Null until the service is registered
+ * @return the service registration path.
+ */
+ public String getSelfRegistrationPath() {
+ return selfRegistrationPath;
+ }
+
+ /**
+ * Get the absolute path to where the service has registered itself.
+ * This includes the base registry path
+ * Null until the service is registered
+ * @return the service registration path.
+ */
+ public String getAbsoluteSelfRegistrationPath() {
+ if (selfRegistrationPath == null) {
+ return null;
+ }
+ String root = registryOperations.getConfig().getTrimmed(
+ RegistryConstants.KEY_REGISTRY_ZK_ROOT,
+ RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
+ return RegistryPathUtils.join(root, selfRegistrationPath);
+ }
+
+ /**
+ * Add a component under the slider name/entry
+ * @param componentName component name
+ * @param record record to put
+ * @throws IOException
+ */
+ public void putComponent(String componentName,
+ ServiceRecord record) throws
+ IOException {
+ putComponent(sliderServiceClass, instanceName,
+ componentName,
+ record);
+ }
+
+ /**
+ * Add a component
+ * @param serviceClass service class to use under ~user
+ * @param componentName component name
+ * @param record record to put
+ * @throws IOException
+ */
+ public void putComponent(String serviceClass,
+ String serviceName,
+ String componentName,
+ ServiceRecord record) throws IOException {
+ String path = RegistryUtils.componentPath(
+ user, serviceClass, serviceName, componentName);
+ registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
+ registryOperations.bind(path, record, BindFlags.OVERWRITE);
+ }
+
+ /**
+ * Add a service under a path, optionally purging any history
+ * @param username user
+ * @param serviceClass service class to use under ~user
+ * @param serviceName name of the service
+ * @param record service record
+ * @param deleteTreeFirst perform recursive delete of the path first.
+ * @return the path the service was created at
+ * @throws IOException
+ */
+ public String putService(String username,
+ String serviceClass,
+ String serviceName,
+ ServiceRecord record,
+ boolean deleteTreeFirst) throws IOException {
+ String path = RegistryUtils.servicePath(
+ username, serviceClass, serviceName);
+ if (deleteTreeFirst) {
+ registryOperations.delete(path, true);
+ }
+ registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
+ registryOperations.bind(path, record, BindFlags.OVERWRITE);
+ return path;
+ }
+
+ /**
+ * Add a service under a path for the current user
+ * @param record service record
+ * @param deleteTreeFirst perform recursive delete of the path first
+ * @return the path the service was created at
+ * @throws IOException
+ */
+ public String registerSelf(
+ ServiceRecord record,
+ boolean deleteTreeFirst) throws IOException {
+ selfRegistrationPath =
+ putService(user, sliderServiceClass, instanceName, record, deleteTreeFirst);
+ setSelfRegistration(record);
+ return selfRegistrationPath;
+ }
+
+ /**
+ * Delete a component
+ * @param containerId component name
+ * @throws IOException
+ */
+ public void deleteComponent(ComponentInstanceId instanceId,
+ String containerId) throws IOException {
+ String path = RegistryUtils.componentPath(
+ user, sliderServiceClass, instanceName,
+ containerId);
+ LOG.info(instanceId + ": Deleting registry path " + path);
+ registryOperations.delete(path, false);
+ }
+
+ /**
+ * Delete the children of a path -but not the path itself.
+ * It is not an error if the path does not exist
+ * @param path path to delete
+ * @param recursive flag to request recursive deletes
+ * @throws IOException IO problems
+ */
+ public void deleteChildren(String path, boolean recursive) throws IOException {
+ List<String> childNames = null;
+ try {
+ childNames = registryOperations.list(path);
+ } catch (PathNotFoundException e) {
+ return;
+ }
+ for (String childName : childNames) {
+ String child = join(path, childName);
+ registryOperations.delete(child, recursive);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/BaseRestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/BaseRestClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/BaseRestClient.java
new file mode 100644
index 0000000..2d01bef
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/BaseRestClient.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.rest;
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientHandlerException;
+import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+import java.net.URI;
+
+
+/**
+ * This is a base class for Jersey REST clients in Slider.
+ * It supports the execution of operations —with
+ * exceptions uprated to IOExceptions when needed.
+ * <p>
+ * Subclasses can use these operations to provide an API-like view
+ * of the REST model
+ */
+public class BaseRestClient {
+ private static final Logger log =
+ LoggerFactory.getLogger(BaseRestClient.class);
+ private final Client client;
+
+ public BaseRestClient(
+ Client client) {
+ Preconditions.checkNotNull(client, "null jersey client");
+ this.client = client;
+ }
+
+ /**
+ * Get the jersey client
+ * @return jersey client
+ */
+ public Client getClient() {
+ return client;
+ }
+
+ /**
+ * Execute the operation. Failures are raised as IOException subclasses
+ * @param method method to execute
+ * @param resource resource to work against
+ * @param c class to build
+ * @param <T> type expected
+ * @return an instance of the type T
+ * @throws IOException on any failure
+ */
+ public <T> T exec(HttpVerb method, WebResource resource, Class<T> c)
+ throws IOException {
+ try {
+ Preconditions.checkArgument(c != null);
+ log.debug("{}} {}", method, resource.getURI());
+ return resource.accept(MediaType.APPLICATION_JSON_TYPE)
+ .method(method.getVerb(), c);
+ } catch (ClientHandlerException ex) {
+ throw ExceptionConverter.convertJerseyException(method.getVerb(),
+ resource.getURI().toString(),
+ ex);
+ } catch (UniformInterfaceException ex) {
+ throw UgiJerseyBinding.uprateFaults(method,
+ resource.getURI().toString(),
+ ex);
+ }
+ }
+
+ /**
+ * Execute the operation. Failures are raised as IOException subclasses
+ * @param method method to execute
+ * @param resource resource to work against
+ * @param t type to work with
+ * @param <T> type expected
+ * @return an instance of the type T
+ * @throws IOException on any failure
+ */
+ public <T> T exec(HttpVerb method, WebResource resource, GenericType<T> t)
+ throws IOException {
+ try {
+ Preconditions.checkArgument(t != null);
+ log.debug("{}} {}", method, resource.getURI());
+ resource.accept(MediaType.APPLICATION_JSON_TYPE);
+ return resource.method(method.getVerb(), t);
+ } catch (ClientHandlerException ex) {
+ throw ExceptionConverter.convertJerseyException(method.getVerb(),
+ resource.getURI().toString(),
+ ex);
+ } catch (UniformInterfaceException ex) {
+ throw UgiJerseyBinding.uprateFaults(method, resource.getURI().toString(),
+ ex);
+ }
+ }
+
+
+ /**
+ * Execute the GET operation. Failures are raised as IOException subclasses
+ * @param resource resource to work against
+ * @param c class to build
+ * @param <T> type expected
+ * @return an instance of the type T
+ * @throws IOException on any failure
+ */
+ public <T> T get(WebResource resource, Class<T> c) throws IOException {
+ return exec(HttpVerb.GET, resource, c);
+ }
+
+ /**
+ * Create a Web resource from the client.
+ *
+ * @param u the URI of the resource.
+ * @return the Web resource.
+ */
+ public WebResource resource(URI u) {
+ return client.resource(u);
+ }
+
+ /**
+ * Create a Web resource from the client.
+ *
+ * @param url the URI of the resource.
+ * @return the Web resource.
+ */
+
+ public WebResource resource(String url) {
+ return client.resource(url);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/ExceptionConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/ExceptionConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/ExceptionConverter.java
new file mode 100644
index 0000000..12fdc79
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/ExceptionConverter.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.rest;
+
+import com.sun.jersey.api.client.ClientHandlerException;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import org.apache.hadoop.fs.InvalidRequestException;
+import org.apache.hadoop.fs.PathAccessDeniedException;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.yarn.webapp.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletResponse;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * static methods to convert exceptions into different types, including
+ * extraction of details and finer-grained conversions.
+ */
+public class ExceptionConverter {
+ private static final Logger
+ log = LoggerFactory.getLogger(ExceptionConverter.class);
+
+ /**
+ * Uprate error codes 400 and up into faults;
+ * 404 is converted to a {@link FileNotFoundException},
+ * 401 to {@link ForbiddenException}
+ * FileNotFoundException for an unknown resource
+ * PathAccessDeniedException for access denied
+ * PathIOException for anything else
+ * @param verb HTTP Verb used
+ * @param targetURL URL being targeted
+ * @param exception original exception
+ * @return a new exception, the original one nested as a cause
+ */
+ public static IOException convertJerseyException(String verb,
+ String targetURL,
+ UniformInterfaceException exception) {
+
+ IOException ioe = null;
+ ClientResponse response = exception.getResponse();
+ if (response != null) {
+ int status = response.getStatus();
+ String body = "";
+ try {
+ if (response.hasEntity()) {
+ body = response.getEntity(String.class);
+ log.error("{} {} returned status {} and body\n{}",
+ verb, targetURL, status, body);
+ } else {
+ log.error("{} {} returned status {} and empty body",
+ verb, targetURL, status);
+ }
+ } catch (Exception e) {
+ log.warn("Failed to extract body from client response", e);
+ }
+
+ if (status == HttpServletResponse.SC_UNAUTHORIZED
+ || status == HttpServletResponse.SC_FORBIDDEN) {
+ ioe = new PathAccessDeniedException(targetURL);
+ } else if (status == HttpServletResponse.SC_BAD_REQUEST
+ || status == HttpServletResponse.SC_NOT_ACCEPTABLE
+ || status == HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE) {
+ // bad request
+ ioe = new InvalidRequestException(
+ String.format("Bad %s request: status code %d against %s",
+ verb, status, targetURL));
+ } else if (status > 400 && status < 500) {
+ ioe = new FileNotFoundException(targetURL);
+ }
+ if (ioe == null) {
+ ioe = new PathIOException(targetURL,
+ verb + " " + targetURL
+ + " failed with status code : " + status
+ + ":" + exception);
+ }
+ } else {
+ ioe = new PathIOException(targetURL,
+ verb + " " + targetURL + " failed: " + exception);
+ }
+ ioe.initCause(exception);
+ return ioe;
+ }
+
+ /**
+ * Handle a client-side Jersey exception.
+ * <p>
+ * If there's an inner IOException, return that.
+ * <p>
+ * Otherwise: create a new wrapper IOE including verb and target details
+ * @param verb HTTP Verb used
+ * @param targetURL URL being targeted
+ * @param exception original exception
+ * @return an exception to throw
+ */
+ public static IOException convertJerseyException(String verb,
+ String targetURL,
+ ClientHandlerException exception) {
+ if (exception.getCause() instanceof IOException) {
+ return (IOException)exception.getCause();
+ } else {
+ IOException ioe = new IOException(
+ verb + " " + targetURL + " failed: " + exception);
+ ioe.initCause(exception);
+ return ioe;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/HttpVerb.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/HttpVerb.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/HttpVerb.java
new file mode 100644
index 0000000..93f9082
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/HttpVerb.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.rest;
+
+/**
+ * Http verbs with details on what they support in terms of submit and
+ * response bodies.
+ * <p>
+ * Those verbs which do support bodies in the response MAY NOT return it;
+ * if the response code is 204 then the answer is "no body", but the operation
+ * is considered a success.
+ */
+public enum HttpVerb {
+ GET("GET", false, true),
+ POST("POST", true, true),
+ PUT("PUT", true, true),
+ DELETE("DELETE", false, true),
+ HEAD("HEAD", false, false);
+
+ private final String verb;
+ private final boolean hasUploadBody;
+ private final boolean hasResponseBody;
+
+ HttpVerb(String verb, boolean hasUploadBody, boolean hasResponseBody) {
+ this.verb = verb;
+ this.hasUploadBody = hasUploadBody;
+ this.hasResponseBody = hasResponseBody;
+ }
+
+ public String getVerb() {
+ return verb;
+ }
+
+ public boolean hasUploadBody() {
+ return hasUploadBody;
+ }
+
+ public boolean hasResponseBody() {
+ return hasResponseBody;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/SliderURLConnectionFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/SliderURLConnectionFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/SliderURLConnectionFactory.java
new file mode 100644
index 0000000..fcd7f55
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/SliderURLConnectionFactory.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.rest;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.web.KerberosUgiAuthenticator;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+import java.security.GeneralSecurityException;
+
+/**
+ * Factory for URL connections; used behind the scenes in the Jersey integration.
+ * <p>
+ * Derived from the WebHDFS implementation.
+ */
+public class SliderURLConnectionFactory {
+ private static final Logger log =
+ LoggerFactory.getLogger(SliderURLConnectionFactory.class);
+
+ /**
+ * Timeout for socket connects and reads
+ */
+ public final static int DEFAULT_SOCKET_TIMEOUT = 60 * 1000; // 1 minute
+ private final ConnectionConfigurator connConfigurator;
+
+ private static final ConnectionConfigurator DEFAULT_CONFIGURATOR = new BasicConfigurator();
+
+ /**
+ * Construct a new URLConnectionFactory based on the configuration. It will
+ * try to load SSL certificates when it is specified.
+ */
+ public static SliderURLConnectionFactory newInstance(Configuration conf) {
+ ConnectionConfigurator conn;
+ try {
+ conn = newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
+ } catch (Exception e) {
+ log.debug("Cannot load customized SSL configuration.", e);
+ conn = DEFAULT_CONFIGURATOR;
+ }
+ return new SliderURLConnectionFactory(conn);
+ }
+
+ private SliderURLConnectionFactory(ConnectionConfigurator connConfigurator) {
+ this.connConfigurator = connConfigurator;
+ }
+
+ /**
+ * Create a new ConnectionConfigurator for SSL connections
+ */
+ private static ConnectionConfigurator newSslConnConfigurator(final int timeout,
+ Configuration conf) throws IOException, GeneralSecurityException {
+ final SSLFactory factory;
+ final SSLSocketFactory sf;
+ final HostnameVerifier hv;
+
+ factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+ factory.init();
+ sf = factory.createSSLSocketFactory();
+ hv = factory.getHostnameVerifier();
+
+ return new ConnectionConfigurator() {
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ if (conn instanceof HttpsURLConnection) {
+ HttpsURLConnection c = (HttpsURLConnection) conn;
+ c.setSSLSocketFactory(sf);
+ c.setHostnameVerifier(hv);
+ }
+ SliderURLConnectionFactory.setupConnection(conn, timeout);
+ return conn;
+ }
+ };
+ }
+
+ /**
+ * Opens a url with read and connect timeouts
+ *
+ * @param url
+ * to open
+ * @return URLConnection
+ * @throws IOException
+ */
+ public URLConnection openConnection(URL url) throws IOException {
+ try {
+ return openConnection(url, false);
+ } catch (AuthenticationException e) {
+ // Unreachable
+ return null;
+ }
+ }
+
+ /**
+ * Opens a url with read and connect timeouts
+ *
+ * @param url
+ * URL to open
+ * @param isSpnego
+ * whether the url should be authenticated via SPNEGO
+ * @return URLConnection
+ * @throws IOException
+ * @throws AuthenticationException
+ */
+ public URLConnection openConnection(URL url, boolean isSpnego)
+ throws IOException, AuthenticationException {
+ if (isSpnego) {
+ log.debug("open AuthenticatedURL connection {}", url);
+ UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
+ final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
+ return new AuthenticatedURL(new KerberosUgiAuthenticator(),
+ connConfigurator).openConnection(url, authToken);
+ } else {
+ log.debug("open URL connection {}", url);
+ URLConnection connection = url.openConnection();
+ if (connection instanceof HttpURLConnection) {
+ connConfigurator.configure((HttpURLConnection) connection);
+ }
+ return connection;
+ }
+ }
+
+ /**
+ * Sets connection parameters on the given URLConnection
+ *
+ * @param connection
+ * URLConnection to set
+ * @param socketTimeout
+ * the connection and read timeout of the connection.
+ */
+ private static void setupConnection(URLConnection connection, int socketTimeout) {
+ connection.setConnectTimeout(socketTimeout);
+ connection.setReadTimeout(socketTimeout);
+ connection.setUseCaches(false);
+ if (connection instanceof HttpURLConnection) {
+ ((HttpURLConnection) connection).setInstanceFollowRedirects(true);
+ }
+ }
+
+ private static class BasicConfigurator implements ConnectionConfigurator {
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ SliderURLConnectionFactory.setupConnection(conn, DEFAULT_SOCKET_TIMEOUT);
+ return conn;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
new file mode 100644
index 0000000..b3fdef9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.rest;
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+/**
+ * Class to bond to a Jersey client, for UGI integration and SPNEGO.
+ * <p>
+ * Usage: create an instance, then when creating a Jersey <code>Client</code>
+ * pass in to the constructor the handler provided by {@link #getHandler()}
+ *
+ * see <a href="https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html">Jersey docs</a>
+ */
+public class UgiJerseyBinding implements
+ HttpURLConnectionFactory {
+ private static final Logger log =
+ LoggerFactory.getLogger(UgiJerseyBinding.class);
+
+ private final UrlConnectionOperations operations;
+ private final URLConnectionClientHandler handler;
+
+ /**
+ * Construct an instance
+ * @param operations operations instance
+ */
+ @SuppressWarnings("ThisEscapedInObjectConstruction")
+ public UgiJerseyBinding(UrlConnectionOperations operations) {
+ Preconditions.checkArgument(operations != null, "Null operations");
+ this.operations = operations;
+ handler = new URLConnectionClientHandler(this);
+ }
+
+ /**
+ * Create an instance off the configuration. The SPNEGO policy
+ * is derived from the current UGI settings.
+ * @param conf config
+ */
+ public UgiJerseyBinding(Configuration conf) {
+ this(new UrlConnectionOperations(conf));
+ }
+
+ /**
+ * Get a URL connection.
+ * @param url URL to connect to
+ * @return the connection
+ * @throws IOException any problem. {@link AuthenticationException}
+ * errors are wrapped
+ */
+ @Override
+ public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
+ try {
+ // open a connection handling status codes and so redirections
+ // but as it opens a connection, it's less useful than you think.
+
+ return operations.openConnection(url);
+ } catch (AuthenticationException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public UrlConnectionOperations getOperations() {
+ return operations;
+ }
+
+ public URLConnectionClientHandler getHandler() {
+ return handler;
+ }
+
+ /**
+ * Get the SPNEGO flag (as found in the operations instance
+ * @return the spnego policy
+ */
+ public boolean isUseSpnego() {
+ return operations.isUseSpnego();
+ }
+
+
+ /**
+ * Uprate error codes 400 and up into faults;
+ * <p>
+ * see {@link ExceptionConverter#convertJerseyException(String, String, UniformInterfaceException)}
+ */
+ public static IOException uprateFaults(HttpVerb verb, String url,
+ UniformInterfaceException ex)
+ throws IOException {
+ return ExceptionConverter.convertJerseyException(verb.getVerb(),
+ url, ex);
+ }
+
+ /**
+ * Create the standard Jersey client Config
+ * @return the recommended Jersey Client config
+ */
+ public ClientConfig createJerseyClientConfig() {
+ ClientConfig clientConfig = new DefaultClientConfig();
+ clientConfig.getFeatures().put(JSONConfiguration.FEATURE_POJO_MAPPING, true);
+ return clientConfig;
+ }
+
+ /**
+ * Create a jersey client bonded to this handler, using the
+ * supplied client config
+ * @param clientConfig client configuratin
+ * @return a new client instance to use
+ */
+ public Client createJerseyClient(ClientConfig clientConfig) {
+ return new Client(getHandler(), clientConfig);
+ }
+
+ /**
+ * Create a jersey client bonded to this handler, using the
+ * client config created with {@link #createJerseyClientConfig()}
+ * @return a new client instance to use
+ */
+ public Client createJerseyClient() {
+ return createJerseyClient(createJerseyClientConfig());
+ }
+
+}
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UrlConnectionOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UrlConnectionOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UrlConnectionOperations.java
new file mode 100644
index 0000000..d7f768e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UrlConnectionOperations.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.rest;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+/**
+ * Operations on the JDK UrlConnection class.
+ *
+ */
+public class UrlConnectionOperations extends Configured {
+ private static final Logger log =
+ LoggerFactory.getLogger(UrlConnectionOperations.class);
+
+ private SliderURLConnectionFactory connectionFactory;
+
+ private boolean useSpnego = false;
+
+ /**
+ * Create an instance off the configuration. The SPNEGO policy
+ * is derived from the current UGI settings.
+ * @param conf config
+ */
+ public UrlConnectionOperations(Configuration conf) {
+ super(conf);
+ connectionFactory = SliderURLConnectionFactory.newInstance(conf);
+ if (UserGroupInformation.isSecurityEnabled()) {
+ log.debug("SPNEGO is enabled");
+ setUseSpnego(true);
+ }
+ }
+
+
+ public boolean isUseSpnego() {
+ return useSpnego;
+ }
+
+ public void setUseSpnego(boolean useSpnego) {
+ this.useSpnego = useSpnego;
+ }
+
+ /**
+ * Opens a url with cache disabled, redirect handled in
+ * (JDK) implementation.
+ *
+ * @param url to open
+ * @return URLConnection
+ * @throws IOException
+ * @throws AuthenticationException authentication failure
+ */
+ public HttpURLConnection openConnection(URL url) throws
+ IOException,
+ AuthenticationException {
+ Preconditions.checkArgument(url.getPort() != 0, "no port");
+ return (HttpURLConnection) connectionFactory.openConnection(url, useSpnego);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/ServiceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/ServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/ServiceMonitor.java
new file mode 100644
index 0000000..98a76ea
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/ServiceMonitor.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.servicemonitor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.component.ComponentEvent;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
+import org.apache.hadoop.yarn.service.component.ComponentState;
+import org.apache.hadoop.yarn.service.servicemonitor.probe.ProbeStatus;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.RUNNING_BUT_UNREADY;
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX;
+import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.BECOME_NOT_READY;
+import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.BECOME_READY;
+import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.READY;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_WINDOW;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.DEFAULT_READINESS_CHECK_INTERVAL;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.READINESS_CHECK_INTERVAL;
+
+public class ServiceMonitor extends AbstractService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceMonitor.class);
+
+ public ScheduledExecutorService executorService;
+ private Map<ContainerId, ComponentInstance> liveInstances = null;
+ private ServiceContext context;
+ private Configuration conf;
+
+ public ServiceMonitor(String name, ServiceContext context) {
+ super(name);
+ liveInstances = context.scheduler.getLiveInstances();
+ this.context = context;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ executorService = Executors.newScheduledThreadPool(1);
+ this.conf = conf;
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ long readinessCheckInterval = YarnServiceConf
+ .getLong(READINESS_CHECK_INTERVAL, DEFAULT_READINESS_CHECK_INTERVAL,
+ context.application.getConfiguration(), conf);
+
+ executorService
+ .scheduleAtFixedRate(new ReadinessChecker(), readinessCheckInterval,
+ readinessCheckInterval, TimeUnit.SECONDS);
+
+ // Default 6 hours.
+ long failureResetInterval = YarnServiceConf
+ .getLong(CONTAINER_FAILURE_WINDOW, 21600,
+ context.application.getConfiguration(), conf);
+
+ executorService
+ .scheduleAtFixedRate(new ContainerFailureReset(), failureResetInterval,
+ failureResetInterval, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void serviceStop() throws Exception {
+ if (executorService != null) {
+ executorService.shutdownNow();
+ }
+ }
+
+ private class ReadinessChecker implements Runnable {
+
+ @Override
+ public void run() {
+
+ // check if the comp instance are ready
+ for (Map.Entry<ContainerId, ComponentInstance> entry : liveInstances
+ .entrySet()) {
+ ComponentInstance instance = entry.getValue();
+
+ ProbeStatus status = instance.ping();
+ if (status.isSuccess()) {
+ if (instance.getState() == RUNNING_BUT_UNREADY) {
+ // synchronously update the state.
+ instance.handle(
+ new ComponentInstanceEvent(entry.getKey(), BECOME_READY));
+ }
+ } else {
+ if (instance.getState() == READY) {
+ instance.handle(
+ new ComponentInstanceEvent(entry.getKey(), BECOME_NOT_READY));
+ }
+ }
+ }
+
+ for (Component component : context.scheduler.getAllComponents()
+ .values()) {
+ // If comp hasn't started yet and its dependencies are satisfied
+ if (component.getState() == ComponentState.INIT && component
+ .areDependenciesReady()) {
+ LOG.info("[COMPONENT {}]: Dependencies satisfied, ramping up.",
+ component.getName());
+ ComponentEvent event = new ComponentEvent(component.getName(), FLEX)
+ .setDesired(component.getComponentSpec().getNumberOfContainers());
+ component.handle(event);
+ }
+ }
+ }
+ }
+
+ private class ContainerFailureReset implements Runnable {
+ @Override
+ public void run() {
+ for (Component component : context.scheduler.getAllComponents().values()) {
+ component.resetCompFailureCount();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/HttpProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/HttpProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/HttpProbe.java
new file mode 100644
index 0000000..10c1160
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/HttpProbe.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.servicemonitor.probe;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Map;
+
+public class HttpProbe extends Probe {
+ protected static final Logger log = LoggerFactory.getLogger(HttpProbe.class);
+
+ private static final String HOST_TOKEN = "${THIS_HOST}";
+
+ private final String urlString;
+ private final int timeout;
+ private final int min, max;
+
+
+ public HttpProbe(String url, int timeout, int min, int max, Configuration
+ conf) {
+ super("Http probe of " + url + " [" + min + "-" + max + "]", conf);
+ this.urlString = url;
+ this.timeout = timeout;
+ this.min = min;
+ this.max = max;
+ }
+
+ public static HttpProbe create(Map<String, String> props)
+ throws IOException {
+ String urlString = getProperty(props, WEB_PROBE_URL, null);
+ new URL(urlString);
+ int timeout = getPropertyInt(props, WEB_PROBE_CONNECT_TIMEOUT,
+ WEB_PROBE_CONNECT_TIMEOUT_DEFAULT);
+ int minSuccess = getPropertyInt(props, WEB_PROBE_MIN_SUCCESS,
+ WEB_PROBE_MIN_SUCCESS_DEFAULT);
+ int maxSuccess = getPropertyInt(props, WEB_PROBE_MAX_SUCCESS,
+ WEB_PROBE_MAX_SUCCESS_DEFAULT);
+ return new HttpProbe(urlString, timeout, minSuccess, maxSuccess, null);
+ }
+
+
+ private static HttpURLConnection getConnection(URL url, int timeout) throws
+ IOException {
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setInstanceFollowRedirects(true);
+ connection.setConnectTimeout(timeout);
+ return connection;
+ }
+
+ @Override
+ public ProbeStatus ping(ComponentInstance instance) {
+ ProbeStatus status = new ProbeStatus();
+ ContainerStatus containerStatus = instance.getContainerStatus();
+ if (containerStatus == null || SliderUtils.isEmpty(containerStatus.getIPs())
+ || StringUtils.isEmpty(containerStatus.getHost())) {
+ status.fail(this, new IOException("IP is not available yet"));
+ return status;
+ }
+
+ String ip = containerStatus.getIPs().get(0);
+ HttpURLConnection connection = null;
+ try {
+ URL url = new URL(urlString.replace(HOST_TOKEN, ip));
+ connection = getConnection(url, this.timeout);
+ int rc = connection.getResponseCode();
+ if (rc < min || rc > max) {
+ String error = "Probe " + url + " error code: " + rc;
+ log.info(error);
+ status.fail(this,
+ new IOException(error));
+ } else {
+ status.succeed(this);
+ }
+ } catch (Throwable e) {
+ String error = "Probe " + urlString + " failed for IP " + ip + ": " + e;
+ log.info(error, e);
+ status.fail(this,
+ new IOException(error, e));
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
+ }
+ }
+ return status;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/LogEntryBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/LogEntryBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/LogEntryBuilder.java
new file mode 100644
index 0000000..b575d69
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/LogEntryBuilder.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.servicemonitor.probe;
+
+/**
+ * Build up log entries for ease of splunk
+ */
+public class LogEntryBuilder {
+
+ private final StringBuilder builder = new StringBuilder();
+
+ public LogEntryBuilder() {
+ }
+
+ public LogEntryBuilder(String text) {
+ elt(text);
+ }
+
+
+ public LogEntryBuilder(String name, Object value) {
+ entry(name, value);
+ }
+
+ public LogEntryBuilder elt(String text) {
+ addComma();
+ builder.append(text);
+ return this;
+ }
+
+ public LogEntryBuilder elt(String name, Object value) {
+ addComma();
+ entry(name, value);
+ return this;
+ }
+
+ private void addComma() {
+ if (!isEmpty()) {
+ builder.append(", ");
+ }
+ }
+
+ private void entry(String name, Object value) {
+ builder.append(name).append('=');
+ if (value != null) {
+ builder.append('"').append(value.toString()).append('"');
+ } else {
+ builder.append("null");
+ }
+ }
+
+ @Override
+ public String toString() {
+ return builder.toString();
+ }
+
+ private boolean isEmpty() {
+ return builder.length() == 0;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorKeys.java
new file mode 100644
index 0000000..f5f3d99
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorKeys.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.servicemonitor.probe;
+
+/**
+ * Config keys for monitoring
+ */
+public interface MonitorKeys {
+
+ /**
+ * Port probing key : port to attempt to create a TCP connection to {@value}.
+ */
+ String PORT_PROBE_PORT = "port";
+ /**
+ * Port probing key : timeout for the the connection attempt {@value}.
+ */
+ String PORT_PROBE_CONNECT_TIMEOUT = "timeout";
+ /**
+ * Port probing default : timeout for the connection attempt {@value}.
+ */
+ int PORT_PROBE_CONNECT_TIMEOUT_DEFAULT = 1000;
+
+ /**
+ * Web probing key : URL {@value}.
+ */
+ String WEB_PROBE_URL = "url";
+ /**
+ * Web probing key : min success code {@value}.
+ */
+ String WEB_PROBE_MIN_SUCCESS = "min.success";
+ /**
+ * Web probing key : max success code {@value}.
+ */
+ String WEB_PROBE_MAX_SUCCESS = "max.success";
+ /**
+ * Web probing default : min successful response code {@value}.
+ */
+ int WEB_PROBE_MIN_SUCCESS_DEFAULT = 200;
+ /**
+ * Web probing default : max successful response code {@value}.
+ */
+ int WEB_PROBE_MAX_SUCCESS_DEFAULT = 299;
+ /**
+ * Web probing key : timeout for the connection attempt {@value}
+ */
+ String WEB_PROBE_CONNECT_TIMEOUT = "timeout";
+ /**
+ * Port probing default : timeout for the connection attempt {@value}.
+ */
+ int WEB_PROBE_CONNECT_TIMEOUT_DEFAULT = 1000;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorUtils.java
new file mode 100644
index 0000000..46d1fdb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorUtils.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.servicemonitor.probe;
+
+import org.apache.hadoop.yarn.service.api.records.ReadinessCheck;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Formatter;
+import java.util.Locale;
+
+/**
+ * Various utils to work with the monitor
+ */
+public final class MonitorUtils {
+ protected static final Logger LOG = LoggerFactory.getLogger(MonitorUtils
+ .class);
+
+ private MonitorUtils() {
+ }
+
+ public static String toPlural(int val) {
+ return val != 1 ? "s" : "";
+ }
+
+ /**
+ * Convert milliseconds to human time -the exact format is unspecified
+ * @param milliseconds a time in milliseconds
+ * @return a time that is converted to human intervals
+ */
+ public static String millisToHumanTime(long milliseconds) {
+ StringBuilder sb = new StringBuilder();
+ // Send all output to the Appendable object sb
+ Formatter formatter = new Formatter(sb, Locale.US);
+
+ long s = Math.abs(milliseconds / 1000);
+ long m = Math.abs(milliseconds % 1000);
+ if (milliseconds > 0) {
+ formatter.format("%d.%03ds", s, m);
+ } else if (milliseconds == 0) {
+ formatter.format("0");
+ } else {
+ formatter.format("-%d.%03ds", s, m);
+ }
+ return sb.toString();
+ }
+
+ public static Probe getProbe(ReadinessCheck readinessCheck) {
+ if (readinessCheck == null) {
+ return null;
+ }
+ if (readinessCheck.getType() == null) {
+ return null;
+ }
+ try {
+ switch (readinessCheck.getType()) {
+ case HTTP:
+ return HttpProbe.create(readinessCheck.getProps());
+ case PORT:
+ return PortProbe.create(readinessCheck.getProps());
+ default:
+ return null;
+ }
+ } catch (Throwable t) {
+ throw new IllegalArgumentException("Error creating readiness check " +
+ t);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/PortProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/PortProbe.java
new file mode 100644
index 0000000..f6cf3ae
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/PortProbe.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.servicemonitor.probe;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.util.Map;
+
+/**
+ * Probe for a port being open.
+ */
+public class PortProbe extends Probe {
+ protected static final Logger log = LoggerFactory.getLogger(PortProbe.class);
+ private final int port;
+ private final int timeout;
+
+ public PortProbe(int port, int timeout) {
+ super("Port probe of " + port + " for " + timeout + "ms", null);
+ this.port = port;
+ this.timeout = timeout;
+ }
+
+ public static PortProbe create(Map<String, String> props)
+ throws IOException {
+ int port = getPropertyInt(props, PORT_PROBE_PORT, null);
+
+ if (port >= 65536) {
+ throw new IOException(PORT_PROBE_PORT + " " + port + " is out of " +
+ "range");
+ }
+
+ int timeout = getPropertyInt(props, PORT_PROBE_CONNECT_TIMEOUT,
+ PORT_PROBE_CONNECT_TIMEOUT_DEFAULT);
+
+ return new PortProbe(port, timeout);
+ }
+
+ /**
+ * Try to connect to the (host,port); a failure to connect within
+ * the specified timeout is a failure.
+ * @param instance role instance
+ * @return the outcome
+ */
+ @Override
+ public ProbeStatus ping(ComponentInstance instance) {
+ ProbeStatus status = new ProbeStatus();
+
+ if (instance.getContainerStatus() == null || SliderUtils
+ .isEmpty(instance.getContainerStatus().getIPs())) {
+ status.fail(this, new IOException(
+ instance.getCompInstanceName() + ": IP is not available yet"));
+ return status;
+ }
+
+ String ip = instance.getContainerStatus().getIPs().get(0);
+ InetSocketAddress sockAddr = new InetSocketAddress(ip, port);
+ Socket socket = new Socket();
+ try {
+ if (log.isDebugEnabled()) {
+ log.debug(instance.getCompInstanceName() + ": Connecting " + sockAddr
+ .toString() + ", timeout=" + MonitorUtils
+ .millisToHumanTime(timeout));
+ }
+ socket.connect(sockAddr, timeout);
+ status.succeed(this);
+ } catch (Throwable e) {
+ String error =
+ instance.getCompInstanceName() + ": Probe " + sockAddr + " failed";
+ log.debug(error, e);
+ status.fail(this, new IOException(error, e));
+ } finally {
+ IOUtils.closeSocket(socket);
+ }
+ return status;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
deleted file mode 100644
index 06dde67..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ /dev/null
@@ -1,2138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.health.HealthCheckRegistry;
-import com.google.common.base.Preconditions;
-import com.google.protobuf.BlockingService;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.service.ServiceOperations;
-import org.apache.hadoop.service.ServiceStateChangeListener;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
-import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
-import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
-import org.apache.hadoop.yarn.service.provider.ProviderService;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.webapp.WebAppException;
-import org.apache.hadoop.yarn.webapp.WebApps;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.RoleKeys;
-import org.apache.slider.api.proto.Messages;
-import org.apache.slider.api.proto.SliderClusterAPI;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderAMArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderAMCreateAction;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.common.tools.ConfigHelper;
-import org.apache.slider.common.tools.PortScanner;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.common.tools.SliderVersionInfo;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.exceptions.SliderInternalStateException;
-import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.core.launch.CredentialUtils;
-import org.apache.slider.core.main.ExitCodeProvider;
-import org.apache.slider.core.main.LauncherExitCodes;
-import org.apache.slider.core.main.RunService;
-import org.apache.slider.core.main.ServiceLauncher;
-import org.apache.slider.core.registry.info.CustomRegistryConstants;
-import org.apache.slider.providers.ProviderCompleted;
-import org.apache.slider.server.appmaster.actions.ActionHalt;
-import org.apache.slider.server.appmaster.actions.ActionRegisterServiceInstance;
-import org.apache.slider.server.appmaster.actions.ActionStopSlider;
-import org.apache.slider.server.appmaster.actions.ActionUpgradeContainers;
-import org.apache.slider.server.appmaster.actions.AsyncAction;
-import org.apache.slider.server.appmaster.actions.EscalateOutstandingRequests;
-import org.apache.slider.server.appmaster.actions.MonitorComponentInstances;
-import org.apache.slider.server.appmaster.actions.QueueExecutor;
-import org.apache.slider.server.appmaster.actions.QueueService;
-import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
-import org.apache.slider.server.appmaster.actions.RenewingAction;
-import org.apache.slider.server.appmaster.actions.ResetFailureWindow;
-import org.apache.slider.server.appmaster.actions.ReviewAndFlexApplicationSize;
-import org.apache.slider.server.appmaster.actions.UnregisterComponentInstance;
-import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
-import org.apache.slider.server.appmaster.management.YarnServiceHealthCheck;
-import org.apache.slider.server.appmaster.monkey.ChaosKillAM;
-import org.apache.slider.server.appmaster.monkey.ChaosKillContainer;
-import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.AsyncRMOperationHandler;
-import org.apache.slider.server.appmaster.operations.RMOperationHandler;
-import org.apache.slider.server.appmaster.rpc.RpcBinder;
-import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPBImpl;
-import org.apache.slider.server.appmaster.rpc.SliderIPCService;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
-import org.apache.slider.server.appmaster.state.ProviderAppState;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
-import org.apache.slider.server.appmaster.web.SliderAMWebApp;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.WebAppApiImpl;
-import org.apache.slider.server.appmaster.web.rest.InsecureAmFilterInitializer;
-import org.apache.slider.server.appmaster.web.rest.RestPaths;
-import org.apache.slider.server.appmaster.web.rest.application.ApplicationResouceContentCacheFactory;
-import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
-import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
-import org.apache.slider.server.services.utility.WebAppService;
-import org.apache.slider.server.services.workflow.ServiceThreadFactory;
-import org.apache.slider.server.services.workflow.WorkflowExecutorService;
-import org.apache.slider.server.services.workflow.WorkflowRpcService;
-import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URL;
-import java.nio.ByteBuffer;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.ReentrantLock;
-
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
-import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
-
-/**
- * This is the AM, which directly implements the callbacks from the AM and NM
- */
-public class SliderAppMaster extends AbstractSliderLaunchedService
- implements AMRMClientAsync.CallbackHandler,
- NMClientAsync.CallbackHandler,
- RunService,
- SliderExitCodes,
- SliderKeys,
- ServiceStateChangeListener,
- RoleKeys,
- ProviderCompleted,
- AppMasterActionOperations {
-
- protected static final Logger log =
- LoggerFactory.getLogger(SliderAppMaster.class);
-
- /**
- * log for YARN events
- */
- protected static final Logger LOG_YARN = log;
-
- public static final String SERVICE_CLASSNAME_SHORT = "SliderAppMaster";
- public static final String SERVICE_CLASSNAME =
- "org.apache.slider.server.appmaster." + SERVICE_CLASSNAME_SHORT;
-
- public static final int HEARTBEAT_INTERVAL = 1000;
- public static final int NUM_RPC_HANDLERS = 5;
-
- /**
- * Metrics and monitoring services.
- * Deployed in {@link #serviceInit(Configuration)}
- */
- private final MetricsAndMonitoring metricsAndMonitoring = new MetricsAndMonitoring();
-
- /**
- * metrics registry
- */
- public MetricRegistry metrics;
-
- /** Error string on chaos monkey launch failure action: {@value} */
- public static final String E_TRIGGERED_LAUNCH_FAILURE =
- "Chaos monkey triggered launch failure";
-
- /** YARN RPC to communicate with the Resource Manager or Node Manager */
- private YarnRPC yarnRPC;
-
- /** Handle to communicate with the Resource Manager*/
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private AMRMClientAsync asyncRMClient;
-
- /** Handle to communicate with the timeline service */
- private TimelineV2Client timelineClient;
-
- private boolean timelineServiceEnabled = false;
-
- ServiceTimelinePublisher serviceTimelinePublisher;
-
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private RMOperationHandler rmOperationHandler;
-
- /** Handle to communicate with the Node Manager*/
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- public NMClientAsync nmClientAsync;
-
- /**
- * Credentials for propagating down to launched containers
- */
- private Credentials containerCredentials = new Credentials();
-
- /**
- * Slider IPC: Real service handler
- */
- private SliderIPCService sliderIPCService;
- /**
- * Slider IPC: binding
- */
- private WorkflowRpcService rpcService;
-
- /**
- * Secret manager
- */
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private ClientToAMTokenSecretManager secretManager;
-
- /** Hostname of the container*/
- private String appMasterHostname = "";
- /* Port on which the app master listens for status updates from clients*/
- private int appMasterRpcPort = 0;
- /** Tracking url to which app master publishes info for clients to monitor*/
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private String appMasterTrackingUrl = "";
-
- /** Proxied app master URL (as retrieved from AM report at launch time) */
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private String appMasterProxiedUrl = "";
-
- /** Application Attempt Id ( combination of attemptId and fail count )*/
- private ApplicationAttemptId appAttemptID;
-
- /**
- * App ACLs
- */
- protected Map<ApplicationAccessType, String> applicationACLs;
-
- /**
- * Ongoing state of the cluster: containers, nodes they
- * live on, etc.
- */
- private final AppState appState =
- new AppState(new ProtobufClusterServices(), metricsAndMonitoring);
-
- /**
- * App state for external objects. This is almost entirely
- * a read-only view of the application state. To change the state,
- * Providers (or anything else) are expected to queue async changes.
- */
- private final ProviderAppState stateForProviders =
- new ProviderAppState("undefined", appState);
-
- /**
- * model the state using locks and conditions
- */
- private final ReentrantLock AMExecutionStateLock = new ReentrantLock();
- private final Condition isAMCompleted = AMExecutionStateLock.newCondition();
-
- /**
- * Flag set if the AM is to be shutdown
- */
- private final AtomicBoolean amCompletionFlag = new AtomicBoolean(false);
-
- /**
- * Flag set during the init process
- */
- private final AtomicBoolean initCompleted = new AtomicBoolean(false);
-
- /** Arguments passed in : raw*/
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private SliderAMArgs serviceArgs;
-
- /**
- * ID of the AM container
- */
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private ContainerId appMasterContainerID;
-
- /**
- * Monkey Service -may be null
- */
- private ChaosMonkeyService monkey;
-
- /**
- * ProviderService of this cluster
- */
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private Set<ProviderService> providers = new HashSet<>();
-
- /**
- * The YARN registry service
- */
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private RegistryOperations registryOperations;
-
- /**
- * The stop request received...the exit details are extracted
- * from this
- */
- private volatile ActionStopSlider stopAction;
-
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private RoleLaunchService launchService;
-
- //username -null if it is not known/not to be set
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private String hadoop_user_name;
- private String service_user_name;
-
- private SliderAMWebApp webApp;
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private InetSocketAddress rpcServiceAddress;
-
- /**
- * Executor.
- * Assigned in {@link #serviceInit(Configuration)}
- */
- private WorkflowExecutorService<ExecutorService> executorService;
-
- /**
- * Action queues. Created at instance creation, but
- * added as a child and inited in {@link #serviceInit(Configuration)}
- */
- private final QueueService actionQueues = new QueueService();
- private YarnRegistryViewForProviders yarnRegistryOperations;
- //private FsDelegationTokenManager fsDelegationTokenManager;
- private RegisterApplicationMasterResponse amRegistrationData;
- private PortScanner portScanner;
-
- /**
- * Is security enabled?
- * Set early on in the {@link #createAndRunCluster(String)} operation.
- */
- private boolean securityEnabled;
- private ContentCache contentCache;
-
- /**
- * resource limits
- */
- private Resource maximumResourceCapability;
- private Application application;
- /**
- * Service Constructor
- */
- public SliderAppMaster() {
- super(SERVICE_CLASSNAME_SHORT);
- new HdfsConfiguration();
- new YarnConfiguration();
- }
-
-/* =================================================================== */
-/* service lifecycle methods */
-/* =================================================================== */
-
- @Override //AbstractService
- public synchronized void serviceInit(Configuration conf) throws Exception {
- // slider client if found
-
- Configuration customConf = SliderUtils.loadSliderClientXML();
- // Load in the server configuration - if it is actually on the Classpath
- URL serverXmlUrl = ConfigHelper.getResourceUrl(SLIDER_SERVER_XML);
- if (serverXmlUrl != null) {
- log.info("Loading {} at {}", SLIDER_SERVER_XML, serverXmlUrl);
- Configuration serverConf = ConfigHelper.loadFromResource(SLIDER_SERVER_XML);
- ConfigHelper.mergeConfigurations(customConf, serverConf,
- SLIDER_SERVER_XML, true);
- }
- serviceArgs.applyDefinitions(customConf);
- serviceArgs.applyFileSystemBinding(customConf);
- // conf now contains all customizations
-
- AbstractActionArgs action = serviceArgs.getCoreAction();
- SliderAMCreateAction createAction = (SliderAMCreateAction) action;
-
- // sort out the location of the AM
- String rmAddress = createAction.getRmAddress();
- if (rmAddress != null) {
- log.debug("Setting RM address from the command line: {}", rmAddress);
- SliderUtils.setRmSchedulerAddress(customConf, rmAddress);
- }
-
- log.info("AM configuration:\n{}",
- ConfigHelper.dumpConfigToString(customConf));
- for (Map.Entry<String, String> envs : System.getenv().entrySet()) {
- log.info("System env {}={}", envs.getKey(), envs.getValue());
- }
-
- ConfigHelper.mergeConfigurations(conf, customConf, SLIDER_CLIENT_XML, true);
- //init security with our conf
- if (SliderUtils.isHadoopClusterSecure(conf)) {
- log.info("Secure mode with kerberos realm {}",
- SliderUtils.getKerberosRealm());
- UserGroupInformation.setConfiguration(conf);
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- log.debug("Authenticating as {}", ugi);
- SliderUtils.verifyPrincipalSet(conf, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
- } else {
- log.info("Cluster is insecure");
- }
- log.info("Login user is {}", UserGroupInformation.getLoginUser());
-
- //look at settings of Hadoop Auth, to pick up a problem seen once
- checkAndWarnForAuthTokenProblems();
-
- // validate server env
- boolean dependencyChecks =
- !conf.getBoolean(KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED,
- false);
- SliderUtils.validateSliderServerEnvironment(log, dependencyChecks);
-
- // create and register monitoring services
- addService(metricsAndMonitoring);
- metrics = metricsAndMonitoring.getMetrics();
-/* TODO: turn these one once the metrics testing is more under control
- metrics.registerAll(new ThreadStatesGaugeSet());
- metrics.registerAll(new MemoryUsageGaugeSet());
- metrics.registerAll(new GarbageCollectorMetricSet());
-
-*/
- contentCache = ApplicationResouceContentCacheFactory.createContentCache(stateForProviders);
-
- executorService = new WorkflowExecutorService<>("AmExecutor",
- Executors.newFixedThreadPool(2,
- new ServiceThreadFactory("AmExecutor", true)));
- addService(executorService);
-
- addService(actionQueues);
- if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
- timelineServiceEnabled = true;
- log.info("Enabled YARN timeline service v2. ");
- }
-
- //init all child services
- super.serviceInit(conf);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
- HealthCheckRegistry health = metricsAndMonitoring.getHealth();
- health.register("AM Health", new YarnServiceHealthCheck(this));
- }
-
- /**
- * Start the queue processing
- */
- private void startQueueProcessing() {
- log.info("Queue Processing started");
- executorService.execute(actionQueues);
- executorService.execute(new QueueExecutor(this, actionQueues));
- }
-
-/* =================================================================== */
-/* RunService methods called from ServiceLauncher */
-/* =================================================================== */
-
- /**
- * pick up the args from the service launcher
- * @param config configuration
- * @param args argument list
- */
- @Override // RunService
- public Configuration bindArgs(Configuration config, String... args) throws Exception {
- // let the superclass process it
- Configuration superConf = super.bindArgs(config, args);
-
- //yarn-ify
- YarnConfiguration yarnConfiguration = new YarnConfiguration(
- superConf);
- serviceArgs = new SliderAMArgs(args);
- serviceArgs.parse();
-
- return SliderUtils.patchConfiguration(yarnConfiguration);
- }
-
-
- /**
- * this is called by service launcher; when it returns the application finishes
- * @return the exit code to return by the app
- * @throws Throwable
- */
- @Override
- public int runService() throws Throwable {
- SliderVersionInfo.loadAndPrintVersionInfo(log);
-
- //dump the system properties if in debug mode
- if (log.isDebugEnabled()) {
- log.debug("System properties:\n" + SliderUtils.propertiesToString(System.getProperties()));
- }
-
- //choose the action
- String action = serviceArgs.getAction();
- List<String> actionArgs = serviceArgs.getActionArgs();
- int exitCode;
- switch (action) {
- case SliderActions.ACTION_HELP:
- log.info("{}: {}", getName(), serviceArgs.usage());
- exitCode = SliderExitCodes.EXIT_USAGE;
- break;
- case SliderActions.ACTION_CREATE:
- exitCode = createAndRunCluster(actionArgs.get(0));
- break;
- default:
- throw new SliderException("Unimplemented: " + action);
- }
- log.info("Exiting AM; final exit code = {}", exitCode);
- return exitCode;
- }
-
- /**
- * Initialize a newly created service then add it.
- * Because the service is not started, this MUST be done before
- * the AM itself starts, or it is explicitly added after
- * @param service the service to init
- */
- public Service initAndAddService(Service service) {
- service.init(getConfig());
- addService(service);
- return service;
- }
-
- /* =================================================================== */
-
- /**
- * Create and run the cluster.
- * @param appName cluster name
- * @return exit code
- * @throws Throwable on a failure
- */
- private int createAndRunCluster(String appName) throws Throwable {
- Path appDir = new Path((serviceArgs.getAppDefPath()));
- SliderFileSystem fs = getClusterFS();
- fs.setAppDir(appDir);
- application = ServiceApiUtil.loadApplication(fs, appName);
- log.info("Application Json: " + application);
- stateForProviders.setApplicationName(appName);
- Configuration serviceConf = getConfig();
-
- // obtain security state
- // set the global security flag for the instance definition
-
- // initialize our providers
- for (Component component : application.getComponents()) {
- ProviderFactory factory = ProviderFactory
- .createSliderProviderFactory(component.getArtifact());
- ProviderService providerService = factory.createServerProvider();
- // init the provider BUT DO NOT START IT YET
-// initAndAddService(providerService);
- providers.add(providerService);
- }
-
- InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf);
- log.info("RM is at {}", rmSchedulerAddress);
- yarnRPC = YarnRPC.create(serviceConf);
-
- // set up the YARN client. This may require patching in the RM client-API address if it
- // is (somehow) unset server-side. String clientRMaddr = serviceConf.get(YarnConfiguration.RM_ADDRESS);
- InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(serviceConf);
- if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
- // client addr is being unset. We can lift it from the other RM APIs
- log.warn("Yarn RM address was unbound; attempting to fix up");
- serviceConf.set(YarnConfiguration.RM_ADDRESS,
- String.format("%s:%d", rmSchedulerAddress.getHostString(), clientRpcAddress.getPort() ));
- }
-
- /*
- * Extract the container ID. This is then
- * turned into an (incomplete) container
- */
- appMasterContainerID = ConverterUtils.toContainerId(
- SliderUtils.mandatoryEnvVariable(ApplicationConstants.Environment.CONTAINER_ID.name()));
- appAttemptID = appMasterContainerID.getApplicationAttemptId();
-
- ApplicationId appid = appAttemptID.getApplicationId();
- log.info("AM for ID {}", appid.getId());
-
- Map<String, String> envVars;
- List<Container> liveContainers;
-
- /*
- * It is critical this section is synchronized, to stop async AM events
- * arriving while registering a restarting AM.
- */
- synchronized (appState) {
- int heartbeatInterval = HEARTBEAT_INTERVAL;
-
- // configure AM to wait forever for RM
- getConfig().setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
- -1);
- getConfig().unset(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS);
-
- // add the RM client -this brings the callbacks in
- asyncRMClient = AMRMClientAsync.createAMRMClientAsync(heartbeatInterval, this);
- addService(asyncRMClient);
- //now bring it up
- deployChildService(asyncRMClient);
-
- if (timelineServiceEnabled) {
- timelineClient = TimelineV2Client.createTimelineClient(appid);
- asyncRMClient.registerTimelineV2Client(timelineClient);
- timelineClient.init(getConfig());
- timelineClient.start();
- log.info("Timeline v2 client started.");
-
- serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
- serviceTimelinePublisher.init(getConfig());
- serviceTimelinePublisher.start();
-
- for (ProviderService providerService : providers) {
- }
- appState.setServiceTimelinePublisher(serviceTimelinePublisher);
- log.info("ServiceTimelinePublisher started.");
- }
-
-
- // nmclient relays callbacks back to this class
- nmClientAsync = new NMClientAsyncImpl("nmclient", this);
- deployChildService(nmClientAsync);
-
- // set up secret manager
- secretManager = new ClientToAMTokenSecretManager(appAttemptID, null);
-
- if (securityEnabled) {
- // fix up the ACLs if they are not set
- String acls = serviceConf.get(KEY_PROTOCOL_ACL);
- if (acls == null) {
- getConfig().set(KEY_PROTOCOL_ACL, "*");
- }
- }
-
- //bring up the Slider RPC service
- buildPortScanner();
- startSliderRPCServer();
-
- rpcServiceAddress = rpcService.getConnectAddress();
- appMasterHostname = rpcServiceAddress.getAddress().getCanonicalHostName();
- appMasterRpcPort = rpcServiceAddress.getPort();
- appMasterTrackingUrl = null;
- log.info("AM Server is listening at {}:{}", appMasterHostname, appMasterRpcPort);
-
- log.info("Starting Yarn registry");
- registryOperations = startRegistryOperationsService();
- log.info(registryOperations.toString());
-
- // Start up the WebApp and track the URL for it
- // Web service endpoints: initialize
- WebAppApiImpl webAppApi =
- new WebAppApiImpl(
- stateForProviders,
- registryOperations,
- metricsAndMonitoring,
- actionQueues);
- initAMFilterOptions(serviceConf);
-
- int webAppPort = deployWebApplication(webAppApi);
-
- String scheme = WebAppUtils.HTTP_PREFIX;
- appMasterTrackingUrl = scheme + appMasterHostname + ":" + webAppPort;
-
- // *****************************************************
- // Register self with ResourceManager
- // This will start heartbeating to the RM
- // address = SliderUtils.getRmSchedulerAddress(asyncRMClient.getConfig());
- // *****************************************************
- log.info("Connecting to RM at {}; AM tracking URL={}",
- appMasterRpcPort, appMasterTrackingUrl);
- amRegistrationData = asyncRMClient.registerApplicationMaster(appMasterHostname,
- appMasterRpcPort,
- appMasterTrackingUrl);
- maximumResourceCapability = amRegistrationData.getMaximumResourceCapability();
-
- //TODO should not read local configs !!!
- int minMemory = serviceConf.getInt(RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
- DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
- // validate scheduler vcores allocation setting
- int minCores = serviceConf.getInt(RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
- DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
- int maxMemory = maximumResourceCapability.getMemory();
- int maxCores = maximumResourceCapability.getVirtualCores();
- appState.setContainerLimits(minMemory,maxMemory, minCores, maxCores );
-
- // build the handler for RM request/release operations; this uses
- // the max value as part of its lookup
- rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient, maximumResourceCapability);
-
- stripAMRMToken();
-
-// if (securityEnabled) {
-// secretManager.setMasterKey(
-// amRegistrationData.getClientToAMTokenMasterKey().array());
-// applicationACLs = amRegistrationData.getApplicationACLs();
-//
-// //tell the server what the ACLs are
-// rpcService.getServer().refreshServiceAcl(serviceConf,
-// new SliderAMPolicyProvider());
-// if (securityConfiguration.isKeytabProvided()) {
-// // perform keytab based login to establish kerberos authenticated
-// // principal. Can do so now since AM registration with RM above required
-// // tokens associated to principal
-// String principal = securityConfiguration.getPrincipal();
-// //TODO read key tab file from slider-am.xml
-// File localKeytabFile = new File("todo");
-//// securityConfiguration.getKeytabFile(new AggregateConf());
-// // Now log in...
-// login(principal, localKeytabFile);
-// // obtain new FS reference that should be kerberos based and different
-// // than the previously cached reference
-// fs = new SliderFileSystem(serviceConf);
-// }
-// }
-
- // YARN client.
- // Important: this is only valid at startup, and must be executed within
- // the right UGI context. Use with care.
- YarnClient yarnClient = null;
- List<NodeReport> nodeReports;
- try {
- yarnClient = YarnClient.createYarnClient();
- yarnClient.init(getConfig());
- yarnClient.start();
- nodeReports = getNodeReports(yarnClient);
- log.info("Yarn node report count: {}", nodeReports.size());
- // look up the application itself -this is needed to get the proxied
- // URL of the AM, for registering endpoints.
- // this call must be made after the AM has registered itself, obviously
- ApplicationAttemptReport report = getApplicationAttemptReport(yarnClient);
- appMasterProxiedUrl = report.getTrackingUrl();
- if (SliderUtils.isUnset(appMasterProxiedUrl)) {
- log.warn("Proxied URL is not set in application report");
- appMasterProxiedUrl = appMasterTrackingUrl;
- }
- } finally {
- // at this point yarnClient is no longer needed.
- // stop it immediately
- ServiceOperations.stop(yarnClient);
- yarnClient = null;
- }
-
- // extract container list
-
- liveContainers = amRegistrationData.getContainersFromPreviousAttempts();
- DefaultMetricsSystem.initialize("SliderAppMaster");
- if (timelineServiceEnabled) {
- DefaultMetricsSystem.instance().register("ServiceMetricsSink",
- "For processing metrics to ATS",
- new ServiceMetricsSink(serviceTimelinePublisher));
- log.info("ServiceMetricsSink registered.");
- }
-
- //determine the location for the role history data
- Path historyDir = new Path(appDir, HISTORY_DIR_NAME);
-
- //build the instance
- AppStateBindingInfo binding = new AppStateBindingInfo();
- binding.serviceConfig = null;
- binding.fs = fs.getFileSystem();
- binding.historyPath = historyDir;
- binding.liveContainers = liveContainers;
- binding.releaseSelector = new MostRecentContainerReleaseSelector();
- binding.nodeReports = nodeReports;
- binding.application = application;
- binding.serviceHdfsDir = new Path(fs.buildClusterDirPath(appName),
- SliderKeys.DATA_DIR_NAME).toString();
- appState.buildInstance(binding);
-
- // build up environment variables that the AM wants set in every container
- // irrespective of provider and role.
- envVars = new HashMap<>();
- if (hadoop_user_name != null) {
- envVars.put(HADOOP_USER_NAME, hadoop_user_name);
- }
- String debug_kerberos = System.getenv(HADOOP_JAAS_DEBUG);
- if (debug_kerberos != null) {
- envVars.put(HADOOP_JAAS_DEBUG, debug_kerberos);
- }
- }
- String rolesTmpSubdir = appMasterContainerID.toString() + "/roles";
-
- String amTmpDir = "/tmp";
- //TODO read tmpDir from slider-am.xml
- Path tmpDirPath = new Path(amTmpDir);
- Path launcherTmpDirPath = new Path(tmpDirPath, rolesTmpSubdir);
- fs.getFileSystem().mkdirs(launcherTmpDirPath);
-
- //launcher service
- launchService = new RoleLaunchService(actionQueues,
- fs, envVars);
-
- deployChildService(launchService);
-
- //Give the provider access to the state, and AM
- for (ProviderService providerService : providers) {
-// providerService.setAMState(stateForProviders);
- }
-
- // chaos monkey
- maybeStartMonkey();
-
- // if not a secure cluster, extract the username -it will be
- // propagated to workers
- if (!UserGroupInformation.isSecurityEnabled()) {
- hadoop_user_name = System.getenv(HADOOP_USER_NAME);
- log.info(HADOOP_USER_NAME + "='{}'", hadoop_user_name);
- }
- service_user_name = RegistryUtils.currentUser();
- log.info("Registry service username ={}", service_user_name);
-
-
- // declare the cluster initialized
- log.info("Application Master Initialization Completed");
- initCompleted.set(true);
-
- scheduleFailureWindowResets(application.getConfiguration());
- scheduleEscalation(application.getConfiguration());
- scheduleMonitoring(application.getConfiguration());
-
- try {
- // schedule YARN Registry registration
- queue(new ActionRegisterServiceInstance(appName, appid, application));
-
- // log the YARN and web UIs
- log.info("RM Webapp address {}",
- serviceConf.get(YarnConfiguration.RM_WEBAPP_ADDRESS));
- log.info("Slider webapp address {} proxied at {}",
- appMasterTrackingUrl, appMasterProxiedUrl);
- // launch the real provider; this is expected to trigger a callback that
- // starts the node review process
- launchProviderService();
-
- // start handling any scheduled events
-
- startQueueProcessing();
-
- //now block waiting to be told to exit the process
- waitForAMCompletionSignal();
- } catch(Exception e) {
- log.error("Exception : {}", e, e);
- // call the AM stop command as if it had been queued (but without
- // going via the queue, which may not have started
- onAMStop(new ActionStopSlider(e));
- }
- //shutdown time
- return finish();
- }
-
- /**
- * Get the YARN application Attempt report as the logged in user
- * @param yarnClient client to the RM
- * @return the application report
- * @throws YarnException
- * @throws IOException
- * @throws InterruptedException
- */
- private ApplicationAttemptReport getApplicationAttemptReport(
- final YarnClient yarnClient)
- throws YarnException, IOException, InterruptedException {
- Preconditions.checkNotNull(yarnClient, "Null Yarn client");
- ApplicationAttemptReport report;
- if (securityEnabled) {
- UserGroupInformation ugi = UserGroupInformation.getLoginUser();
- report = ugi.doAs(new PrivilegedExceptionAction<ApplicationAttemptReport>() {
- @Override
- public ApplicationAttemptReport run() throws Exception {
- return yarnClient.getApplicationAttemptReport(appAttemptID);
- }
- });
- } else {
- report = yarnClient.getApplicationAttemptReport(appAttemptID);
- }
- return report;
- }
-
- /**
- * List the node reports: uses {@link YarnClient} as the login user
- * @param yarnClient client to the RM
- * @return the node reports
- * @throws IOException
- * @throws YarnException
- * @throws InterruptedException
- */
- private List<NodeReport> getNodeReports(final YarnClient yarnClient)
- throws IOException, YarnException, InterruptedException {
- Preconditions.checkNotNull(yarnClient, "Null Yarn client");
- List<NodeReport> nodeReports;
- if (securityEnabled) {
- nodeReports = UserGroupInformation.getLoginUser().doAs(
- new PrivilegedExceptionAction<List<NodeReport>>() {
- @Override
- public List<NodeReport> run() throws Exception {
- return yarnClient.getNodeReports(NodeState.RUNNING);
- }
- });
- } else {
- nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
- }
- log.info("Yarn node report count: {}", nodeReports.size());
- return nodeReports;
- }
-
- /**
- * Deploy the web application.
- * <p>
- * Creates and starts the web application, and adds a
- * <code>WebAppService</code> service under the AM, to ensure
- * a managed web application shutdown.
- * @param webAppApi web application API instance
- * @return port the web application is deployed on
- * @throws IOException general problems starting the webapp (network, etc)
- * @throws WebAppException other issues
- */
- private int deployWebApplication(WebAppApiImpl webAppApi)
- throws IOException, SliderException {
-
- try {
- webApp = new SliderAMWebApp(webAppApi);
- HttpConfig.Policy policy = HttpConfig.Policy.HTTP_ONLY;
- int port = getPortToRequest();
- log.info("Launching web application at port {} with policy {}", port, policy);
-
- WebApps.$for(SliderAMWebApp.BASE_PATH,
- WebAppApi.class,
- webAppApi,
- RestPaths.WS_CONTEXT)
- .withHttpPolicy(getConfig(), policy)
- .at("0.0.0.0", port, true)
- .inDevMode()
- .start(webApp);
-
- WebAppService<SliderAMWebApp> webAppService =
- new WebAppService<>("slider", webApp);
-
- deployChildService(webAppService);
- return webApp.port();
- } catch (WebAppException e) {
- if (e.getCause() instanceof IOException) {
- throw (IOException)e.getCause();
- } else {
- throw e;
- }
- }
- }
-
- /**
- * Process the initial user to obtain the set of user
- * supplied credentials (tokens were passed in by client).
- * Removes the AM/RM token.
- * @throws IOException
- */
- private void stripAMRMToken()
- throws IOException {
- List<Text> filteredTokens = new ArrayList<>(3);
- filteredTokens.add(AMRMTokenIdentifier.KIND_NAME);
- containerCredentials = CredentialUtils.filterTokens(
- UserGroupInformation.getCurrentUser().getCredentials(),
- filteredTokens);
- log.info(CredentialUtils.dumpTokens(containerCredentials, "\n"));
- }
-
- /**
- * Build up the port scanner. This may include setting a port range.
- */
- private void buildPortScanner()
- throws BadConfigException {
- portScanner = new PortScanner();
- String portRange = "0";
- //TODO read from slider-am.xml
-// String portRange = instanceDefinition.
-// getAppConfOperations().getGlobalOptions().
-// getOption(SliderKeys.KEY_ALLOWED_PORT_RANGE, "0");
- if (!"0".equals(portRange)) {
- portScanner.setPortRange(portRange);
- }
- }
-
- /**
- * Locate a port to request for a service such as RPC or web/REST.
- * This uses port range definitions in the <code>instanceDefinition</code>
- * to fix the port range —if one is set.
- * <p>
- * The port returned is available at the time of the request; there are
- * no guarantees as to how long that situation will last.
- * @return the port to request.
- * @throws SliderException
- */
- private int getPortToRequest() throws SliderException, IOException {
- return portScanner.getAvailablePort();
- }
-
- protected void login(String principal, File localKeytabFile)
- throws IOException, SliderException {
- log.info("Logging in as {} with keytab {}", principal, localKeytabFile);
- UserGroupInformation.loginUserFromKeytab(principal,
- localKeytabFile.getAbsolutePath());
- validateLoginUser(UserGroupInformation.getLoginUser());
- }
-
- /**
- * Ensure that the user is generated from a keytab and has no HDFS delegation
- * tokens.
- *
- * @param user user to validate
- * @throws SliderException
- */
- protected void validateLoginUser(UserGroupInformation user)
- throws SliderException {
- if (!user.isFromKeytab()) {
- log.error("User is not holding on a keytab in a secure deployment:" +
- " slider will fail as tokens expire");
- }
- Credentials credentials = user.getCredentials();
- Iterator<Token<? extends TokenIdentifier>> iter =
- credentials.getAllTokens().iterator();
- while (iter.hasNext()) {
- Token<? extends TokenIdentifier> token = iter.next();
- log.info("Token {}", token.getKind());
- if (token.getKind().equals(
- DelegationTokenIdentifier.HDFS_DELEGATION_KIND)) {
- log.info("HDFS delegation token {}. Removing...", token);
- iter.remove();
- }
- }
- }
-
- /**
- * Set up the AM filter
- * @param serviceConf configuration to patch
- */
- private void initAMFilterOptions(Configuration serviceConf) {
- // IP filtering
- String amFilterName = AM_FILTER_NAME;
-
- // This is here until YARN supports proxy & redirect operations
- // on verbs other than GET, and is only supported for testing
- if (X_DEV_INSECURE_REQUIRED && serviceConf.getBoolean(X_DEV_INSECURE_WS,
- X_DEV_INSECURE_DEFAULT)) {
- log.warn("Insecure filter enabled: REST operations are unauthenticated");
- amFilterName = InsecureAmFilterInitializer.NAME;
- }
-
- serviceConf.set(HADOOP_HTTP_FILTER_INITIALIZERS, amFilterName);
- }
-
- /**
- * This registers the service instance and its external values
- * @param instanceName name of this instance
- * @param appId application ID
- * @throws IOException
- */
- public void registerServiceInstance(String instanceName,
- ApplicationId appId, Application application) throws IOException {
-
- //Give the provider restricted access to the state, registry
- setupInitialRegistryPaths();
- yarnRegistryOperations = new YarnRegistryViewForProviders(
- registryOperations,
- service_user_name,
- SliderKeys.APP_TYPE,
- instanceName,
- appAttemptID);
- for (ProviderService providerService : providers) {
-// providerService.bindToYarnRegistry(yarnRegistryOperations);
- }
-
- // Yarn registry
- ServiceRecord serviceRecord = new ServiceRecord();
- serviceRecord.set(YarnRegistryAttributes.YARN_ID, appId.toString());
- serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
- PersistencePolicies.APPLICATION);
- serviceRecord.description = "Slider Application Master";
-
- serviceRecord.addExternalEndpoint(
- RegistryTypeUtils.ipcEndpoint(
- CustomRegistryConstants.AM_IPC_PROTOCOL,
- rpcServiceAddress));
-
- // set any provided attributes
- setUserProvidedServiceRecordAttributes(application.getConfiguration(),
- serviceRecord);
-
- // register the service's entry
- log.info("Service Record \n{}", serviceRecord);
- yarnRegistryOperations.registerSelf(serviceRecord, true);
- log.info("Registered service under {}; absolute path {}",
- yarnRegistryOperations.getSelfRegistrationPath(),
- yarnRegistryOperations.getAbsoluteSelfRegistrationPath());
-
- boolean isFirstAttempt = 1 == appAttemptID.getAttemptId();
- // delete the children in case there are any and this is an AM startup.
- // just to make sure everything underneath is purged
- if (isFirstAttempt) {
- yarnRegistryOperations.deleteChildren(
- yarnRegistryOperations.getSelfRegistrationPath(),
- true);
- }
- if (timelineServiceEnabled) {
- serviceTimelinePublisher.serviceAttemptRegistered(application);
- }
- }
-
- /**
- * TODO: purge this once RM is doing the work
- * @throws IOException
- */
- protected void setupInitialRegistryPaths() throws IOException {
- if (registryOperations instanceof RMRegistryOperationsService) {
- RMRegistryOperationsService rmRegOperations =
- (RMRegistryOperationsService) registryOperations;
- rmRegOperations.initUserRegistryAsync(service_user_name);
- }
- }
-
- /**
- * Handler for {@link RegisterComponentInstance action}
- * Register/re-register an ephemeral container that is already in the application state
- * @param id the component
- * @return true if the component is registered
- */
- public boolean registerComponent(ContainerId id, RoleInstance roleInstance)
- throws IOException {
- RoleInstance instance = appState.getOwnedContainer(id);
- if (instance == null) {
- return false;
- }
- // this is where component registrations go
- log.info("Registering component " + roleInstance.getCompInstanceName()
- + ", containerId = " + id);
- org.apache.slider.api.resource.Container container =
- new org.apache.slider.api.resource.Container();
- container.setId(id.toString());
- container.setLaunchTime(new Date());
- container.setState(org.apache.slider.api.resource.ContainerState.RUNNING_BUT_UNREADY);
- container.setBareHost(instance.host);
- // TODO differentiate component name and component instance name ?
- container.setComponentName(roleInstance.getCompInstanceName());
- instance.providerRole.component.addContainer(container);
-
- if (timelineServiceEnabled) {
- serviceTimelinePublisher.componentInstanceStarted(container, null);
- }
- return true;
- }
-
- protected void setUserProvidedServiceRecordAttributes(
- org.apache.slider.api.resource.Configuration conf, ServiceRecord record) {
- String prefix = RoleKeys.SERVICE_RECORD_ATTRIBUTE_PREFIX;
- for (Map.Entry<String, String> entry : conf.getProperties().entrySet()) {
- if (entry.getKey().startsWith(prefix)) {
- String key = entry.getKey().substring(prefix.length() + 1);
- record.set(key, entry.getValue().trim());
- }
- }
- }
-
- /**
- * Handler for {@link UnregisterComponentInstance}
- *
- * unregister a component. At the time this message is received,
- * the component may not have been registered
- */
- public void unregisterComponent(RoleInstance roleInstance) {
- ContainerId containerId = roleInstance.getContainerId();
- log.info(
- "Unregistering component instance " + roleInstance.getCompInstanceName()
- + ", ContainerId = " + containerId);
- if (yarnRegistryOperations == null) {
- log.warn("Processing unregister component event before initialization "
- + "completed; init flag ={}", initCompleted);
- return;
- }
- String cid = RegistryPathUtils.encodeYarnID(containerId.toString());
-// try {
-// yarnRegistryOperations.deleteComponent(cid);
-// } catch (IOException e) {
-// log.warn("Failed to delete container {} : {}", containerId, e, e);
-// }
-
- // remove component instance dir
- try {
- FileSystem fs = getClusterFS().getFileSystem();
- if (roleInstance.compInstanceDir != null && fs
- .exists(roleInstance.compInstanceDir)) {
- boolean deleted = fs.delete(roleInstance.compInstanceDir, true);
- if (!deleted) {
- log.warn("Failed to delete component instance dir: "
- + roleInstance.compInstanceDir);
- }
- }
- } catch (IOException e) {
- log.error("Failed to delete component instance dir: "
- + roleInstance.compInstanceDir, e);
- }
- }
-
- /**
- * looks for a specific case where a token file is provided as an environment
- * variable, yet the file is not there.
- *
- * This surfaced (once) in HBase, where its HDFS library was looking for this,
- * and somehow the token was missing. This is a check in the AM so that
- * if the problem re-occurs, the AM can fail with a more meaningful message.
- *
- */
- private void checkAndWarnForAuthTokenProblems() {
- String fileLocation =
- System.getenv(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION);
- if (fileLocation != null) {
- File tokenFile = new File(fileLocation);
- if (!tokenFile.exists()) {
- log.warn("Token file {} specified in {} not found", tokenFile,
- UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION);
- }
- }
- }
-
- /**
- * Get the filesystem of this cluster
- * @return the FS of the config
- */
- public SliderFileSystem getClusterFS() throws IOException {
- return new SliderFileSystem(getConfig());
- }
-
- /**
- * Get the AM log
- * @return the log of the AM
- */
- public static Logger getLog() {
- return log;
- }
-
- /**
- * Get the application state
- * @return the application state
- */
- public AppState getAppState() {
- return appState;
- }
-
- /**
- * Block until it is signalled that the AM is done
- */
- private void waitForAMCompletionSignal() {
- AMExecutionStateLock.lock();
- try {
- if (!amCompletionFlag.get()) {
- log.debug("blocking until signalled to terminate");
- isAMCompleted.awaitUninterruptibly();
- }
- } finally {
- AMExecutionStateLock.unlock();
- }
- }
-
- /**
- * Signal that the AM is complete .. queues it in a separate thread
- *
- * @param stopActionRequest request containing shutdown details
- */
- public synchronized void signalAMComplete(ActionStopSlider stopActionRequest) {
- // this is a queued action: schedule it through the queues
- schedule(stopActionRequest);
- }
-
- /**
- * Signal that the AM is complete
- *
- * @param stopActionRequest request containing shutdown details
- */
- public synchronized void onAMStop(ActionStopSlider stopActionRequest) {
-
- AMExecutionStateLock.lock();
- try {
- if (amCompletionFlag.compareAndSet(false, true)) {
- // first stop request received
- this.stopAction = stopActionRequest;
- isAMCompleted.signal();
- }
- } finally {
- AMExecutionStateLock.unlock();
- }
- }
-
-
- /**
- * trigger the YARN cluster termination process
- * @return the exit code
- * @throws Exception if the stop action contained an Exception which implements
- * ExitCodeProvider
- */
- private synchronized int finish() throws Exception {
- Preconditions.checkNotNull(stopAction, "null stop action");
- FinalApplicationStatus appStatus;
- log.info("Triggering shutdown of the AM: {}", stopAction);
-
- String appMessage = stopAction.getMessage();
- //stop the daemon & grab its exit code
- int exitCode = stopAction.getExitCode();
- Exception exception = stopAction.getEx();
-
- appStatus = stopAction.getFinalApplicationStatus();
-
- // make sure the AM is actually registered. If not, there's no point
- // trying to unregister it
- if (amRegistrationData == null) {
- log.info("Application attempt not yet registered; skipping unregistration");
- if (exception != null) {
- throw exception;
- }
- return exitCode;
- }
-
- //stop any launches in progress
- launchService.stop();
-
- //now release all containers
- releaseAllContainers(application);
- DefaultMetricsSystem.shutdown();
-
- if (timelineServiceEnabled) {
- serviceTimelinePublisher.serviceAttemptUnregistered(appState, stopAction);
- serviceTimelinePublisher.stop();
- timelineClient.stop();
- }
-
- // When the application completes, it should send a finish application
- // signal to the RM
- log.info("Application completed. Signalling finish to RM");
-
- try {
- log.info("Unregistering AM status={} message={}", appStatus, appMessage);
- asyncRMClient.unregisterApplicationMaster(appStatus, appMessage, null);
- } catch (InvalidApplicationMasterRequestException e) {
- log.info("Application not found in YARN application list;" +
- " it may have been terminated/YARN shutdown in progress: {}", e, e);
- } catch (YarnException | IOException e) {
- log.info("Failed to unregister application: " + e, e);
- }
- if (exception != null) {
- throw exception;
- }
- return exitCode;
- }
-
-
- public Object getProxy(Class protocol, InetSocketAddress addr) {
- return yarnRPC.getProxy(protocol, addr, getConfig());
- }
-
- /**
- * Start the slider RPC server
- */
- private void startSliderRPCServer()
- throws IOException, SliderException {
- verifyIPCAccess();
-
- sliderIPCService = new SliderIPCService(
- this, stateForProviders,
- actionQueues,
- metricsAndMonitoring,
- contentCache);
-
- deployChildService(sliderIPCService);
- SliderClusterProtocolPBImpl protobufRelay =
- new SliderClusterProtocolPBImpl(sliderIPCService);
- BlockingService blockingService = SliderClusterAPI.SliderClusterProtocolPB
- .newReflectiveBlockingService(
- protobufRelay);
-
- int port = getPortToRequest();
- InetSocketAddress rpcAddress = new InetSocketAddress("0.0.0.0", port);
- rpcService =
- new WorkflowRpcService("SliderRPC",
- RpcBinder.createProtobufServer(rpcAddress, getConfig(),
- secretManager,
- NUM_RPC_HANDLERS,
- blockingService,
- null));
- deployChildService(rpcService);
- }
-
- /**
- * verify that if the cluster is authed, the ACLs are set.
- * @throws BadConfigException if Authorization is set without any ACL
- */
- private void verifyIPCAccess() throws BadConfigException {
- boolean authorization = getConfig().getBoolean(
- CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
- false);
- String acls = getConfig().get(KEY_PROTOCOL_ACL);
- if (authorization && SliderUtils.isUnset(acls)) {
- throw new BadConfigException("Application has IPC authorization enabled in " +
- CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION +
- " but no ACLs in " + KEY_PROTOCOL_ACL);
- }
- }
-
-
-/* =================================================================== */
-/* AMRMClientAsync callbacks */
-/* =================================================================== */
-
- /**
- * Callback event when a container is allocated.
- *
- * The app state is updated with the allocation, and builds up a list
- * of assignments and RM operations. The assignments are
- * handed off into the pool of service launchers to asynchronously schedule
- * container launch operations.
- *
- * The operations are run in sequence; they are expected to be 0 or more
- * release operations (to handle over-allocations)
- *
- * @param allocatedContainers list of containers that are now ready to be
- * given work.
- */
- @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
- @Override //AMRMClientAsync
- public void onContainersAllocated(List<Container> allocatedContainers) {
- LOG_YARN.info("onContainersAllocated({})", allocatedContainers.size());
- List<ContainerAssignment> assignments = new ArrayList<>();
- List<AbstractRMOperation> operations = new ArrayList<>();
-
- //app state makes all the decisions
- appState.onContainersAllocated(allocatedContainers, assignments, operations);
-
- //for each assignment: instantiate that role
- for (ContainerAssignment assignment : assignments) {
- //TODO Do we need to pass credentials to containers?
- launchService.launchRole(assignment, application, null);
- }
-
- //for all the operations, exec them
- execute(operations);
- }
-
- @Override //AMRMClientAsync
- public synchronized void onContainersCompleted(List<ContainerStatus> completedContainers) {
- LOG_YARN.info("onContainersCompleted([{}]", completedContainers.size());
- for (ContainerStatus status : completedContainers) {
- ContainerId containerId = status.getContainerId();
- LOG_YARN.info("Container Completion for" +
- " containerID={}," +
- " state={}," +
- " exitStatus={}," +
- " diagnostics={}",
- containerId, status.getState(),
- status.getExitStatus(),
- status.getDiagnostics());
-
- // non complete containers should not be here
- assert (status.getState() == ContainerState.COMPLETE);
- AppState.NodeCompletionResult result = appState.onCompletedContainer(status);
- if (result.containerFailed) {
- RoleInstance ri = result.roleInstance;
- log.error("Role instance {} failed ", ri);
- }
-
- // known nodes trigger notifications
- if(!result.unknownNode) {
- queue(new UnregisterComponentInstance(0,
- TimeUnit.MILLISECONDS, result.roleInstance));
-
- if (timelineServiceEnabled && result.roleInstance != null) {
- serviceTimelinePublisher
- .componentInstanceFinished(result.roleInstance);
- }
- }
- }
-
- reviewRequestAndReleaseNodes("onContainersCompleted");
- }
-
- /**
- * Signal that containers are being upgraded. Containers specified with
- * --containers option and all containers of all roles specified with
- * --components option are merged and upgraded.
- *
- * @param upgradeContainersRequest
- * request containing upgrade details
- */
- public synchronized void onUpgradeContainers(
- ActionUpgradeContainers upgradeContainersRequest) throws IOException,
- SliderException {
- LOG_YARN.info("onUpgradeContainers({})",
- upgradeContainersRequest.getMessage());
- Set<String> containers = upgradeContainersRequest.getContainers() == null ? new HashSet<String>()
- : upgradeContainersRequest.getContainers();
- LOG_YARN.info(" Container list provided (total {}) : {}",
- containers.size(), containers);
- Set<String> components = upgradeContainersRequest.getComponents() == null ? new HashSet<String>()
- : upgradeContainersRequest.getComponents();
- LOG_YARN.info(" Component list provided (total {}) : {}",
- components.size(), components);
- // If components are specified as well, then grab all the containers of
- // each of the components (roles)
- if (CollectionUtils.isNotEmpty(components)) {
- Map<ContainerId, RoleInstance> liveContainers = appState.getLiveContainers();
- if (CollectionUtils.isNotEmpty(liveContainers.keySet())) {
- Map<String, Set<String>> roleContainerMap = prepareRoleContainerMap(liveContainers);
- for (String component : components) {
- Set<String> roleContainers = roleContainerMap.get(component);
- if (roleContainers != null) {
- containers.addAll(roleContainers);
- }
- }
- }
- }
- LOG_YARN.info("Final list of containers to be upgraded (total {}) : {}",
- containers.size(), containers);
- }
-
- // create a reverse map of roles -> set of all live containers
- private Map<String, Set<String>> prepareRoleContainerMap(
- Map<ContainerId, RoleInstance> liveContainers) {
- // liveContainers is ensured to be not empty
- Map<String, Set<String>> roleContainerMap = new HashMap<>();
- for (Map.Entry<ContainerId, RoleInstance> liveContainer : liveContainers
- .entrySet()) {
- RoleInstance role = liveContainer.getValue();
- if (roleContainerMap.containsKey(role.role)) {
- roleContainerMap.get(role.role).add(liveContainer.getKey().toString());
- } else {
- Set<String> containers = new HashSet<String>();
- containers.add(liveContainer.getKey().toString());
- roleContainerMap.put(role.role, containers);
- }
- }
- return roleContainerMap;
- }
-
- /**
- * Implementation of cluster flexing.
- * It should be the only way that anything -even the AM itself on startup-
- * asks for nodes.
- * @throws SliderException slider problems, including invalid configs
- * @throws IOException IO problems
- */
- public void flexCluster(Messages.FlexComponentsRequestProto request)
- throws IOException, SliderException {
- if (request != null) {
- appState.updateComponents(request);
- }
- // reset the scheduled windows...the values
- // may have changed
- appState.resetFailureCounts();
-
- // ask for more containers if needed
- reviewRequestAndReleaseNodes("flexCluster");
- }
-
- /**
- * Schedule the failure window
- * @throws BadConfigException if the window is out of range
- */
- private void scheduleFailureWindowResets(
- org.apache.slider.api.resource.Configuration conf) {
-
- ResetFailureWindow reset = new ResetFailureWindow(rmOperationHandler);
-
- long totalSeconds = SliderUtils.getTimeRange(conf,
- ResourceKeys.CONTAINER_FAILURE_WINDOW,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES,
- 0);
- if (totalSeconds > 0) {
- log.info("Scheduling the failure window reset interval to every {}"
- + " seconds", totalSeconds);
- RenewingAction<ResetFailureWindow> renew =
- new RenewingAction<>(reset, totalSeconds, totalSeconds,
- TimeUnit.SECONDS, 0);
- actionQueues.renewing("failures", renew);
- } else {
- log.info("Failure window reset interval is not set");
- }
- }
-
- /**
- * Schedule the escalation action
- * @throws BadConfigException
- */
- private void scheduleEscalation(
- org.apache.slider.api.resource.Configuration conf) {
- EscalateOutstandingRequests escalate = new EscalateOutstandingRequests();
- long seconds = conf.getPropertyLong(InternalKeys.ESCALATION_CHECK_INTERVAL,
- InternalKeys.DEFAULT_ESCALATION_CHECK_INTERVAL);
- RenewingAction<EscalateOutstandingRequests> renew =
- new RenewingAction<>(escalate, seconds, seconds, TimeUnit.SECONDS, 0);
- actionQueues.renewing("escalation", renew);
- }
-
- /**
- * Schedule monitor action
- */
- private void scheduleMonitoring(
- org.apache.slider.api.resource.Configuration conf) {
- MonitorComponentInstances monitor = new MonitorComponentInstances();
- long seconds = conf.getPropertyLong(InternalKeys.MONITOR_INTERVAL,
- InternalKeys.DEFAULT_MONITOR_INTERVAL);
- RenewingAction<MonitorComponentInstances> renew =
- new RenewingAction<>(monitor, seconds, seconds, TimeUnit.SECONDS, 0);
- actionQueues.renewing("monitoring", renew);
- }
-
- /**
- * Look at where the current node state is and whether it should be changed.
- * @param reason reason for operation
- */
- private synchronized void reviewRequestAndReleaseNodes(String reason) {
- log.info("reviewRequestAndReleaseNodes({})", reason);
- queue(new ReviewAndFlexApplicationSize(reason, 0, TimeUnit.SECONDS));
- }
-
- /**
- * Handle the event requesting a review ... look at the queue and decide
- * whether to act or not
- * @param action action triggering the event. It may be put
- * back into the queue
- * @throws SliderInternalStateException
- */
- public void handleReviewAndFlexApplicationSize(ReviewAndFlexApplicationSize action)
- throws SliderInternalStateException {
-
- if ( actionQueues.hasQueuedActionWithAttribute(
- AsyncAction.ATTR_REVIEWS_APP_SIZE | AsyncAction.ATTR_HALTS_APP)) {
- //TODO Loop all actions to check duplicate ？？
- // this operation isn't needed at all -existing duplicate or shutdown due
- return;
- }
- // if there is an action which changes cluster size, wait
- if (actionQueues.hasQueuedActionWithAttribute(
- AsyncAction.ATTR_CHANGES_APP_SIZE)) {
- // place the action at the back of the queue
- actionQueues.put(action);
- }
-
- executeNodeReview(action.name);
- }
-
- /**
- * Look at where the current node state is -and whether it should be changed
- */
- public synchronized void executeNodeReview(String reason)
- throws SliderInternalStateException {
-
- log.info("in executeNodeReview({})", reason);
- if (amCompletionFlag.get()) {
- log.info("Ignoring node review operation: shutdown in progress");
- }
- try {
- List<AbstractRMOperation> allOperations = appState.reviewRequestAndReleaseNodes();
- //now apply the operations
- execute(allOperations);
- } catch (TriggerClusterTeardownException e) {
- //App state has decided that it is time to exit
- log.error("Cluster teardown triggered {}", e, e);
- queue(new ActionStopSlider(e));
- }
- }
-
- /**
- * Escalate operation as triggered by external timer.
- * <p>
- * Get the list of new operations off the AM, then executest them.
- */
- public void escalateOutstandingRequests() {
- List<AbstractRMOperation> operations = appState.escalateOutstandingRequests();
- execute(operations);
- }
-
- public void monitorComponentInstances() {
- // TODO use health checks?
- // TODO publish timeline events for monitoring changes?
- if (appState.monitorComponentInstances()) {
- // monitoring change
- reviewRequestAndReleaseNodes("monitoring change");
- }
- }
-
-
- /**
- * Shutdown operation: release all containers
- */
- private void releaseAllContainers(Application application) {
- // Add the sleep here (before releasing containers) so that applications get
- // time to perform graceful shutdown
- try {
- long timeout = getContainerReleaseTimeout(application);
- if (timeout > 0) {
- Thread.sleep(timeout);
- }
- } catch (InterruptedException e) {
- log.info("Sleep for container release interrupted");
- } finally {
- List<AbstractRMOperation> operations = appState.releaseAllContainers();
- // now apply the operations
- execute(operations);
- }
- }
-
- private long getContainerReleaseTimeout(Application application) {
- // Get container release timeout in millis or 0 if the property is not set.
- long timeout = application.getConfiguration()
- .getPropertyLong(SliderKeys.APP_CONTAINER_RELEASE_TIMEOUT, 0);
-
- // convert to millis
- long timeoutInMillis = timeout * 1000l;
- log.info("Container release timeout in millis = {}", timeoutInMillis);
- return timeoutInMillis;
- }
-
- /**
- * RM wants to shut down the AM
- */
- @Override //AMRMClientAsync
- public void onShutdownRequest() {
- LOG_YARN.info("Shutdown Request received");
- signalAMComplete(new ActionStopSlider("stop",
- EXIT_SUCCESS,
- FinalApplicationStatus.SUCCEEDED,
- "Shutdown requested from RM"));
- }
-
- /**
- * Monitored nodes have been changed
- * @param updatedNodes list of updated nodes
- */
- @Override //AMRMClientAsync
- public void onNodesUpdated(List<NodeReport> updatedNodes) {
- LOG_YARN.info("onNodesUpdated({})", updatedNodes.size());
- log.info("Updated nodes {}", updatedNodes);
- // Check if any nodes are lost or revived and update state accordingly
-
- AppState.NodeUpdatedOutcome outcome = appState.onNodesUpdated(updatedNodes);
- if (!outcome.operations.isEmpty()) {
- execute(outcome.operations);
- }
- // trigger a review if the cluster changed
- if (outcome.clusterChanged) {
- reviewRequestAndReleaseNodes("nodes updated");
- }
- }
-
- /**
- * heartbeat operation; return the ratio of requested
- * to actual
- * @return progress
- */
- @Override //AMRMClientAsync
- public float getProgress() {
- return appState.getApplicationProgressPercentage();
- }
-
- @Override //AMRMClientAsync
- public void onError(Throwable e) {
- if (e instanceof InvalidResourceRequestException) {
- // stop the cluster
- LOG_YARN.error("AMRMClientAsync.onError() received {}", e, e);
- signalAMComplete(new ActionStopSlider("stop", EXIT_EXCEPTION_THROWN,
- FinalApplicationStatus.FAILED,
- SliderUtils.extractFirstLine(e.getLocalizedMessage())));
- } else if (e instanceof InvalidApplicationMasterRequestException) {
- // halt the AM
- LOG_YARN.error("AMRMClientAsync.onError() received {}", e, e);
- queue(new ActionHalt(EXIT_EXCEPTION_THROWN,
- SliderUtils.extractFirstLine(e.getLocalizedMessage())));
- } else {
- // ignore and log
- LOG_YARN.info("Ignoring AMRMClientAsync.onError() received {}", e);
- }
- }
-
-/* =================================================================== */
-/* RMOperationHandlerActions */
-/* =================================================================== */
-
-
- @Override
- public void execute(List<AbstractRMOperation> operations) {
- rmOperationHandler.execute(operations);
- }
-
- @Override
- public void releaseAssignedContainer(ContainerId containerId) {
- rmOperationHandler.releaseAssignedContainer(containerId);
- }
-
- @Override
- public void addContainerRequest(AMRMClient.ContainerRequest req) {
- rmOperationHandler.addContainerRequest(req);
- }
-
- @Override
- public int cancelContainerRequests(Priority priority1,
- Priority priority2,
- int count) {
- return rmOperationHandler.cancelContainerRequests(priority1, priority2, count);
- }
-
- @Override
- public void cancelSingleRequest(AMRMClient.ContainerRequest request) {
- rmOperationHandler.cancelSingleRequest(request);
- }
-
- @Override
- public void updateBlacklist(List<String> blacklistAdditions,
- List<String> blacklistRemovals) {
- rmOperationHandler.updateBlacklist(blacklistAdditions, blacklistRemovals);
- }
-
-/* =================================================================== */
-/* END */
-/* =================================================================== */
-
- /**
- * Launch the provider service
- * @throws IOException
- * @throws SliderException
- */
- protected synchronized void launchProviderService()
- throws IOException, SliderException {
- // didn't start, so don't register
- for (ProviderService providerService : providers) {
-// providerService.start();
- }
- // and send the started event ourselves
- eventCallbackEvent(null);
- }
-
- /* =================================================================== */
- /* EventCallback from the child or ourselves directly */
- /* =================================================================== */
-
- @Override // ProviderCompleted
- public void eventCallbackEvent(Object parameter) {
- // now ask for the cluster nodes
- try {
- flexCluster(null);
- } catch (Exception e) {
- // cluster flex failure: log
- log.error("Failed to flex cluster nodes: {}", e, e);
- // then what? exit
- queue(new ActionStopSlider(e));
- }
- }
-
-
- /**
- * Async start container request
- * @param container container
- * @param ctx context
- * @param instance node details
- */
- public void startContainer(Container container,
- ContainerLaunchContext ctx,
- RoleInstance instance) throws IOException {
- appState.containerStartSubmitted(container, instance);
-
- nmClientAsync.startContainerAsync(container, ctx);
- }
-
- @Override // NMClientAsync.CallbackHandler
- public void onContainerStopped(ContainerId containerId) {
- // do nothing but log: container events from the AM
- // are the source of container halt details to react to
- log.info("onContainerStopped {} ", containerId);
- }
-
- @Override // NMClientAsync.CallbackHandler
- public void onContainerStarted(ContainerId containerId,
- Map<String, ByteBuffer> allServiceResponse) {
- LOG_YARN.info("Started Container {} ", containerId);
- RoleInstance cinfo = appState.onNodeManagerContainerStarted(containerId);
- if (cinfo != null) {
- LOG_YARN.info("Deployed instance of role {} onto {}",
- cinfo.role, containerId);
- //trigger an async container status
- nmClientAsync.getContainerStatusAsync(containerId,
- cinfo.container.getNodeId());
- // push out a registration
- queue(new RegisterComponentInstance(containerId, cinfo,
- 0, TimeUnit.MILLISECONDS));
-
- } else {
- //this is a hypothetical path not seen. We react by warning
- log.error("Notified of started container that isn't pending {} - releasing",
- containerId);
- //then release it
- asyncRMClient.releaseAssignedContainer(containerId);
- }
- }
-
- @Override // NMClientAsync.CallbackHandler
- public void onStartContainerError(ContainerId containerId, Throwable t) {
- LOG_YARN.error("Failed to start Container {}", containerId, t);
- appState.onNodeManagerContainerStartFailed(containerId, t);
- }
-
- @Override // NMClientAsync.CallbackHandler
- public void onContainerStatusReceived(ContainerId containerId,
- ContainerStatus containerStatus) {
- LOG_YARN.debug("Container Status: id={}, status={}", containerId,
- containerStatus);
- RoleInstance cinfo = appState.getOwnedContainer(containerId);
- if (cinfo == null) {
- LOG_YARN.error("Owned container not found for {}", containerId);
- return;
- }
- ProviderService providerService = ProviderFactory
- .getProviderService(cinfo.providerRole.component.getArtifact());
-// if (providerService.processContainerStatus(containerId, containerStatus)) {
-// try {
-// Thread.sleep(1000);
-// } catch (InterruptedException e) {
-// }
-// LOG_YARN.info("Re-requesting status for role {}, {}",
-// cinfo.role, containerId);
-// //trigger another async container status
-// nmClientAsync.getContainerStatusAsync(containerId,
-// cinfo.container.getNodeId());
-// } else if (timelineServiceEnabled) {
-// RoleInstance instance = appState.getOwnedContainer(containerId);
-// if (instance != null) {
-// org.apache.slider.api.resource.Container container =
-// instance.providerRole.component
-// .getContainer(containerId.toString());
-// if (container != null) {
-// serviceTimelinePublisher.componentInstanceUpdated(container);
-// }
-// }
-// }
- }
-
- @Override // NMClientAsync.CallbackHandler
- public void onGetContainerStatusError(
- ContainerId containerId, Throwable t) {
- LOG_YARN.error("Failed to query the status of Container {}", containerId);
- }
-
- @Override // NMClientAsync.CallbackHandler
- public void onStopContainerError(ContainerId containerId, Throwable t) {
- LOG_YARN.warn("Failed to stop Container {}", containerId);
- }
-
- /**
- * Queue an action for immediate execution in the executor thread
- * @param action action to execute
- */
- public void queue(AsyncAction action) {
- actionQueues.put(action);
- }
-
- /**
- * Schedule an action
- * @param action for delayed execution
- */
- public void schedule(AsyncAction action) {
- actionQueues.schedule(action);
- }
-
-
- /**
- * Handle any exception in a thread. If the exception provides an exit
- * code, that is the one that will be used
- * @param thread thread throwing the exception
- * @param exception exception
- */
- public void onExceptionInThread(Thread thread, Throwable exception) {
- log.error("Exception in {}: {}", thread.getName(), exception, exception);
-
- // if there is a teardown in progress, ignore it
- if (amCompletionFlag.get()) {
- log.info("Ignoring exception: shutdown in progress");
- } else {
- int exitCode = EXIT_EXCEPTION_THROWN;
- if (exception instanceof ExitCodeProvider) {
- exitCode = ((ExitCodeProvider) exception).getExitCode();
- }
- signalAMComplete(new ActionStopSlider("stop",
- exitCode,
- FinalApplicationStatus.FAILED,
- SliderUtils.extractFirstLine(exception.getLocalizedMessage())));
- }
- }
-
- /**
- * TODO Read chaos monkey params from AM configuration rather than app
- * configuration
- * @return true if it started
- */
- private boolean maybeStartMonkey() {
- org.apache.slider.api.resource.Configuration configuration =
- application.getConfiguration();
- boolean enabled = configuration.getPropertyBool(
- InternalKeys.CHAOS_MONKEY_ENABLED,
- InternalKeys.DEFAULT_CHAOS_MONKEY_ENABLED);
- if (!enabled) {
- log.debug("Chaos monkey disabled");
- return false;
- }
-
- long monkeyInterval = SliderUtils.getTimeRange(configuration,
- InternalKeys.CHAOS_MONKEY_INTERVAL,
- InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS,
- InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS,
- InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES,
- 0);
- if (monkeyInterval == 0) {
- log.debug(
- "Chaos monkey not configured with a time interval...not enabling");
- return false;
- }
-
- long monkeyDelay = SliderUtils.getTimeRange(configuration,
- InternalKeys.CHAOS_MONKEY_DELAY,
- 0,
- 0,
- 0,
- (int)monkeyInterval);
-
- log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours -delay {}",
- monkeyInterval, monkeyInterval/(60*60), monkeyDelay);
- monkey = new ChaosMonkeyService(metrics, actionQueues);
- initAndAddService(monkey);
-
- // configure the targets
-
- // launch failure: special case with explicit failure triggered now
- int amLaunchFailProbability = configuration.getPropertyInt(
- InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE,
- 0);
- if (amLaunchFailProbability > 0 && monkey.chaosCheck(
- amLaunchFailProbability)) {
- log.info("Chaos Monkey has triggered AM Launch failure");
- // trigger a failure
- ActionStopSlider stop = new ActionStopSlider("stop",
- 0, TimeUnit.SECONDS,
- LauncherExitCodes.EXIT_FALSE,
- FinalApplicationStatus.FAILED,
- E_TRIGGERED_LAUNCH_FAILURE);
- queue(stop);
- }
-
- int amKillProbability = configuration.getPropertyInt(
- InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE,
- InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE);
- monkey.addTarget("AM killer",
- new ChaosKillAM(actionQueues, -1), amKillProbability);
- int containerKillProbability = configuration.getPropertyInt(
- InternalKeys.CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE,
- InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE);
- monkey.addTarget("Container killer",
- new ChaosKillContainer(appState, actionQueues, rmOperationHandler),
- containerKillProbability);
-
- // and schedule it
- if (monkey.schedule(monkeyDelay, monkeyInterval, TimeUnit.SECONDS)) {
- log.info("Chaos Monkey is running");
- return true;
- } else {
- log.info("Chaos monkey not started");
- return false;
- }
- }
-
- /**
- * This is the main entry point for the service launcher.
- * @param args command line arguments.
- */
- public static void main(String[] args) {
-
- //turn the args to a list
- List<String> argsList = Arrays.asList(args);
- //create a new list, as the ArrayList type doesn't push() on an insert
- List<String> extendedArgs = new ArrayList<String>(argsList);
- //insert the service name
- extendedArgs.add(0, SERVICE_CLASSNAME);
- //now have the service launcher do its work
- ServiceLauncher.serviceMain(extendedArgs);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java
deleted file mode 100644
index a7b94ed..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.slider.api.proto.Messages;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.concurrent.TimeUnit;
-
-public class ActionFlexCluster extends AsyncAction {
-
- final Messages.FlexComponentsRequestProto requestProto;
- public ActionFlexCluster(String name, long delay, TimeUnit timeUnit,
- Messages.FlexComponentsRequestProto requestProto) {
- super(name, delay, timeUnit, ATTR_CHANGES_APP_SIZE);
- this.requestProto = requestProto;
- }
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
- appMaster.flexCluster(requestProto);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
deleted file mode 100644
index ee1bb72..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Exit an emergency JVM halt.
- * @see ExitUtil#halt(int, String)
- */
-public class ActionHalt extends AsyncAction {
-
- private final int status;
- private final String text;
-
- public ActionHalt(
- int status,
- String text) {
- this(status, text, 0, TimeUnit.MILLISECONDS);
- }
-
- public ActionHalt(
- int status,
- String text,
- long delay, TimeUnit timeUnit) {
-
- // do not declare that this action halts the cluster ... keep it a surprise
- super("Halt", delay, timeUnit);
- this.status = status;
- this.text = text;
- }
-
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
- ExitUtil.halt(status, text);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
new file mode 100644
index 0000000..b57225a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * The current status of a submitted application, returned as a response to the
+ * GET API.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "The current status of a submitted application, returned as a response to the GET API.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ApplicationStatus extends BaseResource {
+ private static final long serialVersionUID = -3469885905347851034L;
+
+ private String diagnostics = null;
+ private ApplicationState state = null;
+ private Integer code = null;
+
+ /**
+ * Diagnostic information (if any) for the reason of the current state of the
+ * application. It typically has a non-null value, if the application is in a
+ * non-running state.
+ **/
+ public ApplicationStatus diagnostics(String diagnostics) {
+ this.diagnostics = diagnostics;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Diagnostic information (if any) for the reason of the current state of the application. It typically has a non-null value, if the application is in a non-running state.")
+ @JsonProperty("diagnostics")
+ public String getDiagnostics() {
+ return diagnostics;
+ }
+
+ public void setDiagnostics(String diagnostics) {
+ this.diagnostics = diagnostics;
+ }
+
+ /**
+ * Application state.
+ **/
+ public ApplicationStatus state(ApplicationState state) {
+ this.state = state;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Application state.")
+ @JsonProperty("state")
+ public ApplicationState getState() {
+ return state;
+ }
+
+ public void setState(ApplicationState state) {
+ this.state = state;
+ }
+
+ /**
+ * An error code specific to a scenario which app owners should be able to use
+ * to understand the failure in addition to the diagnostic information.
+ **/
+ public ApplicationStatus code(Integer code) {
+ this.code = code;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "An error code specific to a scenario which app owners should be able to use to understand the failure in addition to the diagnostic information.")
+ @JsonProperty("code")
+ public Integer getCode() {
+ return code;
+ }
+
+ public void setCode(Integer code) {
+ this.code = code;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ApplicationStatus applicationStatus = (ApplicationStatus) o;
+ return Objects.equals(this.diagnostics, applicationStatus.diagnostics)
+ && Objects.equals(this.state, applicationStatus.state)
+ && Objects.equals(this.code, applicationStatus.code);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(diagnostics, state, code);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class ApplicationStatus {\n");
+
+ sb.append(" diagnostics: ").append(toIndentedString(diagnostics))
+ .append("\n");
+ sb.append(" state: ").append(toIndentedString(state)).append("\n");
+ sb.append(" code: ").append(toIndentedString(code)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
new file mode 100644
index 0000000..0ddc374
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Artifact of an application component.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "Artifact of an application component")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Artifact implements Serializable {
+ private static final long serialVersionUID = 3608929500111099035L;
+
+ private String id = null;
+
+ public enum TypeEnum {
+ DOCKER("DOCKER"), TARBALL("TARBALL"), APPLICATION("APPLICATION");
+
+ private String value;
+
+ TypeEnum(String value) {
+ this.value = value;
+ }
+
+ @Override
+ @JsonValue
+ public String toString() {
+ return value;
+ }
+ }
+
+ private TypeEnum type = TypeEnum.DOCKER;
+ private String uri = null;
+
+ /**
+ * Artifact id. Examples are package location uri for tarball based apps,
+ * image name for docker, etc.
+ **/
+ public Artifact id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", required = true, value = "Artifact id. Examples are package location uri for tarball based apps, image name for docker, etc.")
+ @JsonProperty("id")
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
+ * Artifact type, like docker, tarball, etc. (optional).
+ **/
+ public Artifact type(TypeEnum type) {
+ this.type = type;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact type, like docker, tarball, etc. (optional).")
+ @JsonProperty("type")
+ public TypeEnum getType() {
+ return type;
+ }
+
+ public void setType(TypeEnum type) {
+ this.type = type;
+ }
+
+ /**
+ * Artifact location to support multiple artifact stores (optional).
+ **/
+ public Artifact uri(String uri) {
+ this.uri = uri;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact location to support multiple artifact stores (optional).")
+ @JsonProperty("uri")
+ public String getUri() {
+ return uri;
+ }
+
+ public void setUri(String uri) {
+ this.uri = uri;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Artifact artifact = (Artifact) o;
+ return Objects.equals(this.id, artifact.id)
+ && Objects.equals(this.type, artifact.type)
+ && Objects.equals(this.uri, artifact.uri);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, type, uri);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Artifact {\n");
+
+ sb.append(" id: ").append(toIndentedString(id)).append("\n");
+ sb.append(" type: ").append(toIndentedString(type)).append("\n");
+ sb.append(" uri: ").append(toIndentedString(uri)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
new file mode 100644
index 0000000..a87c97f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.io.Serializable;
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class BaseResource implements Serializable {
+ private static final long serialVersionUID = 1492603053176889431L;
+
+ private String uri;
+
+ /**
+ * Resource location, e.g. \
+ * "/applications/helloworld/containers/container_e3751_1458061340047_0008_01_000002\
+ * "
+ **/
+ public String getUri() {
+ return uri;
+ }
+
+ public void setUri(String uri) {
+ this.uri = uri;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("BaseResource [uri=");
+ builder.append(uri);
+ builder.append("]");
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
new file mode 100644
index 0000000..633e862
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
@@ -0,0 +1,412 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * One or more components of the application. If the application is HBase say,
+ * then the component can be a simple role like master or regionserver. If the
+ * application is a complex business webapp then a component can be other
+ * applications say Kafka or Storm. Thereby it opens up the support for complex
+ * and nested applications.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "One or more components of the application. If the application is HBase say, then the component can be a simple role like master or regionserver. If the application is a complex business webapp then a component can be other applications say Kafka or Storm. Thereby it opens up the support for complex and nested applications.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Component implements Serializable {
+ private static final long serialVersionUID = -8430058381509087805L;
+
+ private String name = null;
+ private List<String> dependencies = new ArrayList<String>();
+ private ReadinessCheck readinessCheck = null;
+ private Artifact artifact = null;
+ private String launchCommand = null;
+ private Resource resource = null;
+ private Long numberOfContainers = null;
+ private Boolean runPrivilegedContainer = false;
+ private PlacementPolicy placementPolicy = null;
+ private Configuration configuration = new Configuration();
+ private List<String> quicklinks = new ArrayList<String>();
+ private List<Container> containers =
+ Collections.synchronizedList(new ArrayList<Container>());
+
+ /**
+ * Name of the application component (mandatory).
+ **/
+ public Component name(String name) {
+ this.name = name;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", required = true, value = "Name of the application component (mandatory).")
+ @JsonProperty("name")
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * An array of application components which should be in READY state (as
+ * defined by readiness check), before this component can be started. The
+ * dependencies across all components of an application should be represented
+ * as a DAG.
+ **/
+ public Component dependencies(List<String> dependencies) {
+ this.dependencies = dependencies;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "An array of application components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an application should be represented as a DAG.")
+ @JsonProperty("dependencies")
+ public List<String> getDependencies() {
+ return dependencies;
+ }
+
+ public void setDependencies(List<String> dependencies) {
+ this.dependencies = dependencies;
+ }
+
+ /**
+ * Readiness check for this app-component.
+ **/
+ public Component readinessCheck(ReadinessCheck readinessCheck) {
+ this.readinessCheck = readinessCheck;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Readiness check for this app-component.")
+ @JsonProperty("readiness_check")
+ public ReadinessCheck getReadinessCheck() {
+ return readinessCheck;
+ }
+
+ @XmlElement(name = "readiness_check")
+ public void setReadinessCheck(ReadinessCheck readinessCheck) {
+ this.readinessCheck = readinessCheck;
+ }
+
+ /**
+ * Artifact of the component (optional). If not specified, the application
+ * level global artifact takes effect.
+ **/
+ public Component artifact(Artifact artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact of the component (optional). If not specified, the application level global artifact takes effect.")
+ @JsonProperty("artifact")
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void setArtifact(Artifact artifact) {
+ this.artifact = artifact;
+ }
+
+ /**
+ * The custom launch command of this component (optional). When specified at
+ * the component level, it overrides the value specified at the global level
+ * (if any).
+ **/
+ public Component launchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The custom launch command of this component (optional). When specified at the component level, it overrides the value specified at the global level (if any).")
+ @JsonProperty("launch_command")
+ public String getLaunchCommand() {
+ return launchCommand;
+ }
+
+ @XmlElement(name = "launch_command")
+ public void setLaunchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ }
+
+ /**
+ * Resource of this component (optional). If not specified, the application
+ * level global resource takes effect.
+ **/
+ public Component resource(Resource resource) {
+ this.resource = resource;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Resource of this component (optional). If not specified, the application level global resource takes effect.")
+ @JsonProperty("resource")
+ public Resource getResource() {
+ return resource;
+ }
+
+ public void setResource(Resource resource) {
+ this.resource = resource;
+ }
+
+ /**
+ * Number of containers for this app-component (optional). If not specified,
+ * the application level global number_of_containers takes effect.
+ **/
+ public Component numberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Number of containers for this app-component (optional). If not specified, the application level global number_of_containers takes effect.")
+ @JsonProperty("number_of_containers")
+ public Long getNumberOfContainers() {
+ return numberOfContainers;
+ }
+
+ @XmlElement(name = "number_of_containers")
+ public void setNumberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ }
+
+ @ApiModelProperty(example = "null", value = "Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.")
+ @JsonProperty("containers")
+ public List<Container> getContainers() {
+ return containers;
+ }
+
+ public void setContainers(List<Container> containers) {
+ this.containers = containers;
+ }
+
+ public void addContainer(Container container) {
+ this.containers.add(container);
+ }
+
+ public void removeContainer(Container container) {
+ containers.remove(container);
+ }
+ public Container getContainer(String id) {
+ for (Container container : containers) {
+ if (container.getId().equals(id)) {
+ return container;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Run all containers of this component in privileged mode (YARN-4262).
+ **/
+ public Component runPrivilegedContainer(Boolean runPrivilegedContainer) {
+ this.runPrivilegedContainer = runPrivilegedContainer;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Run all containers of this component in privileged mode (YARN-4262).")
+ @JsonProperty("run_privileged_container")
+ public Boolean getRunPrivilegedContainer() {
+ return runPrivilegedContainer;
+ }
+
+ @XmlElement(name = "run_privileged_container")
+ public void setRunPrivilegedContainer(Boolean runPrivilegedContainer) {
+ this.runPrivilegedContainer = runPrivilegedContainer;
+ }
+
+ /**
+ * Advanced scheduling and placement policies for all containers of this
+ * component (optional). If not specified, the app level placement_policy
+ * takes effect. Refer to the description at the global level for more
+ * details.
+ **/
+ public Component placementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the app level placement_policy takes effect. Refer to the description at the global level for more details.")
+ @JsonProperty("placement_policy")
+ public PlacementPolicy getPlacementPolicy() {
+ return placementPolicy;
+ }
+
+ @XmlElement(name = "placement_policy")
+ public void setPlacementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ }
+
+ /**
+ * Config properties for this app-component.
+ **/
+ public Component configuration(Configuration configuration) {
+ this.configuration = configuration;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Config properties for this app-component.")
+ @JsonProperty("configuration")
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public void setConfiguration(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ /**
+ * A list of quicklink keys defined at the application level, and to be
+ * resolved by this component.
+ **/
+ public Component quicklinks(List<String> quicklinks) {
+ this.quicklinks = quicklinks;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A list of quicklink keys defined at the application level, and to be resolved by this component.")
+ @JsonProperty("quicklinks")
+ public List<String> getQuicklinks() {
+ return quicklinks;
+ }
+
+ public void setQuicklinks(List<String> quicklinks) {
+ this.quicklinks = quicklinks;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Component component = (Component) o;
+ return Objects.equals(this.name, component.name)
+ && Objects.equals(this.dependencies, component.dependencies)
+ && Objects.equals(this.readinessCheck, component.readinessCheck)
+ && Objects.equals(this.artifact, component.artifact)
+ && Objects.equals(this.launchCommand, component.launchCommand)
+ && Objects.equals(this.resource, component.resource)
+ && Objects.equals(this.numberOfContainers, component.numberOfContainers)
+ && Objects.equals(this.runPrivilegedContainer,
+ component.runPrivilegedContainer)
+ && Objects.equals(this.placementPolicy, component.placementPolicy)
+ && Objects.equals(this.configuration, component.configuration)
+ && Objects.equals(this.quicklinks, component.quicklinks);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, dependencies, readinessCheck, artifact,
+ launchCommand, resource, numberOfContainers,
+ runPrivilegedContainer, placementPolicy, configuration, quicklinks);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Component {\n");
+
+ sb.append(" name: ").append(toIndentedString(name)).append("\n");
+ sb.append(" dependencies: ").append(toIndentedString(dependencies))
+ .append("\n");
+ sb.append(" readinessCheck: ").append(toIndentedString(readinessCheck))
+ .append("\n");
+ sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
+ sb.append(" launchCommand: ").append(toIndentedString(launchCommand))
+ .append("\n");
+ sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
+ sb.append(" numberOfContainers: ")
+ .append(toIndentedString(numberOfContainers)).append("\n");
+ sb.append(" containers: ").append(toIndentedString(containers))
+ .append("\n");
+ sb.append(" runPrivilegedContainer: ")
+ .append(toIndentedString(runPrivilegedContainer)).append("\n");
+ sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy))
+ .append("\n");
+ sb.append(" configuration: ").append(toIndentedString(configuration))
+ .append("\n");
+ sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
+ .append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+
+ /**
+ * Merge from another component into this component without overwriting.
+ */
+ public void mergeFrom(Component that) {
+ if (this.getArtifact() == null) {
+ this.setArtifact(that.getArtifact());
+ }
+ if (this.getResource() == null) {
+ this.setResource(that.getResource());
+ }
+ if (this.getNumberOfContainers() == null) {
+ this.setNumberOfContainers(that.getNumberOfContainers());
+ }
+ if (this.getLaunchCommand() == null) {
+ this.setLaunchCommand(that.getLaunchCommand());
+ }
+ this.getConfiguration().mergeFrom(that.getConfiguration());
+ if (this.getQuicklinks() == null) {
+ this.setQuicklinks(that.getQuicklinks());
+ }
+ if (this.getRunPrivilegedContainer() == null) {
+ this.setRunPrivilegedContainer(that.getRunPrivilegedContainer());
+ }
+ if (this.getDependencies() == null) {
+ this.setDependencies(that.getDependencies());
+ }
+ if (this.getPlacementPolicy() == null) {
+ this.setPlacementPolicy(that.getPlacementPolicy());
+ }
+ if (this.getReadinessCheck() == null) {
+ this.setReadinessCheck(that.getReadinessCheck());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
new file mode 100644
index 0000000..2fb494e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * A config file that needs to be created and made available as a volume in an
+ * application component container.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "A config file that needs to be created and made available as a volume in an application component container.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ConfigFile implements Serializable {
+ private static final long serialVersionUID = -7009402089417704612L;
+
+ public enum TypeEnum {
+ XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
+ "TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),;
+
+ private String value;
+
+ TypeEnum(String value) {
+ this.value = value;
+ }
+
+ @Override
+ @JsonValue
+ public String toString() {
+ return value;
+ }
+ }
+
+ private TypeEnum type = null;
+ private String destFile = null;
+ private String srcFile = null;
+ private Map<String, String> props = new HashMap<>();
+
+ public ConfigFile copy() {
+ ConfigFile copy = new ConfigFile();
+ copy.setType(this.getType());
+ copy.setSrcFile(this.getSrcFile());
+ copy.setDestFile(this.getDestFile());
+ if (this.getProps() != null && !this.getProps().isEmpty()) {
+ copy.getProps().putAll(this.getProps());
+ }
+ return copy;
+ }
+
+ /**
+ * Config file in the standard format like xml, properties, json, yaml,
+ * template.
+ **/
+ public ConfigFile type(TypeEnum type) {
+ this.type = type;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Config file in the standard format like xml, properties, json, yaml, template.")
+ @JsonProperty("type")
+ public TypeEnum getType() {
+ return type;
+ }
+
+ public void setType(TypeEnum type) {
+ this.type = type;
+ }
+
+ /**
+ * The absolute path that this configuration file should be mounted as, in the
+ * application container.
+ **/
+ public ConfigFile destFile(String destFile) {
+ this.destFile = destFile;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The absolute path that this configuration file should be mounted as, in the application container.")
+ @JsonProperty("dest_file")
+ public String getDestFile() {
+ return destFile;
+ }
+
+ @XmlElement(name = "dest_file")
+ public void setDestFile(String destFile) {
+ this.destFile = destFile;
+ }
+
+ /**
+ * This provides the source location of the configuration file, the content
+ * of which is dumped to dest_file post property substitutions, in the format
+ * as specified in type. Typically the src_file would point to a source
+ * controlled network accessible file maintained by tools like puppet, chef,
+ * or hdfs etc. Currently, only hdfs is supported.
+ **/
+ public ConfigFile srcFile(String srcFile) {
+ this.srcFile = srcFile;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "This provides the source location of the configuration file, "
+ + "the content of which is dumped to dest_file post property substitutions, in the format as specified in type. "
+ + "Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.")
+ @JsonProperty("src_file")
+ public String getSrcFile() {
+ return srcFile;
+ }
+
+ @XmlElement(name = "src_file")
+ public void setSrcFile(String srcFile) {
+ this.srcFile = srcFile;
+ }
+
+ /**
+ A blob of key value pairs that will be dumped in the dest_file in the format
+ as specified in type. If src_file is specified, src_file content are dumped
+ in the dest_file and these properties will overwrite, if any, existing
+ properties in src_file or be added as new properties in src_file.
+ **/
+ public ConfigFile props(Map<String, String> props) {
+ this.props = props;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be dumped in the dest_file in the format as specified in type."
+ + " If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any,"
+ + " existing properties in src_file or be added as new properties in src_file.")
+ @JsonProperty("props")
+ public Map<String, String> getProps() {
+ return props;
+ }
+
+ public void setProps(Map<String, String> props) {
+ this.props = props;
+ }
+
+ public long getLong(String name, long defaultValue) {
+ if (name == null) {
+ return defaultValue;
+ }
+ String value = props.get(name.trim());
+ return Long.parseLong(value);
+ }
+
+ public boolean getBoolean(String name, boolean defaultValue) {
+ if (name == null) {
+ return defaultValue;
+ }
+ return Boolean.valueOf(props.get(name.trim()));
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ConfigFile configFile = (ConfigFile) o;
+ return Objects.equals(this.type, configFile.type)
+ && Objects.equals(this.destFile, configFile.destFile)
+ && Objects.equals(this.srcFile, configFile.srcFile);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(type, destFile, srcFile, props);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class ConfigFile {\n");
+
+ sb.append(" type: ").append(toIndentedString(type)).append("\n");
+ sb.append(" destFile: ").append(toIndentedString(destFile)).append("\n");
+ sb.append(" srcFile: ").append(toIndentedString(srcFile)).append("\n");
+ sb.append(" props: ").append(toIndentedString(props)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
new file mode 100644
index 0000000..e10305a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.util.Locale;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public enum ConfigFormat {
+
+ JSON("json"),
+ PROPERTIES("properties"),
+ XML("xml"),
+ HADOOP_XML("hadoop_xml"),
+ ENV("env"),
+ TEMPLATE("template"),
+ YAML("yaml"),
+ ;
+ ConfigFormat(String suffix) {
+ this.suffix = suffix;
+ }
+
+ private final String suffix;
+
+ public String getSuffix() {
+ return suffix;
+ }
+
+
+ @Override
+ public String toString() {
+ return suffix;
+ }
+
+ /**
+ * Get a matching format or null
+ * @param type
+ * @return the format
+ */
+ public static ConfigFormat resolve(String type) {
+ for (ConfigFormat format: values()) {
+ if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) {
+ return format;
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
new file mode 100644
index 0000000..0ac508b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Set of configuration properties that can be injected into the application
+ * components via envs, files and custom pluggable helper docker containers.
+ * Files of several standard formats like xml, properties, json, yaml and
+ * templates will be supported.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "Set of configuration properties that can be injected into the application components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Configuration implements Serializable {
+ private static final long serialVersionUID = -4330788704981074466L;
+
+ private Map<String, String> properties = new HashMap<String, String>();
+ private Map<String, String> env = new HashMap<String, String>();
+ private List<ConfigFile> files = new ArrayList<ConfigFile>();
+
+ /**
+ * A blob of key-value pairs of common application properties.
+ **/
+ public Configuration properties(Map<String, String> properties) {
+ this.properties = properties;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key-value pairs of common application properties.")
+ @JsonProperty("properties")
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Map<String, String> properties) {
+ this.properties = properties;
+ }
+
+ /**
+ * A blob of key-value pairs which will be appended to the default system
+ * properties and handed off to the application at start time. All placeholder
+ * references to properties will be substituted before injection.
+ **/
+ public Configuration env(Map<String, String> env) {
+ this.env = env;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key-value pairs which will be appended to the default system properties and handed off to the application at start time. All placeholder references to properties will be substituted before injection.")
+ @JsonProperty("env")
+ public Map<String, String> getEnv() {
+ return env;
+ }
+
+ public void setEnv(Map<String, String> env) {
+ this.env = env;
+ }
+
+ /**
+ * Array of list of files that needs to be created and made available as
+ * volumes in the application component containers.
+ **/
+ public Configuration files(List<ConfigFile> files) {
+ this.files = files;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Array of list of files that needs to be created and made available as volumes in the application component containers.")
+ @JsonProperty("files")
+ public List<ConfigFile> getFiles() {
+ return files;
+ }
+
+ public void setFiles(List<ConfigFile> files) {
+ this.files = files;
+ }
+
+ public long getPropertyLong(String name, long defaultValue) {
+ String value = getProperty(name);
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ }
+ return Long.parseLong(value);
+ }
+
+ public int getPropertyInt(String name, int defaultValue) {
+ String value = getProperty(name);
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ }
+ return Integer.parseInt(value);
+ }
+
+ public boolean getPropertyBool(String name, boolean defaultValue) {
+ String value = getProperty(name);
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ }
+ return Boolean.parseBoolean(value);
+ }
+
+ public String getProperty(String name, String defaultValue) {
+ String value = getProperty(name);
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ }
+ return value;
+ }
+
+ public void setProperty(String name, String value) {
+ properties.put(name, value);
+ }
+
+ public String getProperty(String name) {
+ return properties.get(name.trim());
+ }
+
+ public String getEnv(String name) {
+ return env.get(name.trim());
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Configuration configuration = (Configuration) o;
+ return Objects.equals(this.properties, configuration.properties)
+ && Objects.equals(this.env, configuration.env)
+ && Objects.equals(this.files, configuration.files);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(properties, env, files);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Configuration {\n");
+
+ sb.append(" properties: ").append(toIndentedString(properties))
+ .append("\n");
+ sb.append(" env: ").append(toIndentedString(env)).append("\n");
+ sb.append(" files: ").append(toIndentedString(files)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+
+ /**
+ * Merge all properties and envs from that configuration to this configration.
+ * For ConfigFiles, all properties and envs of that ConfigFile are merged into
+ * this ConfigFile.
+ */
+ public synchronized void mergeFrom(Configuration that) {
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
+ .getProperties());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
+
+ Map<String, ConfigFile> thatMap = new HashMap<>();
+ for (ConfigFile file : that.getFiles()) {
+ thatMap.put(file.getDestFile(), file.copy());
+ }
+ for (ConfigFile thisFile : files) {
+ if(thatMap.containsKey(thisFile.getDestFile())) {
+ ConfigFile thatFile = thatMap.get(thisFile.getDestFile());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProps(),
+ thatFile.getProps());
+ thatMap.remove(thisFile.getDestFile());
+ }
+ }
+ // add remaining new files from that Configration
+ for (ConfigFile thatFile : thatMap.values()) {
+ files.add(thatFile.copy());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
new file mode 100644
index 0000000..8b687bb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Date;
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * An instance of a running application container.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "An instance of a running application container")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Container extends BaseResource {
+ private static final long serialVersionUID = -8955788064529288L;
+
+ private String id = null;
+ private Date launchTime = null;
+ private String ip = null;
+ private String hostname = null;
+ private String bareHost = null;
+ private ContainerState state = null;
+ private String componentName = null;
+ private Resource resource = null;
+ private Artifact artifact = null;
+ private Boolean privilegedContainer = null;
+
+ /**
+ * Unique container id of a running application, e.g.
+ * container_e3751_1458061340047_0008_01_000002.
+ **/
+ public Container id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Unique container id of a running application, e.g. container_e3751_1458061340047_0008_01_000002.")
+ @JsonProperty("id")
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
+ * The time when the container was created, e.g. 2016-03-16T01:01:49.000Z.
+ * This will most likely be different from cluster launch time.
+ **/
+ public Container launchTime(Date launchTime) {
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.")
+ @JsonProperty("launch_time")
+ public Date getLaunchTime() {
+ return launchTime == null ? null : (Date) launchTime.clone();
+ }
+
+ @XmlElement(name = "launch_time")
+ public void setLaunchTime(Date launchTime) {
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
+ }
+
+ /**
+ * IP address of a running container, e.g. 172.31.42.141. The IP address and
+ * hostname attribute values are dependent on the cluster/docker network setup
+ * as per YARN-4007.
+ **/
+ public Container ip(String ip) {
+ this.ip = ip;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.")
+ @JsonProperty("ip")
+ public String getIp() {
+ return ip;
+ }
+
+ public void setIp(String ip) {
+ this.ip = ip;
+ }
+
+ /**
+ * Fully qualified hostname of a running container, e.g.
+ * ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and
+ * hostname attribute values are dependent on the cluster/docker network setup
+ * as per YARN-4007.
+ **/
+ public Container hostname(String hostname) {
+ this.hostname = hostname;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.")
+ @JsonProperty("hostname")
+ public String getHostname() {
+ return hostname;
+ }
+
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
+
+ /**
+ * The bare node or host in which the container is running, e.g.
+ * cn008.example.com.
+ **/
+ public Container bareHost(String bareHost) {
+ this.bareHost = bareHost;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The bare node or host in which the container is running, e.g. cn008.example.com.")
+ @JsonProperty("bare_host")
+ public String getBareHost() {
+ return bareHost;
+ }
+
+ @XmlElement(name = "bare_host")
+ public void setBareHost(String bareHost) {
+ this.bareHost = bareHost;
+ }
+
+ /**
+ * State of the container of an application.
+ **/
+ public Container state(ContainerState state) {
+ this.state = state;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "State of the container of an application.")
+ @JsonProperty("state")
+ public ContainerState getState() {
+ return state;
+ }
+
+ public void setState(ContainerState state) {
+ this.state = state;
+ }
+
+ /**
+ * Name of the component that this container instance belongs to.
+ **/
+ public Container componentName(String componentName) {
+ this.componentName = componentName;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Name of the component that this container instance belongs to.")
+ @JsonProperty("component_name")
+ public String getComponentName() {
+ return componentName;
+ }
+
+ @XmlElement(name = "component_name")
+ public void setComponentName(String componentName) {
+ this.componentName = componentName;
+ }
+
+ /**
+ * Resource used for this container.
+ **/
+ public Container resource(Resource resource) {
+ this.resource = resource;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Resource used for this container.")
+ @JsonProperty("resource")
+ public Resource getResource() {
+ return resource;
+ }
+
+ public void setResource(Resource resource) {
+ this.resource = resource;
+ }
+
+ /**
+ * Artifact used for this container.
+ **/
+ public Container artifact(Artifact artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact used for this container.")
+ @JsonProperty("artifact")
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void setArtifact(Artifact artifact) {
+ this.artifact = artifact;
+ }
+
+ /**
+ * Container running in privileged mode or not.
+ **/
+ public Container privilegedContainer(Boolean privilegedContainer) {
+ this.privilegedContainer = privilegedContainer;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Container running in privileged mode or not.")
+ @JsonProperty("privileged_container")
+ public Boolean getPrivilegedContainer() {
+ return privilegedContainer;
+ }
+
+ public void setPrivilegedContainer(Boolean privilegedContainer) {
+ this.privilegedContainer = privilegedContainer;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Container container = (Container) o;
+ return Objects.equals(this.id, container.id);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Container {\n");
+
+ sb.append(" id: ").append(toIndentedString(id)).append("\n");
+ sb.append(" launchTime: ").append(toIndentedString(launchTime))
+ .append("\n");
+ sb.append(" ip: ").append(toIndentedString(ip)).append("\n");
+ sb.append(" hostname: ").append(toIndentedString(hostname)).append("\n");
+ sb.append(" bareHost: ").append(toIndentedString(bareHost)).append("\n");
+ sb.append(" state: ").append(toIndentedString(state)).append("\n");
+ sb.append(" componentName: ").append(toIndentedString(componentName))
+ .append("\n");
+ sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
+ sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
+ sb.append(" privilegedContainer: ")
+ .append(toIndentedString(privilegedContainer)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
new file mode 100644
index 0000000..bf09ff2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * The current state of the container of an application.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public enum ContainerState {
+ RUNNING_BUT_UNREADY, READY, STOPPED
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
new file mode 100644
index 0000000..c64b1b5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class Error {
+
+ private Integer code = null;
+ private String message = null;
+ private String fields = null;
+
+ /**
+ **/
+ public Error code(Integer code) {
+ this.code = code;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "")
+ @JsonProperty("code")
+ public Integer getCode() {
+ return code;
+ }
+
+ public void setCode(Integer code) {
+ this.code = code;
+ }
+
+ /**
+ **/
+ public Error message(String message) {
+ this.message = message;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "")
+ @JsonProperty("message")
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ /**
+ **/
+ public Error fields(String fields) {
+ this.fields = fields;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "")
+ @JsonProperty("fields")
+ public String getFields() {
+ return fields;
+ }
+
+ public void setFields(String fields) {
+ this.fields = fields;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Error error = (Error) o;
+ return Objects.equals(this.code, error.code)
+ && Objects.equals(this.message, error.message)
+ && Objects.equals(this.fields, error.fields);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(code, message, fields);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Error {\n");
+
+ sb.append(" code: ").append(toIndentedString(code)).append("\n");
+ sb.append(" message: ").append(toIndentedString(message)).append("\n");
+ sb.append(" fields: ").append(toIndentedString(fields)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
new file mode 100644
index 0000000..7d1b889
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Placement policy of an instance of an application. This feature is in the
+ * works in YARN-4902.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "Placement policy of an instance of an application. This feature is in the works in YARN-4902.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class PlacementPolicy implements Serializable {
+ private static final long serialVersionUID = 4341110649551172231L;
+
+ private String label = null;
+
+ /**
+ * Assigns an app to a named partition of the cluster where the application
+ * desires to run (optional). If not specified all apps are submitted to a
+ * default label of the app owner. One or more labels can be setup for each
+ * application owner account with required constraints like no-preemption,
+ * sla-99999, preemption-ok, etc.
+ **/
+ public PlacementPolicy label(String label) {
+ this.label = label;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Assigns an app to a named partition of the cluster where the application desires to run (optional). If not specified all apps are submitted to a default label of the app owner. One or more labels can be setup for each application owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.")
+ @JsonProperty("label")
+ public String getLabel() {
+ return label;
+ }
+
+ public void setLabel(String label) {
+ this.label = label;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PlacementPolicy placementPolicy = (PlacementPolicy) o;
+ return Objects.equals(this.label, placementPolicy.label);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(label);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class PlacementPolicy {\n");
+
+ sb.append(" label: ").append(toIndentedString(label)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
new file mode 100644
index 0000000..eadbb48
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * A custom command or a pluggable helper container to determine the readiness
+ * of a container of a component. Readiness for every application is different.
+ * Hence the need for a simple interface, with scope to support advanced
+ * usecases.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every application is different. Hence the need for a simple interface, with scope to support advanced usecases.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class ReadinessCheck implements Serializable {
+ private static final long serialVersionUID = -3836839816887186801L;
+
+ public enum TypeEnum {
+ HTTP("HTTP"),
+ PORT("PORT");
+
+ private String value;
+
+ TypeEnum(String value) {
+ this.value = value;
+ }
+
+ @Override
+ @JsonValue
+ public String toString() {
+ return value;
+ }
+ }
+
+ private TypeEnum type = null;
+ private Map<String, String> props = new HashMap<String, String>();
+ private Artifact artifact = null;
+
+ /**
+ * E.g. HTTP (YARN will perform a simple REST call at a regular interval and
+ * expect a 204 No content).
+ **/
+ public ReadinessCheck type(TypeEnum type) {
+ this.type = type;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).")
+ @JsonProperty("type")
+ public TypeEnum getType() {
+ return type;
+ }
+
+ public void setType(TypeEnum type) {
+ this.type = type;
+ }
+
+ public ReadinessCheck props(Map<String, String> props) {
+ this.props = props;
+ return this;
+ }
+
+ public ReadinessCheck putPropsItem(String key, String propsItem) {
+ this.props.put(key, propsItem);
+ return this;
+ }
+
+ /**
+ * A blob of key value pairs that will be used to configure the check.
+ * @return props
+ **/
+ @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be used to configure the check.")
+ public Map<String, String> getProps() {
+ return props;
+ }
+
+ public void setProps(Map<String, String> props) {
+ this.props = props;
+ }
+
+ /**
+ * Artifact of the pluggable readiness check helper container (optional). If
+ * specified, this helper container typically hosts the http uri and
+ * encapsulates the complex scripts required to perform actual container
+ * readiness check. At the end it is expected to respond a 204 No content just
+ * like the simplified use case. This pluggable framework benefits application
+ * owners who can run applications without any packaging modifications. Note,
+ * artifacts of type docker only is supported for now.
+ **/
+ public ReadinessCheck artifact(Artifact artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits application owners who can run applications without any packaging modifications. Note, artifacts of type docker only is supported for now.")
+ @JsonProperty("artifact")
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void setArtifact(Artifact artifact) {
+ this.artifact = artifact;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ReadinessCheck readinessCheck = (ReadinessCheck) o;
+ return Objects.equals(this.type, readinessCheck.type) &&
+ Objects.equals(this.props, readinessCheck.props) &&
+ Objects.equals(this.artifact, readinessCheck.artifact);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(type, props, artifact);
+ }
+
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class ReadinessCheck {\n");
+
+ sb.append(" type: ").append(toIndentedString(type)).append("\n");
+ sb.append(" props: ").append(toIndentedString(props)).append("\n");
+ sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
new file mode 100644
index 0000000..bda79c9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Resource determines the amount of resources (vcores, memory, network, etc.)
+ * usable by a container. This field determines the resource to be applied for
+ * all the containers of a component or application. The resource specified at
+ * the app (or global) level can be overriden at the component level. Only one
+ * of profile OR cpu & memory are exepected. It raises a validation
+ * exception otherwise.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class Resource extends BaseResource implements Cloneable {
+ private static final long serialVersionUID = -6431667797380250037L;
+
+ private String profile = null;
+ private Integer cpus = 1;
+ private String memory = null;
+
+ /**
+ * Each resource profile has a unique id which is associated with a
+ * cluster-level predefined memory, cpus, etc.
+ **/
+ public Resource profile(String profile) {
+ this.profile = profile;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.")
+ @JsonProperty("profile")
+ public String getProfile() {
+ return profile;
+ }
+
+ public void setProfile(String profile) {
+ this.profile = profile;
+ }
+
+ /**
+ * Amount of vcores allocated to each container (optional but overrides cpus
+ * in profile if specified).
+ **/
+ public Resource cpus(Integer cpus) {
+ this.cpus = cpus;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).")
+ @JsonProperty("cpus")
+ public Integer getCpus() {
+ return cpus;
+ }
+
+ public void setCpus(Integer cpus) {
+ this.cpus = cpus;
+ }
+
+ /**
+ * Amount of memory allocated to each container (optional but overrides memory
+ * in profile if specified). Currently accepts only an integer value and
+ * default unit is in MB.
+ **/
+ public Resource memory(String memory) {
+ this.memory = memory;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.")
+ @JsonProperty("memory")
+ public String getMemory() {
+ return memory;
+ }
+
+ public void setMemory(String memory) {
+ this.memory = memory;
+ }
+
+ public long getMemoryMB() {
+ if (this.memory == null) {
+ return 0;
+ }
+ return Long.valueOf(memory);
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Resource resource = (Resource) o;
+ return Objects.equals(this.profile, resource.profile)
+ && Objects.equals(this.cpus, resource.cpus)
+ && Objects.equals(this.memory, resource.memory);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(profile, cpus, memory);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Resource {\n");
+
+ sb.append(" profile: ").append(toIndentedString(profile)).append("\n");
+ sb.append(" cpus: ").append(toIndentedString(cpus)).append("\n");
+ sb.append(" memory: ").append(toIndentedString(memory)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ return super.clone();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
index 5574ebd..c7421ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
@@ -20,8 +20,9 @@ package org.apache.hadoop.yarn.service.client;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.api.resource.Application;
+import org.apache.hadoop.yarn.service.api.records.Application;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,7 +33,7 @@ public class ServiceCLI {
LoggerFactory.getLogger(ServiceClient.class);
protected ServiceClient client;
- public int exec(ClientArgs args) throws Throwable {
+ int exec(ClientArgs args) throws Throwable {
if (StringUtils.isEmpty(args.getAction())) {
System.out.println(args.usage());
return -1;
@@ -55,7 +56,7 @@ public class ServiceCLI {
client.actionFlexByCLI(args);
break;
case ACTION_STOP:
- client.actionStop(args.getClusterName());
+ client.actionStop(args.getClusterName(), false);
break;
case ACTION_DESTROY: // Destroy can happen only if app is already stopped
client.actionDestroy(args.getClusterName());
@@ -90,7 +91,12 @@ public class ServiceCLI {
public static void main(String[] args) throws Throwable {
ClientArgs clientArgs = new ClientArgs(args);
- clientArgs.parse();
+ try {
+ clientArgs.parse();
+ } catch (BadCommandArgumentsException e) {
+ System.err.println(e.getMessage());
+ System.exit(-1);
+ }
ServiceCLI cli = new ServiceCLI();
int res = cli.exec(clientArgs);
System.exit(res);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java
deleted file mode 100644
index ece65ba..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history;
-
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockFactory;
-import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
-import org.apache.slider.server.appmaster.state.ContainerOutcome;
-import org.apache.slider.server.appmaster.state.NodeEntry;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.RoleHistory;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Testing finding nodes for new instances.
- *
- * This stresses the non-AA codepath
- */
-public class TestRoleHistoryFindNodesForNewInstances extends
- BaseMockAppStateTest {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestRoleHistoryFindNodesForNewInstances.class);
-
- public TestRoleHistoryFindNodesForNewInstances() throws BadConfigException {
- }
-
- @Override
- public String getTestName() {
- return "TestFindNodesForNewInstances";
- }
-
- private NodeInstance age1Active4;
- private NodeInstance age2Active2;
- private NodeInstance age3Active0;
- private NodeInstance age4Active1;
- private NodeInstance age2Active0;
-
- private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
-
- private RoleStatus roleStat;
- private RoleStatus roleStat2;
-
- @Override
- public void setup() throws Exception {
- super.setup();
-
- age1Active4 = nodeInstance(1, 4, 0, 0);
- age2Active2 = nodeInstance(2, 2, 0, 1);
- age3Active0 = nodeInstance(3, 0, 0, 0);
- age4Active1 = nodeInstance(4, 1, 0, 0);
- age2Active0 = nodeInstance(2, 0, 0, 0);
-
- roleHistory.insert(Arrays.asList(age2Active2, age2Active0, age4Active1,
- age1Active4, age3Active0));
- roleHistory.buildRecentNodeLists();
-
- roleStat = getRole0Status();
- roleStat2 = getRole2Status();
- }
-
- public List<NodeInstance> findNodes(int count) {
- return findNodes(count, roleStat);
- }
-
- public List<NodeInstance> findNodes(int count, RoleStatus roleStatus) {
- List <NodeInstance> found = new ArrayList<>();
- for (int i = 0; i < count; i++) {
- NodeInstance f = roleHistory.findRecentNodeForNewInstance(roleStatus);
- if (f != null) {
- found.add(f);
- }
- }
- return found;
- }
-
- //@Test
- public void testFind1NodeR0() throws Throwable {
- NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat);
- LOG.info("found: {}", found);
- assertTrue(Arrays.asList(age3Active0).contains(found));
- }
-
- //@Test
- public void testFind2NodeR0() throws Throwable {
- NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat);
- LOG.info("found: {}", found);
- assertTrue(Arrays.asList(age2Active0, age3Active0).contains(found));
- NodeInstance found2 = roleHistory.findRecentNodeForNewInstance(roleStat);
- LOG.info("found: {}", found2);
- assertTrue(Arrays.asList(age2Active0, age3Active0).contains(found2));
- assertNotEquals(found, found2);
- }
-
- //@Test
- public void testFind3NodeR0ReturnsNull() throws Throwable {
- assertEquals(2, findNodes(2).size());
- NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat);
- assertNull(found);
- }
-
- //@Test
- public void testFindNodesOneEntry() throws Throwable {
- List<NodeInstance> foundNodes = findNodes(4, roleStat2);
- assertEquals(0, foundNodes.size());
- }
-
- //@Test
- public void testFindNodesIndependent() throws Throwable {
- assertEquals(2, findNodes(2).size());
- roleHistory.dump();
- assertEquals(0, findNodes(3, roleStat2).size());
- }
-
- //@Test
- public void testFindNodesFallsBackWhenUsed() throws Throwable {
- // mark age2 and active 0 as busy, expect a null back
- age2Active0.get(getRole0Status().getKey()).onStartCompleted();
- assertNotEquals(0, age2Active0.getActiveRoleInstances(getRole0Status()
- .getKey()));
- age3Active0.get(getRole0Status().getKey()).onStartCompleted();
- assertNotEquals(0, age3Active0.getActiveRoleInstances(getRole0Status()
- .getKey()));
- NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat);
- if (found != null) {
- LOG.info(found.toFullString());
- }
- assertNull(found);
- }
- //@Test
- public void testFindNodesSkipsFailingNode() throws Throwable {
- // mark age2 and active 0 as busy, expect a null back
-
- NodeEntry entry0 = age2Active0.get(getRole0Status().getKey());
- entry0.containerCompleted(
- false,
- ContainerOutcome.Failed);
- assertTrue(entry0.getFailed() > 0);
- assertTrue(entry0.getFailedRecently() > 0);
- entry0.containerCompleted(
- false,
- ContainerOutcome.Failed);
- assertFalse(age2Active0.exceedsFailureThreshold(roleStat));
- // set failure to 1
- roleStat.getProviderRole().nodeFailureThreshold = 1;
- // threshold is now exceeded
- assertTrue(age2Active0.exceedsFailureThreshold(roleStat));
-
- // get the role & expect age3 to be picked up, even though it is older
- NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat);
- assertEquals(age3Active0, found);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.java
deleted file mode 100644
index 4d4cf62..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history;
-
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockFactory;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Unit test to verify the comparators sort as expected.
- */
-public class TestRoleHistoryNIComparators extends BaseMockAppStateTest {
-
- private NodeInstance age1Active4;
- private NodeInstance age2Active2;
- private NodeInstance age3Active0;
- private NodeInstance age4Active1;
- private NodeInstance empty = new NodeInstance("empty", MockFactory
- .ROLE_COUNT);
- private NodeInstance age6failing;
- private NodeInstance age1failing;
-
- private List<NodeInstance> nodes;
- private List<NodeInstance> nodesPlusEmpty;
- private List<NodeInstance> allnodes;
-
- private RoleStatus role0Status;
-
- @Override
- public void setup() throws Exception {
- super.setup();
-
- role0Status = getRole0Status();
-
- age1Active4 = nodeInstance(1001, 4, 0, 0);
- age2Active2 = nodeInstance(1002, 2, 0, 0);
- age3Active0 = nodeInstance(1003, 0, 0, 0);
- age4Active1 = nodeInstance(1004, 1, 0, 0);
- age6failing = nodeInstance(1006, 0, 0, 0);
- age1failing = nodeInstance(1001, 0, 0, 0);
-
- age6failing.get(role0Status.getKey()).setFailedRecently(2);
- age1failing.get(role0Status.getKey()).setFailedRecently(1);
-
- nodes = Arrays.asList(age2Active2, age4Active1, age1Active4, age3Active0);
- nodesPlusEmpty = Arrays.asList(age2Active2, age4Active1, age1Active4,
- age3Active0, empty);
- allnodes = Arrays.asList(age6failing, age2Active2, age4Active1,
- age1Active4, age3Active0, age1failing);
- }
-
- @Override
- public String getTestName() {
- return "TestNIComparators";
- }
-
- //@Test
- public void testPreferred() throws Throwable {
- Collections.sort(nodes, new NodeInstance.Preferred(role0Status.getKey()));
- assertListEquals(nodes, Arrays.asList(age4Active1, age3Active0,
- age2Active2, age1Active4));
- }
-
- /**
- * The preferred sort still includes failures; up to next phase in process
- * to handle that.
- * @throws Throwable
- */
- //@Test
- public void testPreferredWithFailures() throws Throwable {
- Collections.sort(allnodes, new NodeInstance.Preferred(role0Status
- .getKey()));
- assertEquals(allnodes.get(0), age6failing);
- assertEquals(allnodes.get(1), age4Active1);
- }
-
- //@Test
- public void testPreferredComparatorDowngradesFailures() throws Throwable {
- NodeInstance.Preferred preferred = new NodeInstance.Preferred(role0Status
- .getKey());
- assertEquals(-1, preferred.compare(age6failing, age1failing));
- assertEquals(1, preferred.compare(age1failing, age6failing));
- }
-
- //@Test
- public void testNewerThanNoRole() throws Throwable {
- Collections.sort(nodesPlusEmpty, new NodeInstance.Preferred(role0Status
- .getKey()));
- assertListEquals(nodesPlusEmpty, Arrays.asList(age4Active1, age3Active0,
- age2Active2, age1Active4, empty));
- }
-
- //@Test
- public void testMoreActiveThan() throws Throwable {
-
- Collections.sort(nodes, new NodeInstance.MoreActiveThan(role0Status
- .getKey()));
- assertListEquals(nodes, Arrays.asList(age1Active4, age2Active2,
- age4Active1, age3Active0));
- }
-
- //@Test
- public void testMoreActiveThanEmpty() throws Throwable {
-
- Collections.sort(nodesPlusEmpty, new NodeInstance.MoreActiveThan(
- role0Status.getKey()));
- assertListEquals(nodesPlusEmpty, Arrays.asList(age1Active4, age2Active2,
- age4Active1, age3Active0, empty));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
deleted file mode 100644
index c1fc28f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.providers.PlacementPolicy;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockAppState;
-import org.apache.slider.server.appmaster.model.mock.MockContainer;
-import org.apache.slider.server.appmaster.model.mock.MockNodeId;
-import org.apache.slider.server.appmaster.model.mock.MockPriority;
-import org.apache.slider.server.appmaster.model.mock.MockResource;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome;
-import org.apache.slider.server.appmaster.state.ContainerAllocationResults;
-import org.apache.slider.server.appmaster.state.ContainerPriority;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.OutstandingRequest;
-import org.apache.slider.server.appmaster.state.OutstandingRequestTracker;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Test outstanding request tracker.
- */
-public class TestRoleHistoryOutstandingRequestTracker extends
- BaseMockAppStateTest {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestRoleHistoryOutstandingRequestTracker.class);
-
- public static final String WORKERS_LABEL = "workers";
- private NodeInstance host1 = new NodeInstance("host1", 3);
- private NodeInstance host2 = new NodeInstance("host2", 3);
- private MockResource resource = factory.newResource(48, 1);
-
- private OutstandingRequestTracker tracker = new OutstandingRequestTracker();
-
- public static final String WORKER = "worker";
-
- @Override
- public Application buildApplication() {
- Application application = super.buildApplication();
- Component component = new Component().name("worker").numberOfContainers(0L);
- component.getConfiguration().setProperty(ResourceKeys.YARN_LABEL_EXPRESSION,
- WORKERS_LABEL);
- application.getComponents().add(component);
- return application;
- }
-
- //@Test
- public void testAddRetrieveEntry() throws Throwable {
- OutstandingRequest request = tracker.newRequest(host1, 0);
- assertEquals(tracker.lookupPlacedRequest(0, "host1"), request);
- assertEquals(tracker.removePlacedRequest(request), request);
- assertNull(tracker.lookupPlacedRequest(0, "host1"));
- }
-
- //@Test
- public void testAddCompleteEntry() throws Throwable {
- OutstandingRequest req1 = tracker.newRequest(host1, 0);
- req1.buildContainerRequest(resource, getRole0Status(), 0);
-
- tracker.newRequest(host2, 0).buildContainerRequest(resource,
- getRole0Status(), 0);
- tracker.newRequest(host1, 1).buildContainerRequest(resource,
- getRole0Status(), 0);
-
- ContainerAllocationResults allocation = tracker.onContainerAllocated(1,
- "host1", null);
- assertEquals(allocation.outcome, ContainerAllocationOutcome.Placed);
- assertTrue(allocation.operations.get(0) instanceof CancelSingleRequest);
-
- assertNull(tracker.lookupPlacedRequest(1, "host1"));
- assertNotNull(tracker.lookupPlacedRequest(0, "host1"));
- }
-
- //@Test
- public void testResetOpenRequests() throws Throwable {
- OutstandingRequest req1 = tracker.newRequest(null, 0);
- assertFalse(req1.isLocated());
- tracker.newRequest(host1, 0);
- List<OutstandingRequest> openRequests = tracker.listOpenRequests();
- assertEquals(1, openRequests.size());
- tracker.resetOutstandingRequests(0);
- assertTrue(tracker.listOpenRequests().isEmpty());
- assertTrue(tracker.listPlacedRequests().isEmpty());
- }
-
- //@Test
- public void testRemoveOpenRequestUnissued() throws Throwable {
- OutstandingRequest req1 = tracker.newRequest(null, 0);
- req1.buildContainerRequest(resource, getRole0Status(), 0);
- assertEquals(1, tracker.listOpenRequests().size());
- MockContainer c1 = factory.newContainer(null, new MockPriority(0));
- c1.setResource(resource);
-
- ContainerAllocationResults allocation =
- tracker.onContainerAllocated(0, "host1", c1);
- ContainerAllocationOutcome outcome = allocation.outcome;
- assertEquals(outcome, ContainerAllocationOutcome.Unallocated);
- assertTrue(allocation.operations.isEmpty());
- assertEquals(1, tracker.listOpenRequests().size());
- }
-
- //@Test
- public void testIssuedOpenRequest() throws Throwable {
- OutstandingRequest req1 = tracker.newRequest(null, 0);
- req1.buildContainerRequest(resource, getRole0Status(), 0);
- assertEquals(1, tracker.listOpenRequests().size());
-
- int pri = ContainerPriority.buildPriority(0, false);
- assertTrue(pri > 0);
- MockNodeId nodeId = factory.newNodeId("hostname-1");
- MockContainer c1 = factory.newContainer(nodeId, new MockPriority(pri));
-
- c1.setResource(resource);
-
- ContainerRequest issued = req1.getIssuedRequest();
- assertEquals(issued.getCapability(), resource);
- assertEquals(issued.getPriority().getPriority(), c1.getPriority()
- .getPriority());
- assertTrue(req1.resourceRequirementsMatch(resource));
-
- ContainerAllocationResults allocation =
- tracker.onContainerAllocated(0, nodeId.getHost(), c1);
- assertEquals(0, tracker.listOpenRequests().size());
- assertTrue(allocation.operations.get(0) instanceof CancelSingleRequest);
-
- assertEquals(allocation.outcome, ContainerAllocationOutcome.Open);
- assertEquals(allocation.origin, req1);
- }
-
- //@Test
- public void testResetEntries() throws Throwable {
- tracker.newRequest(host1, 0);
- tracker.newRequest(host2, 0);
- tracker.newRequest(host1, 1);
- List<NodeInstance> canceled = tracker.resetOutstandingRequests(0);
- assertEquals(2, canceled.size());
- assertTrue(canceled.contains(host1));
- assertTrue(canceled.contains(host2));
- assertNotNull(tracker.lookupPlacedRequest(1, "host1"));
- assertNull(tracker.lookupPlacedRequest(0, "host1"));
- canceled = tracker.resetOutstandingRequests(0);
- assertEquals(0, canceled.size());
- assertEquals(1, tracker.resetOutstandingRequests(1).size());
- }
-
- //@Test
- public void testEscalation() throws Throwable {
- // first request: default placement
- assertEquals(getRole0Status().getPlacementPolicy(), PlacementPolicy
- .DEFAULT);
- Resource res0 = newResource(getRole0Status());
- OutstandingRequest outstanding0 = tracker.newRequest(host1,
- getRole0Status().getKey());
- ContainerRequest initialRequest =
- outstanding0.buildContainerRequest(res0, getRole0Status(), 0);
- assertNotNull(outstanding0.getIssuedRequest());
- assertTrue(outstanding0.isLocated());
- assertFalse(outstanding0.isEscalated());
- assertFalse(initialRequest.getRelaxLocality());
- assertEquals(1, tracker.listPlacedRequests().size());
-
- // second. This one doesn't get launched. This is to verify that the
- // escalation process skips entries which are in the list but have not
- // been issued, which can be a race condition between request issuance &
- // escalation.
- // (not one observed outside test authoring, but retained for completeness)
- Resource res2 = newResource(getRole2Status());
- OutstandingRequest outstanding2 = tracker.newRequest(host1,
- getRole2Status().getKey());
-
- // simulate some time escalation of role 1 MUST now be triggered
- long interval = getRole0Status().getPlacementTimeoutSeconds() * 1000 + 500;
- long now = interval;
- final List<AbstractRMOperation> escalations = tracker
- .escalateOutstandingRequests(now);
-
- assertTrue(outstanding0.isEscalated());
- assertFalse(outstanding2.isEscalated());
-
- // two entries
- assertEquals(2, escalations.size());
- AbstractRMOperation e1 = escalations.get(0);
- assertTrue(e1 instanceof CancelSingleRequest);
- final CancelSingleRequest cancel = (CancelSingleRequest) e1;
- assertEquals(initialRequest, cancel.getRequest());
- AbstractRMOperation e2 = escalations.get(1);
- assertTrue(e2 instanceof ContainerRequestOperation);
- ContainerRequestOperation escRequest = (ContainerRequestOperation) e2;
- assertTrue(escRequest.getRequest().getRelaxLocality());
-
- // build that second request from an anti-affine entry
- // these get placed as well
- now += interval;
- ContainerRequest containerReq2 =
- outstanding2.buildContainerRequest(res2, getRole2Status(), now);
- // escalate a little bit more
- final List<AbstractRMOperation> escalations2 = tracker
- .escalateOutstandingRequests(now);
- // and expect no new entries
- assertEquals(0, escalations2.size());
-
- // go past the role2 timeout
- now += getRole2Status().getPlacementTimeoutSeconds() * 1000 + 500;
- // escalate a little bit more
- final List<AbstractRMOperation> escalations3 = tracker
- .escalateOutstandingRequests(now);
- // and expect another escalation
- assertEquals(2, escalations3.size());
- assertTrue(outstanding2.isEscalated());
-
- // finally add a strict entry to the mix
- Resource res3 = newResource(getRole1Status());
- OutstandingRequest outstanding3 = tracker.newRequest(host1,
- getRole1Status().getKey());
-
- final ProviderRole providerRole1 = getRole1Status().getProviderRole();
- assertEquals(providerRole1.placementPolicy, PlacementPolicy.STRICT);
- now += interval;
- assertFalse(outstanding3.mayEscalate());
- final List<AbstractRMOperation> escalations4 = tracker
- .escalateOutstandingRequests(now);
- assertTrue(escalations4.isEmpty());
-
- }
-
- /**
- * If the placement does include a label, the initial request must
- * <i>not</i> include it.
- * The escalation request will contain the label, while
- * leaving out the node list.
- * retains the node list, but sets relaxLocality==true
- * @throws Throwable
- */
- //@Test
- public void testRequestLabelledPlacement() throws Throwable {
- NodeInstance ni = new NodeInstance("host1", 0);
- OutstandingRequest req1 = tracker.newRequest(ni, 0);
- Resource res0 = factory.newResource(48, 1);
-
- RoleStatus workerRole = lookupRole(WORKER);
- // initial request
- ContainerRequest yarnRequest =
- req1.buildContainerRequest(res0, workerRole, 0);
- assertEquals(req1.label, WORKERS_LABEL);
-
- assertNull(yarnRequest.getNodeLabelExpression());
- assertFalse(yarnRequest.getRelaxLocality());
- // escalation
- ContainerRequest yarnRequest2 = req1.escalate();
- assertNull(yarnRequest2.getNodes());
- assertTrue(yarnRequest2.getRelaxLocality());
- assertEquals(yarnRequest2.getNodeLabelExpression(), WORKERS_LABEL);
- }
-
- /**
- * If the placement doesnt include a label, then the escalation request
- * retains the node list, but sets relaxLocality==true.
- * @throws Throwable
- */
- //@Test
- public void testRequestUnlabelledPlacement() throws Throwable {
- NodeInstance ni = new NodeInstance("host1", 0);
- OutstandingRequest req1 = tracker.newRequest(ni, 0);
- Resource res0 = factory.newResource(48, 1);
-
- // initial request
- ContainerRequest yarnRequest = req1.buildContainerRequest(res0,
- getRole0Status(), 0);
- assertNotNull(yarnRequest.getNodes());
- assertTrue(SliderUtils.isUnset(yarnRequest.getNodeLabelExpression()));
- assertFalse(yarnRequest.getRelaxLocality());
- ContainerRequest yarnRequest2 = req1.escalate();
- assertNotNull(yarnRequest2.getNodes());
- assertTrue(yarnRequest2.getRelaxLocality());
- }
-
- //@Test(expected = IllegalArgumentException.class)
- public void testAARequestNoNodes() throws Throwable {
- tracker.newAARequest(getRole0Status().getKey(), new ArrayList<>(), "");
- }
-
- //@Test
- public void testAARequest() throws Throwable {
- int role0 = getRole0Status().getKey();
- OutstandingRequest request = tracker.newAARequest(role0, Arrays
- .asList(host1), "");
- assertEquals(host1.hostname, request.hostname);
- assertFalse(request.isLocated());
- }
-
- //@Test
- public void testAARequestPair() throws Throwable {
- int role0 = getRole0Status().getKey();
- OutstandingRequest request = tracker.newAARequest(role0, Arrays.asList(
- host1, host2), "");
- assertEquals(host1.hostname, request.hostname);
- assertFalse(request.isLocated());
- ContainerRequest yarnRequest = request.buildContainerRequest(
- getRole0Status().copyResourceRequirements(new MockResource(0, 0)),
- getRole0Status(),
- 0);
- assertFalse(yarnRequest.getRelaxLocality());
- assertFalse(request.mayEscalate());
-
- assertEquals(2, yarnRequest.getNodes().size());
- }
-
- //@Test
- public void testBuildResourceRequirements() throws Throwable {
- // Store original values
- Application application = appState.getClusterStatus();
- Component role0 = application.getComponent(getRole0Status().getName());
- String origMem = role0.getResource().getMemory();
- Integer origVcores = role0.getResource().getCpus();
-
- // Resource values to be used for this test
- int testMem = 32768;
- int testVcores = 2;
- role0.resource(new org.apache.slider.api.resource.Resource().memory(Integer
- .toString(testMem)).cpus(testVcores));
-
- // Test normalization disabled
- LOG.info("Test normalization: disabled");
- role0.getConfiguration().setProperty(
- ResourceKeys.YARN_RESOURCE_NORMALIZATION_ENABLED, "false");
- MockResource requestedRes = new MockResource(testMem, testVcores);
- MockResource expectedRes = new MockResource(testMem, testVcores);
- LOG.info("Resource requested: {}", requestedRes);
- Resource resFinal = appState.buildResourceRequirements(getRole0Status());
- LOG.info("Resource actual: {}", resFinal);
- assertTrue(Resources.equals(expectedRes, resFinal));
-
- // Test normalization enabled
- LOG.info("Test normalization: enabled");
- role0.getConfiguration().setProperty(
- ResourceKeys.YARN_RESOURCE_NORMALIZATION_ENABLED, "true");
- expectedRes = new MockResource(MockAppState.RM_MAX_RAM, testVcores);
- LOG.info("Resource requested: {}", requestedRes);
- resFinal = appState.buildResourceRequirements(getRole0Status());
- LOG.info("Resource actual: {}", resFinal);
- assertTrue(Resources.equals(expectedRes, resFinal));
-
- // revert resource configuration to original value
- role0.resource(new org.apache.slider.api.resource.Resource().memory(origMem)
- .cpus(origVcores));
- }
-
- public Resource newResource(RoleStatus r) {
- return appState.buildResourceRequirements(r);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java
deleted file mode 100644
index e3770a5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java
+++ /dev/null
@@ -1,371 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.providers.PlacementPolicy;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockFactory;
-import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
-import org.apache.slider.server.appmaster.state.NodeEntry;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.RoleHistory;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.apache.slider.server.avro.LoadedRoleHistory;
-import org.apache.slider.server.avro.RoleHistoryWriter;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Test fole history reading and writing.
- */
-public class TestRoleHistoryRW extends BaseMockAppStateTest {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestRoleHistoryRW.class);
-
- private static long time = System.currentTimeMillis();
- public static final String HISTORY_V1_6_ROLE =
- "org/apache/slider/server/avro/history-v01-6-role.json";
- public static final String HISTORY_V1_3_ROLE =
- "org/apache/slider/server/avro/history-v01-3-role.json";
- public static final String HISTORY_V1B_1_ROLE =
- "org/apache/slider/server/avro/history_v01b_1_role.json";
-
- private RoleStatus role0Status;
- private RoleStatus role1Status;
-
- static final ProviderRole PROVIDER_ROLE3 = new ProviderRole(
- "role3",
- 3,
- PlacementPolicy.STRICT,
- 3,
- 3,
- ResourceKeys.DEF_YARN_LABEL_EXPRESSION);
-
- @Override
- public String getTestName() {
- return "TestHistoryRW";
- }
-
- @Override
- public void setup() throws Exception {
- super.setup();
- role0Status = getRole0Status();
- role1Status = getRole1Status();
- }
-
- //@Test
- public void testWriteReadEmpty() throws Throwable {
- RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
- roleHistory.onStart(fs, historyPath);
- Path history = roleHistory.saveHistory(time++);
- assertTrue(fs.getFileStatus(history).isFile());
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- historyWriter.read(fs, history);
- }
-
- //@Test
- public void testWriteReadData() throws Throwable {
- RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
- assertFalse(roleHistory.onStart(fs, historyPath));
- String addr = "localhost";
- NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr);
- NodeEntry ne1 = instance.getOrCreate(0);
- ne1.setLastUsed(0xf00d);
-
- Path history = roleHistory.saveHistory(time++);
- assertTrue(fs.getFileStatus(history).isFile());
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES);
-
-
- LoadedRoleHistory loadedRoleHistory = historyWriter.read(fs, history);
- assertTrue(0 < loadedRoleHistory.size());
- rh2.rebuild(loadedRoleHistory);
- NodeInstance ni2 = rh2.getExistingNodeInstance(addr);
- assertNotNull(ni2);
- NodeEntry ne2 = ni2.get(0);
- assertNotNull(ne2);
- assertEquals(ne2.getLastUsed(), ne1.getLastUsed());
- }
-
- //@Test
- public void testWriteReadActiveData() throws Throwable {
- RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
- roleHistory.onStart(fs, historyPath);
- String addr = "localhost";
- String addr2 = "rack1server5";
- NodeInstance localhost = roleHistory.getOrCreateNodeInstance(addr);
- NodeEntry orig1 = localhost.getOrCreate(role0Status.getKey());
- orig1.setLastUsed(0x10);
- NodeInstance rack1server5 = roleHistory.getOrCreateNodeInstance(addr2);
- NodeEntry orig2 = rack1server5.getOrCreate(role1Status.getKey());
- orig2.setLive(3);
- assertFalse(orig2.isAvailable());
- NodeEntry orig3 = localhost.getOrCreate(role1Status.getKey());
- orig3.setLastUsed(0x20);
- orig3.setLive(1);
- assertFalse(orig3.isAvailable());
- orig3.release();
- assertTrue(orig3.isAvailable());
- roleHistory.dump();
-
- long savetime = 0x0001000;
- Path history = roleHistory.saveHistory(savetime);
- assertTrue(fs.getFileStatus(history).isFile());
- describe("Loaded");
- LOG.info("testWriteReadActiveData in {}", history);
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES);
- LoadedRoleHistory loadedRoleHistory = historyWriter.read(fs, history);
- assertEquals(3, loadedRoleHistory.size());
- rh2.rebuild(loadedRoleHistory);
- rh2.dump();
-
- assertEquals(2, rh2.getClusterSize());
- NodeInstance ni2 = rh2.getExistingNodeInstance(addr);
- assertNotNull(ni2);
- NodeEntry loadedNE = ni2.get(role0Status.getKey());
- assertEquals(loadedNE.getLastUsed(), orig1.getLastUsed());
- NodeInstance ni2b = rh2.getExistingNodeInstance(addr2);
- assertNotNull(ni2b);
- NodeEntry loadedNE2 = ni2b.get(role1Status.getKey());
- assertNotNull(loadedNE2);
- assertEquals(loadedNE2.getLastUsed(), savetime);
- assertEquals(rh2.getThawedDataTime(), savetime);
-
- // now start it
- rh2.buildRecentNodeLists();
- describe("starting");
- rh2.dump();
- List<NodeInstance> available0 = rh2.cloneRecentNodeList(role0Status
- .getKey());
- assertEquals(1, available0.size());
-
- NodeInstance entry = available0.get(0);
- assertEquals(entry.hostname, "localhost");
- assertEquals(entry, localhost);
- List<NodeInstance> available1 = rh2.cloneRecentNodeList(role1Status
- .getKey());
- assertEquals(2, available1.size());
- //and verify that even if last used was set, the save time is picked up
- assertEquals(entry.get(role1Status.getKey()).getLastUsed(), roleHistory
- .getSaveTime());
-
- }
-
- //@Test
- public void testWriteThaw() throws Throwable {
- RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
- assertFalse(roleHistory.onStart(fs, historyPath));
- String addr = "localhost";
- NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr);
- NodeEntry ne1 = instance.getOrCreate(0);
- ne1.setLastUsed(0xf00d);
-
- Path history = roleHistory.saveHistory(time++);
- long savetime =roleHistory.getSaveTime();
- assertTrue(fs.getFileStatus(history).isFile());
- RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES);
- assertTrue(rh2.onStart(fs, historyPath));
- NodeInstance ni2 = rh2.getExistingNodeInstance(addr);
- assertNotNull(ni2);
- NodeEntry ne2 = ni2.get(0);
- assertNotNull(ne2);
- assertEquals(ne2.getLastUsed(), ne1.getLastUsed());
- assertEquals(rh2.getThawedDataTime(), savetime);
- }
-
-
- //@Test
- public void testPurgeOlderEntries() throws Throwable {
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- time = 1;
- Path file1 = touch(historyWriter, time++);
- Path file2 = touch(historyWriter, time++);
- Path file3 = touch(historyWriter, time++);
- Path file4 = touch(historyWriter, time++);
- Path file5 = touch(historyWriter, time++);
- Path file6 = touch(historyWriter, time++);
-
- assertEquals(0, historyWriter.purgeOlderHistoryEntries(fs, file1));
- assertEquals(1, historyWriter.purgeOlderHistoryEntries(fs, file2));
- assertEquals(0, historyWriter.purgeOlderHistoryEntries(fs, file2));
- assertEquals(3, historyWriter.purgeOlderHistoryEntries(fs, file5));
- assertEquals(1, historyWriter.purgeOlderHistoryEntries(fs, file6));
- try {
- // make an impossible assertion that will fail if the method
- // actually completes
- assertEquals(-1, historyWriter.purgeOlderHistoryEntries(fs, file1));
- } catch (FileNotFoundException ignored) {
- // expected
- }
-
- }
-
- public Path touch(RoleHistoryWriter historyWriter, long timeMs)
- throws IOException {
- Path path = historyWriter.createHistoryFilename(historyPath, timeMs);
- FSDataOutputStream out = fs.create(path);
- out.close();
- return path;
- }
-
- //@Test
- public void testSkipEmptyFileOnRead() throws Throwable {
- describe("verify that empty histories are skipped on read; old histories " +
- "purged");
- RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
- roleHistory.onStart(fs, historyPath);
- time = 0;
- Path oldhistory = roleHistory.saveHistory(time++);
-
- String addr = "localhost";
- NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr);
- NodeEntry ne1 = instance.getOrCreate(0);
- ne1.setLastUsed(0xf00d);
-
- Path goodhistory = roleHistory.saveHistory(time++);
-
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- Path touched = touch(historyWriter, time++);
-
- RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES);
- assertTrue(rh2.onStart(fs, historyPath));
- NodeInstance ni2 = rh2.getExistingNodeInstance(addr);
- assertNotNull(ni2);
-
- //and assert the older file got purged
- assertFalse(fs.exists(oldhistory));
- assertTrue(fs.exists(goodhistory));
- assertTrue(fs.exists(touched));
- }
-
- //@Test
- public void testSkipBrokenFileOnRead() throws Throwable {
- describe("verify that empty histories are skipped on read; old histories " +
- "purged");
- RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
- roleHistory.onStart(fs, historyPath);
- time = 0;
- Path oldhistory = roleHistory.saveHistory(time++);
-
- String addr = "localhost";
- NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr);
- NodeEntry ne1 = instance.getOrCreate(0);
- ne1.setLastUsed(0xf00d);
-
- Path goodhistory = roleHistory.saveHistory(time++);
-
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- Path badfile = historyWriter.createHistoryFilename(historyPath, time++);
- FSDataOutputStream out = fs.create(badfile);
- out.writeBytes("{broken:true}");
- out.close();
-
- RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES);
- describe("IGNORE STACK TRACE BELOW");
-
- assertTrue(rh2.onStart(fs, historyPath));
-
- describe("IGNORE STACK TRACE ABOVE");
- NodeInstance ni2 = rh2.getExistingNodeInstance(addr);
- assertNotNull(ni2);
-
- //and assert the older file got purged
- assertFalse(fs.exists(oldhistory));
- assertTrue(fs.exists(goodhistory));
- assertTrue(fs.exists(badfile));
- }
-
- /**
- * Test that a v1 JSON file can be read. Here the number of roles
- * matches the current state.
- * @throws Throwable
- */
- //@Test
- public void testReloadDataV13Role() throws Throwable {
- String source = HISTORY_V1_3_ROLE;
- RoleHistoryWriter writer = new RoleHistoryWriter();
-
- LoadedRoleHistory loadedRoleHistory = writer.read(source);
- assertEquals(4, loadedRoleHistory.size());
- RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
- assertEquals(0, roleHistory.rebuild(loadedRoleHistory));
- }
-
- /**
- * Test that a v1 JSON file can be read. Here more roles than expected
- * @throws Throwable
- */
- //@Test
- public void testReloadDataV16Role() throws Throwable {
- String source = HISTORY_V1_6_ROLE;
- RoleHistoryWriter writer = new RoleHistoryWriter();
-
- LoadedRoleHistory loadedRoleHistory = writer.read(source);
- assertEquals(6, loadedRoleHistory.size());
- RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
- assertEquals(3, roleHistory.rebuild(loadedRoleHistory));
- }
-
- /**
- * Test that a v1 JSON file can be read. Here the number of roles
- * is less than the current state.
- * @throws Throwable
- */
- //@Test
- public void testReloadLessRoles() throws Throwable {
- String source = HISTORY_V1_3_ROLE;
- RoleHistoryWriter writer = new RoleHistoryWriter();
-
- LoadedRoleHistory loadedRoleHistory = writer.read(source);
- assertEquals(4, loadedRoleHistory.size());
- List<ProviderRole> expandedRoles = new ArrayList(MockFactory.ROLES);
- expandedRoles.add(PROVIDER_ROLE3);
- RoleHistory roleHistory = new MockRoleHistory(expandedRoles);
- assertEquals(0, roleHistory.rebuild(loadedRoleHistory));
- }
-
- /**
- * Test that a v1b JSON file can be read. Here more roles than expected
- * @throws Throwable
- */
- //@Test
- public void testReloadDataV1B1Role() throws Throwable {
- String source = HISTORY_V1B_1_ROLE;
- RoleHistoryWriter writer = new RoleHistoryWriter();
-
- LoadedRoleHistory loadedRoleHistory = writer.read(source);
- assertEquals(1, loadedRoleHistory.size());
- assertEquals(2, loadedRoleHistory.roleMap.size());
- RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
- assertEquals(0, roleHistory.rebuild(loadedRoleHistory));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java
deleted file mode 100644
index 033b509..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockFactory;
-import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
-import org.apache.slider.server.appmaster.state.NodeEntry;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.RoleHistory;
-import org.apache.slider.server.avro.NewerFilesFirst;
-import org.apache.slider.server.avro.RoleHistoryWriter;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Test role history rw ordering.
- */
-public class TestRoleHistoryRWOrdering extends BaseMockAppStateTest {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestRoleHistoryRWOrdering.class);
-
- private List<Path> paths = pathlist(
- Arrays.asList(
- "hdfs://localhost/history-0406c.json",
- "hdfs://localhost/history-5fffa.json",
- "hdfs://localhost/history-0001a.json",
- "hdfs://localhost/history-0001f.json"
- )
- );
- private Path h0406c = paths.get(0);
- private Path h5fffa = paths.get(1);
- private Path h0001a = paths.get(3);
-
- public TestRoleHistoryRWOrdering() throws URISyntaxException {
- }
-
- List<Path> pathlist(List<String> pathnames) throws URISyntaxException {
- List<Path> pathList = new ArrayList<>();
- for (String p : pathnames) {
- pathList.add(new Path(new URI(p)));
- }
- return pathList;
- }
-
- @Override
- public String getTestName() {
- return "TestHistoryRWOrdering";
- }
-
- /**
- * This tests regexp pattern matching. It uses the current time so isn't
- * repeatable -but it does test a wider range of values in the process
- * @throws Throwable
- */
- //@Test
- public void testPatternRoundTrip() throws Throwable {
- describe("test pattern matching of names");
- long value=System.currentTimeMillis();
- String name = String.format(SliderKeys.HISTORY_FILENAME_CREATION_PATTERN,
- value);
- String matchpattern = SliderKeys.HISTORY_FILENAME_MATCH_PATTERN;
- Pattern pattern = Pattern.compile(matchpattern);
- Matcher matcher = pattern.matcher(name);
- if (!matcher.find()) {
- throw new Exception("No match for pattern $matchpattern in $name");
- }
- }
-
- //@Test
- public void testWriteSequenceReadData() throws Throwable {
- describe("test that if multiple entries are written, the newest is picked" +
- " up");
- long time = System.currentTimeMillis();
-
- RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
- assertFalse(roleHistory.onStart(fs, historyPath));
- String addr = "localhost";
- NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr);
- NodeEntry ne1 = instance.getOrCreate(0);
- ne1.setLastUsed(0xf00d);
-
- Path history1 = roleHistory.saveHistory(time++);
- Path history2 = roleHistory.saveHistory(time++);
- Path history3 = roleHistory.saveHistory(time);
-
- //inject a later file with a different name
- sliderFileSystem.cat(new Path(historyPath, "file.json"), true, "hello," +
- " world");
-
-
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
-
- List<Path> entries = historyWriter.findAllHistoryEntries(
- fs,
- historyPath,
- false);
- assertEquals(entries.size(), 3);
- assertEquals(entries.get(0), history3);
- assertEquals(entries.get(1), history2);
- assertEquals(entries.get(2), history1);
- }
-
- //@Test
- public void testPathStructure() throws Throwable {
- assertEquals(h5fffa.getName(), "history-5fffa.json");
- }
-
- //@Test
- public void testPathnameComparator() throws Throwable {
-
- NewerFilesFirst newerName = new NewerFilesFirst();
-
- LOG.info("{} name is {}", h5fffa, h5fffa.getName());
- LOG.info("{} name is {}", h0406c, h0406c.getName());
- assertEquals(newerName.compare(h5fffa, h5fffa), 0);
- assertTrue(newerName.compare(h5fffa, h0406c) < 0);
- assertTrue(newerName.compare(h5fffa, h0001a) < 0);
- assertTrue(newerName.compare(h0001a, h5fffa) > 0);
-
- }
-
- //@Test
- public void testPathSort() throws Throwable {
- List<Path> paths2 = new ArrayList<>(paths);
- RoleHistoryWriter.sortHistoryPaths(paths2);
- assertListEquals(paths2,
- Arrays.asList(
- paths.get(1),
- paths.get(0),
- paths.get(3),
- paths.get(2)
- ));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java
deleted file mode 100644
index b84689c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java
+++ /dev/null
@@ -1,298 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.providers.PlacementPolicy;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockContainer;
-import org.apache.slider.server.appmaster.model.mock.MockFactory;
-import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
-import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome;
-import org.apache.slider.server.appmaster.state.NodeEntry;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.OutstandingRequest;
-import org.apache.slider.server.appmaster.state.RoleHistory;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Test the RH availability list and request tracking: that hosts
- * get removed and added.
- */
-public class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestRoleHistoryRequestTracking.class);
-
- private String roleName = "test";
-
- private NodeInstance age1Active4;
- private NodeInstance age2Active2;
- private NodeInstance age2Active0;
- private NodeInstance age3Active0;
- private NodeInstance age4Active1;
-
- private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
- // 1MB, 1 vcore
- private Resource resource = Resource.newInstance(1, 1);
-
- private RoleStatus roleStatus;
-
- public TestRoleHistoryRequestTracking() throws BadConfigException {
- }
-
- AMRMClient.ContainerRequest requestContainer(RoleStatus rs) {
- return roleHistory.requestContainerForRole(rs).getIssuedRequest();
- }
-
- @Override
- public String getTestName() {
- return "TestRoleHistoryAvailableList";
- }
-
- @Override
- public void setup() throws Exception {
- super.setup();
-
- age1Active4 = nodeInstance(1, 4, 0, 0);
- age2Active2 = nodeInstance(2, 2, 0, 1);
- age2Active0 = nodeInstance(2, 0, 0, 0);
- age3Active0 = nodeInstance(3, 0, 0, 0);
- age4Active1 = nodeInstance(4, 1, 0, 0);
-
- roleHistory.insert(Arrays.asList(age2Active2, age2Active0, age4Active1,
- age1Active4, age3Active0));
- roleHistory.buildRecentNodeLists();
- roleStatus = getRole0Status();
- roleStatus.setResourceRequirements(Resource.newInstance(1, 1));
- }
-
- //@Test
- public void testAvailableListBuiltForRoles() throws Throwable {
- List<NodeInstance> available0 = roleHistory.cloneRecentNodeList(
- roleStatus.getKey());
- assertListEquals(Arrays.asList(age3Active0, age2Active0), available0);
- }
-
- //@Test
- public void testRequestedNodeOffList() throws Throwable {
- NodeInstance ni = roleHistory.findRecentNodeForNewInstance(roleStatus);
- assertEquals(age3Active0, ni);
- assertListEquals(Arrays.asList(age2Active0),
- roleHistory.cloneRecentNodeList(roleStatus.getKey()));
- roleHistory.requestInstanceOnNode(ni,
- roleStatus,
- resource
- );
- }
-
- //@Test
- public void testRequestedNodeOffListWithFailures() throws Throwable {
- assertFalse(roleHistory.cloneRecentNodeList(roleStatus.getKey()).isEmpty());
-
- NodeEntry age3role0 = recordAsFailed(age3Active0, roleStatus.getKey(), 4);
- assertTrue(age3Active0.isConsideredUnreliable(roleStatus.getKey(),
- roleStatus.getNodeFailureThreshold()));
- recordAsFailed(age2Active0, roleStatus.getKey(), 4);
- assertTrue(age2Active0.isConsideredUnreliable(roleStatus.getKey(),
- roleStatus.getNodeFailureThreshold()));
- // expect to get a null node back
- NodeInstance ni = roleHistory.findRecentNodeForNewInstance(roleStatus);
- assertNull(ni);
-
- // which is translated to a no-location request
- AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni,
- roleStatus,
- resource).getIssuedRequest();
-
- assertNull(req.getNodes());
-
- LOG.info("resetting failure count");
- age3role0.resetFailedRecently();
- roleHistory.dump();
- assertEquals(0, age3role0.getFailedRecently());
- assertFalse(age3Active0.isConsideredUnreliable(roleStatus.getKey(),
- roleStatus.getNodeFailureThreshold()));
- assertFalse(roleHistory.cloneRecentNodeList(roleStatus.getKey()).isEmpty());
- // looking for a node should now find one
- ni = roleHistory.findRecentNodeForNewInstance(roleStatus);
- assertEquals(ni, age3Active0);
- req = roleHistory.requestInstanceOnNode(ni, roleStatus, resource)
- .getIssuedRequest();
- assertEquals(1, req.getNodes().size());
- }
-
- /**
- * Verify that strict placement policies generate requests for nodes
- * irrespective of their failed status.
- * @throws Throwable
- */
- //@Test
- public void testStrictPlacementIgnoresFailures() throws Throwable {
-
- RoleStatus targetRole = getRole1Status();
- final ProviderRole providerRole1 = targetRole.getProviderRole();
- assertEquals(providerRole1.placementPolicy, PlacementPolicy.STRICT);
- int key1 = targetRole.getKey();
- int key0 = getRole0Status().getKey();
-
- List<NodeInstance> nodes0 = Arrays.asList(age1Active4, age2Active0,
- age2Active2, age3Active0, age4Active1);
- recordAllFailed(key0, 4, nodes0);
- recordAllFailed(key1, 4, nodes0);
-
- // trigger a list rebuild
- roleHistory.buildRecentNodeLists();
- List<NodeInstance> recentRole0 = roleHistory.cloneRecentNodeList(key0);
- assertTrue(recentRole0.indexOf(age3Active0) < recentRole0
- .indexOf(age2Active0));
-
- // the non-strict role has no suitable nodes
- assertNull(roleHistory.findRecentNodeForNewInstance(getRole0Status()));
-
-
- NodeInstance ni = roleHistory.findRecentNodeForNewInstance(targetRole);
- assertNotNull(ni);
-
- NodeInstance ni2 = roleHistory.findRecentNodeForNewInstance(targetRole);
- assertNotNull(ni2);
- assertNotEquals(ni, ni2);
- }
-
- //@Test
- public void testFindAndRequestNode() throws Throwable {
- AMRMClient.ContainerRequest req = requestContainer(roleStatus);
-
- assertEquals(age3Active0.hostname, req.getNodes().get(0));
- List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(roleStatus
- .getKey());
- assertListEquals(Arrays.asList(age2Active0), a2);
- }
-
- //@Test
- public void testRequestedNodeIntoReqList() throws Throwable {
- requestContainer(roleStatus);
- List<OutstandingRequest> requests = roleHistory.listPlacedRequests();
- assertEquals(1, requests.size());
- assertEquals(age3Active0.hostname, requests.get(0).hostname);
- }
-
- //@Test
- public void testCompletedRequestDropsNode() throws Throwable {
- AMRMClient.ContainerRequest req = requestContainer(roleStatus);
- List<OutstandingRequest> requests = roleHistory.listPlacedRequests();
- assertEquals(1, requests.size());
- String hostname = requests.get(0).hostname;
- assertEquals(age3Active0.hostname, hostname);
- assertEquals(hostname, req.getNodes().get(0));
- MockContainer container = factory.newContainer(req, hostname);
- assertOnContainerAllocated(container, 2, 1);
- assertNoOutstandingPlacedRequests();
- }
-
- public void assertOnContainerAllocated(Container c1, int p1, int p2) {
- assertNotEquals(ContainerAllocationOutcome.Open, roleHistory
- .onContainerAllocated(c1, p1, p2).outcome);
- }
-
- public void assertOnContainerAllocationOpen(Container c1, int p1, int p2) {
- assertEquals(ContainerAllocationOutcome.Open, roleHistory
- .onContainerAllocated(c1, p1, p2).outcome);
- }
-
- void assertNoOutstandingPlacedRequests() {
- assertTrue(roleHistory.listPlacedRequests().isEmpty());
- }
-
- public void assertOutstandingPlacedRequests(int i) {
- assertEquals(i, roleHistory.listPlacedRequests().size());
- }
-
- //@Test
- public void testTwoRequests() throws Throwable {
- AMRMClient.ContainerRequest req = requestContainer(roleStatus);
- AMRMClient.ContainerRequest req2 = requestContainer(roleStatus);
- List<OutstandingRequest> requests = roleHistory.listPlacedRequests();
- assertEquals(2, requests.size());
- MockContainer container = factory.newContainer(req, req.getNodes().get(0));
- assertOnContainerAllocated(container, 2, 1);
- assertOutstandingPlacedRequests(1);
- container = factory.newContainer(req2, req2.getNodes().get(0));
- assertOnContainerAllocated(container, 2, 2);
- assertNoOutstandingPlacedRequests();
- }
-
- //@Test
- public void testThreeRequestsOneUnsatisified() throws Throwable {
- AMRMClient.ContainerRequest req = requestContainer(roleStatus);
- AMRMClient.ContainerRequest req2 = requestContainer(roleStatus);
- AMRMClient.ContainerRequest req3 = requestContainer(roleStatus);
- List<OutstandingRequest> requests = roleHistory.listPlacedRequests();
- assertEquals(2, requests.size());
- MockContainer container = factory.newContainer(req, req.getNodes().get(0));
- assertOnContainerAllocated(container, 2, 1);
- assertOutstandingPlacedRequests(1);
-
- container = factory.newContainer(req3, "three");
- assertOnContainerAllocationOpen(container, 3, 2);
- assertOutstandingPlacedRequests(1);
-
- // the final allocation will trigger a cleanup
- container = factory.newContainer(req2, "four");
- // no node dropped
- assertEquals(ContainerAllocationOutcome.Unallocated,
- roleHistory.onContainerAllocated(container, 3, 3).outcome);
- // yet the list is now empty
- assertNoOutstandingPlacedRequests();
- roleHistory.listOpenRequests().isEmpty();
-
- // and the remainder goes onto the available list
- List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(roleStatus
- .getKey());
- assertListEquals(Arrays.asList(age2Active0), a2);
- }
-
- //@Test
- public void testThreeRequests() throws Throwable {
- AMRMClient.ContainerRequest req = requestContainer(roleStatus);
- AMRMClient.ContainerRequest req2 = requestContainer(roleStatus);
- AMRMClient.ContainerRequest req3 = requestContainer(roleStatus);
- assertOutstandingPlacedRequests(2);
- assertNull(req3.getNodes());
- MockContainer container = factory.newContainer(req, req.getNodes().get(0));
- assertOnContainerAllocated(container, 3, 1);
- assertOutstandingPlacedRequests(1);
- container = factory.newContainer(req2, req2.getNodes().get(0));
- assertOnContainerAllocated(container, 3, 2);
- assertNoOutstandingPlacedRequests();
- container = factory.newContainer(req3, "three");
- assertOnContainerAllocationOpen(container, 3, 3);
- assertNoOutstandingPlacedRequests();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java
deleted file mode 100644
index a271859..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history;
-
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.server.appmaster.actions.ResetFailureWindow;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockAM;
-import org.apache.slider.server.appmaster.model.mock.MockFactory;
-import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler;
-import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.RoleHistory;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-/**
- * Test updating blacklist.
- */
-public class TestRoleHistoryUpdateBlacklist extends BaseMockAppStateTest {
- private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
- private Collection<RoleStatus> roleStatuses;
- private RoleStatus roleStatus;
- private NodeInstance ni;
-
- public TestRoleHistoryUpdateBlacklist() throws BadConfigException {
- }
-
- @Override
- public String getTestName() {
- return "TestUpdateBlacklist";
- }
-
- @Override
- public void setup() throws Exception {
- super.setup();
- ni = nodeInstance(1, 0, 0, 0);
- roleHistory.insert(Arrays.asList(ni));
- roleHistory.buildRecentNodeLists();
- appState.setRoleHistory(roleHistory);
- roleStatus = getRole0Status();
- roleStatuses = Arrays.asList(roleStatus);
- }
-
- //@Test
- public void testUpdateBlacklist() {
- assertFalse(ni.isBlacklisted());
-
- // at threshold, blacklist is unmodified
- recordAsFailed(ni, roleStatus.getKey(), MockFactory.NODE_FAILURE_THRESHOLD);
- UpdateBlacklistOperation op = roleHistory.updateBlacklist(roleStatuses);
- assertNull(op);
- assertFalse(ni.isBlacklisted());
-
- // threshold is reached, node goes on blacklist
- recordAsFailed(ni, roleStatus.getKey(), 1);
- op = roleHistory.updateBlacklist(roleStatuses);
- assertNotNull(op);
- assertTrue(ni.isBlacklisted());
-
- // blacklist remains unmodified
- op = roleHistory.updateBlacklist(roleStatuses);
- assertNull(op);
- assertTrue(ni.isBlacklisted());
-
- // failure threshold reset, node goes off blacklist
- ni.resetFailedRecently();
- op = roleHistory.updateBlacklist(roleStatuses);
- assertNotNull(op);
- assertFalse(ni.isBlacklisted());
- }
-
- //@Test
- public void testBlacklistOperations()
- throws Exception {
- recordAsFailed(ni, roleStatus.getKey(), MockFactory
- .NODE_FAILURE_THRESHOLD + 1);
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- assertListLength(ops, 1);
- AbstractRMOperation op = ops.get(0);
- assertTrue(op instanceof UpdateBlacklistOperation);
- assertTrue(ni.isBlacklisted());
-
- MockRMOperationHandler handler = new MockRMOperationHandler();
- assertEquals(0, handler.getBlacklisted());
- handler.execute(ops);
- assertEquals(1, handler.getBlacklisted());
-
- ResetFailureWindow resetter = new ResetFailureWindow(handler);
- resetter.execute(new MockAM(), null, appState);
- assertEquals(0, handler.getBlacklisted());
- assertFalse(ni.isBlacklisted());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java
deleted file mode 100644
index 419f2fb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.slider.common.tools.SliderUtils;
-
-/**
- * Provides allocation services to a cluster -both random and placed.
- *
- * Important: container allocations need an app attempt ID put into the
- * container ID
- */
-public class Allocator {
-
- private final MockYarnCluster cluster;
- /**
- * Rolling index into the cluster used for the next "random" assignment.
- */
- private int rollingIndex = 0;
-
- Allocator(MockYarnCluster cluster) {
- this.cluster = cluster;
- }
-
- /**
- * Allocate a node using the list of nodes in the container as the
- * hints.
- * @param request request
- * @return the allocated container -or null for none
- */
- MockContainer allocate(AMRMClient.ContainerRequest request) {
- MockYarnCluster.MockYarnClusterNode node = null;
- MockYarnCluster.MockYarnClusterContainer allocated = null;
- if (SliderUtils.isNotEmpty(request.getNodes())) {
- for (String host : request.getNodes()) {
- node = cluster.lookup(host);
- allocated = node.allocate();
- if (allocated != null) {
- break;
- }
- }
- }
-
- if (allocated != null) {
- return createContainerRecord(request, allocated, node);
- } else {
- if (request.getRelaxLocality() || request.getNodes().isEmpty()) {
- // fallback to anywhere
- return allocateRandom(request);
- } else {
- //no match and locality can't be requested
- return null;
- }
- }
- }
-
- /**
- * Allocate a node without any positioning -use whatever policy this allocator
- * chooses.
- * @param request request
- * @return the allocated container -or null for none
- */
- MockContainer allocateRandom(AMRMClient.ContainerRequest request) {
- int start = rollingIndex;
- MockYarnCluster.MockYarnClusterNode node = cluster.nodeAt(rollingIndex);
- MockYarnCluster.MockYarnClusterContainer allocated = node.allocate();
- // if there is no space, try again -but stop when all the nodes
- // have failed
- while (allocated == null && start != nextIndex()) {
- node = cluster.nodeAt(rollingIndex);
- allocated = node.allocate();
- }
-
- //here the allocation is set, so create the response
- return createContainerRecord(request, allocated, node);
- }
-
- /**
- * Create a container record -if one was allocated.
- * @param allocated allocation -may be null
- * @param node node with the container
- * @return a container record, or null if there was no allocation
- */
- public MockContainer createContainerRecord(
- AMRMClient.ContainerRequest request,
- MockYarnCluster.MockYarnClusterContainer allocated,
- MockYarnCluster.MockYarnClusterNode node) {
- if (allocated == null) {
- // no space
- return null;
- }
- MockContainer container = new MockContainer();
- container.setId(new MockContainerId(allocated.getCid()));
- container.setNodeId(node.getNodeId());
- container.setNodeHttpAddress(node.httpAddress());
- container.setPriority(request.getPriority());
- container.setResource(request.getCapability());
- return container;
- }
-
- public int nextIndex() {
- rollingIndex = (rollingIndex + 1) % cluster.getClusterSize();
- return rollingIndex;
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[63/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
new file mode 100644
index 0000000..684f655
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.hadoop.yarn.service.api.records.ReadinessCheck;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Formatter;
+import java.util.Locale;
+
+/**
+ * Various utils to work with the monitor
+ */
+public final class MonitorUtils {
+ protected static final Logger LOG = LoggerFactory.getLogger(MonitorUtils
+ .class);
+
+ private MonitorUtils() {
+ }
+
+ public static String toPlural(int val) {
+ return val != 1 ? "s" : "";
+ }
+
+ /**
+ * Convert milliseconds to human time -the exact format is unspecified
+ * @param milliseconds a time in milliseconds
+ * @return a time that is converted to human intervals
+ */
+ public static String millisToHumanTime(long milliseconds) {
+ StringBuilder sb = new StringBuilder();
+ // Send all output to the Appendable object sb
+ Formatter formatter = new Formatter(sb, Locale.US);
+
+ long s = Math.abs(milliseconds / 1000);
+ long m = Math.abs(milliseconds % 1000);
+ if (milliseconds > 0) {
+ formatter.format("%d.%03ds", s, m);
+ } else if (milliseconds == 0) {
+ formatter.format("0");
+ } else {
+ formatter.format("-%d.%03ds", s, m);
+ }
+ return sb.toString();
+ }
+
+ public static Probe getProbe(ReadinessCheck readinessCheck) {
+ if (readinessCheck == null) {
+ return null;
+ }
+ if (readinessCheck.getType() == null) {
+ return null;
+ }
+ try {
+ switch (readinessCheck.getType()) {
+ case HTTP:
+ return HttpProbe.create(readinessCheck.getProps());
+ case PORT:
+ return PortProbe.create(readinessCheck.getProps());
+ default:
+ return null;
+ }
+ } catch (Throwable t) {
+ throw new IllegalArgumentException("Error creating readiness check " +
+ t);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
new file mode 100644
index 0000000..aba5859
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.util.Map;
+
+/**
+ * Probe for a port being open.
+ */
+public class PortProbe extends Probe {
+ protected static final Logger log = LoggerFactory.getLogger(PortProbe.class);
+ private final int port;
+ private final int timeout;
+
+ public PortProbe(int port, int timeout) {
+ super("Port probe of " + port + " for " + timeout + "ms", null);
+ this.port = port;
+ this.timeout = timeout;
+ }
+
+ public static PortProbe create(Map<String, String> props)
+ throws IOException {
+ int port = getPropertyInt(props, PORT_PROBE_PORT, null);
+
+ if (port >= 65536) {
+ throw new IOException(PORT_PROBE_PORT + " " + port + " is out of " +
+ "range");
+ }
+
+ int timeout = getPropertyInt(props, PORT_PROBE_CONNECT_TIMEOUT,
+ PORT_PROBE_CONNECT_TIMEOUT_DEFAULT);
+
+ return new PortProbe(port, timeout);
+ }
+
+ /**
+ * Try to connect to the (host,port); a failure to connect within
+ * the specified timeout is a failure.
+ * @param instance role instance
+ * @return the outcome
+ */
+ @Override
+ public ProbeStatus ping(ComponentInstance instance) {
+ ProbeStatus status = new ProbeStatus();
+
+ if (instance.getContainerStatus() == null || SliderUtils
+ .isEmpty(instance.getContainerStatus().getIPs())) {
+ status.fail(this, new IOException(
+ instance.getCompInstanceName() + ": IP is not available yet"));
+ return status;
+ }
+
+ String ip = instance.getContainerStatus().getIPs().get(0);
+ InetSocketAddress sockAddr = new InetSocketAddress(ip, port);
+ Socket socket = new Socket();
+ try {
+ if (log.isDebugEnabled()) {
+ log.debug(instance.getCompInstanceName() + ": Connecting " + sockAddr
+ .toString() + ", timeout=" + MonitorUtils
+ .millisToHumanTime(timeout));
+ }
+ socket.connect(sockAddr, timeout);
+ status.succeed(this);
+ } catch (Throwable e) {
+ String error =
+ instance.getCompInstanceName() + ": Probe " + sockAddr + " failed";
+ log.debug(error, e);
+ status.fail(this, new IOException(error, e));
+ } finally {
+ IOUtils.closeSocket(socket);
+ }
+ return status;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java
new file mode 100644
index 0000000..3237a2b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Base class of all probes.
+ */
+public abstract class Probe implements MonitorKeys {
+
+ protected final Configuration conf;
+ private String name;
+
+ /**
+ * Create a probe of a specific name
+ *
+ * @param name probe name
+ * @param conf configuration being stored.
+ */
+ public Probe(String name, Configuration conf) {
+ this.name = name;
+ this.conf = conf;
+ }
+
+
+ protected void setName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+
+ @Override
+ public String toString() {
+ return getName();
+ }
+
+ public static String getProperty(Map<String, String> props, String name,
+ String defaultValue) throws IOException {
+ String value = props.get(name);
+ if (StringUtils.isEmpty(value)) {
+ if (defaultValue == null) {
+ throw new IOException(name + " not specified");
+ }
+ return defaultValue;
+ }
+ return value;
+ }
+
+ public static int getPropertyInt(Map<String, String> props, String name,
+ Integer defaultValue) throws IOException {
+ String value = props.get(name);
+ if (StringUtils.isEmpty(value)) {
+ if (defaultValue == null) {
+ throw new IOException(name + " not specified");
+ }
+ return defaultValue;
+ }
+ return Integer.parseInt(value);
+ }
+
+ /**
+ * perform any prelaunch initialization
+ */
+ public void init() throws IOException {
+
+ }
+
+ /**
+ * Ping the endpoint. All exceptions must be caught and included in the
+ * (failure) status.
+ *
+ * @param instance instance to ping
+ * @return the status
+ */
+ public abstract ProbeStatus ping(ComponentInstance instance);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java
new file mode 100644
index 0000000..bc62dcd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * Status message of a probe. This is designed to be sent over the wire, though the exception
+ * Had better be unserializable at the far end if that is to work.
+ */
+public final class ProbeStatus implements Serializable {
+ private static final long serialVersionUID = 165468L;
+
+ private long timestamp;
+ private String timestampText;
+ private boolean success;
+ private boolean realOutcome;
+ private String message;
+ private Throwable thrown;
+ private transient Probe originator;
+
+ public ProbeStatus() {
+ }
+
+ public ProbeStatus(long timestamp, String message, Throwable thrown) {
+ this.success = false;
+ this.message = message;
+ this.thrown = thrown;
+ setTimestamp(timestamp);
+ }
+
+ public ProbeStatus(long timestamp, String message) {
+ this.success = true;
+ setTimestamp(timestamp);
+ this.message = message;
+ this.thrown = null;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ timestampText = new Date(timestamp).toString();
+ }
+
+ public boolean isSuccess() {
+ return success;
+ }
+
+ /**
+ * Set both the success and the real outcome bits to the same value
+ * @param success the new value
+ */
+ public void setSuccess(boolean success) {
+ this.success = success;
+ realOutcome = success;
+ }
+
+ public String getTimestampText() {
+ return timestampText;
+ }
+
+ public boolean getRealOutcome() {
+ return realOutcome;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ public Throwable getThrown() {
+ return thrown;
+ }
+
+ public void setThrown(Throwable thrown) {
+ this.thrown = thrown;
+ }
+
+ /**
+ * Get the probe that generated this result. May be null
+ * @return a possibly null reference to a probe
+ */
+ public Probe getOriginator() {
+ return originator;
+ }
+
+ /**
+ * The probe has succeeded -capture the current timestamp, set
+ * success to true, and record any other data needed.
+ * @param probe probe
+ */
+ public void succeed(Probe probe) {
+ finish(probe, true, probe.getName(), null);
+ }
+
+ /**
+ * A probe has failed either because the test returned false, or an exception
+ * was thrown. The {@link #success} field is set to false, any exception
+ * thrown is recorded.
+ * @param probe probe that failed
+ * @param thrown an exception that was thrown.
+ */
+ public void fail(Probe probe, Throwable thrown) {
+ finish(probe, false, "Failure in " + probe, thrown);
+ }
+
+ public void finish(Probe probe, boolean succeeded, String text, Throwable thrown) {
+ setTimestamp(System.currentTimeMillis());
+ setSuccess(succeeded);
+ originator = probe;
+ message = text;
+ this.thrown = thrown;
+ }
+
+ @Override
+ public String toString() {
+ LogEntryBuilder builder = new LogEntryBuilder("Probe Status");
+ builder.elt("time", timestampText)
+ .elt("outcome", (success ? "success" : "failure"));
+
+ if (success != realOutcome) {
+ builder.elt("originaloutcome", (realOutcome ? "success" : "failure"));
+ }
+ builder.elt("message", message);
+ if (thrown != null) {
+ builder.elt("exception", thrown);
+ }
+
+ return builder.toString();
+ }
+
+ /**
+ * Flip the success bit on while the real outcome bit is kept false
+ */
+ public void markAsSuccessful() {
+ success = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
new file mode 100644
index 0000000..0d11be2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public abstract class AbstractClientProvider {
+
+ public AbstractClientProvider() {
+ }
+
+ /**
+ * Generates a fixed format of application tags given one or more of
+ * application name, version and description. This allows subsequent query for
+ * an application with a name only, version only or description only or any
+ * combination of those as filters.
+ *
+ * @param appName name of the application
+ * @param appVersion version of the application
+ * @param appDescription brief description of the application
+ * @return
+ */
+ public static final Set<String> createApplicationTags(String appName,
+ String appVersion, String appDescription) {
+ Set<String> tags = new HashSet<>();
+ tags.add(SliderUtils.createNameTag(appName));
+ if (appVersion != null) {
+ tags.add(SliderUtils.createVersionTag(appVersion));
+ }
+ if (appDescription != null) {
+ tags.add(SliderUtils.createDescriptionTag(appDescription));
+ }
+ return tags;
+ }
+
+ /**
+ * Validate the artifact.
+ * @param artifact
+ */
+ public abstract void validateArtifact(Artifact artifact, FileSystem
+ fileSystem) throws IOException;
+
+ protected abstract void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException;
+
+ /**
+ * Validate the config files.
+ * @param configFiles config file list
+ * @param fs file system
+ */
+ public void validateConfigFiles(List<ConfigFile> configFiles,
+ FileSystem fs) throws IOException {
+ Set<String> destFileSet = new HashSet<>();
+
+ for (ConfigFile file : configFiles) {
+ if (file.getType() == null) {
+ throw new IllegalArgumentException("File type is empty");
+ }
+
+ if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
+ .isEmpty(file.getSrcFile())) {
+ throw new IllegalArgumentException(
+ "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE);
+
+ }
+ if (!StringUtils.isEmpty(file.getSrcFile())) {
+ Path p = new Path(file.getSrcFile());
+ if (!fs.exists(p)) {
+ throw new IllegalArgumentException(
+ "Src_file does not exist for config file: " + file
+ .getSrcFile());
+ }
+ }
+
+ if (StringUtils.isEmpty(file.getDestFile())) {
+ throw new IllegalArgumentException("Dest_file is empty.");
+ }
+
+ if (destFileSet.contains(file.getDestFile())) {
+ throw new IllegalArgumentException(
+ "Duplicated ConfigFile exists: " + file.getDestFile());
+ }
+ destFileSet.add(file.getDestFile());
+
+ java.nio.file.Path destPath = Paths.get(file.getDestFile());
+ if (!destPath.isAbsolute() && destPath.getNameCount() > 1) {
+ throw new IllegalArgumentException("Non-absolute dest_file has more " +
+ "than one path element");
+ }
+
+ // provider-specific validation
+ validateConfigFile(file, fs);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
new file mode 100644
index 0000000..8d607ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_RETRY_INTERVAL;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_RETRY_MAX;
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
+
+public abstract class AbstractProviderService implements ProviderService,
+ YarnServiceConstants {
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(AbstractProviderService.class);
+
+ public abstract void processArtifact(AbstractLauncher launcher,
+ ComponentInstance compInstance, SliderFileSystem fileSystem,
+ Service service)
+ throws IOException;
+
+ public void buildContainerLaunchContext(AbstractLauncher launcher,
+ Service service, ComponentInstance instance,
+ SliderFileSystem fileSystem, Configuration yarnConf)
+ throws IOException, SliderException {
+ Component component = instance.getComponent().getComponentSpec();;
+ processArtifact(launcher, instance, fileSystem, service);
+
+ ServiceContext context =
+ instance.getComponent().getScheduler().getContext();
+ // Generate tokens (key-value pair) for config substitution.
+ // Get pre-defined tokens
+ Map<String, String> globalTokens =
+ instance.getComponent().getScheduler().globalTokens;
+ Map<String, String> tokensForSubstitution = ProviderUtils
+ .initCompTokensForSubstitute(instance);
+ tokensForSubstitution.putAll(globalTokens);
+ // Set the environment variables in launcher
+ launcher.putEnv(SliderUtils
+ .buildEnvMap(component.getConfiguration(), tokensForSubstitution));
+ launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
+ launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ if (System.getenv(HADOOP_USER_NAME) != null) {
+ launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
+ }
+ launcher.setEnv("LANG", "en_US.UTF-8");
+ launcher.setEnv("LC_ALL", "en_US.UTF-8");
+ launcher.setEnv("LANGUAGE", "en_US.UTF-8");
+
+ for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
+ tokensForSubstitution.put($(entry.getKey()), entry.getValue());
+ }
+ //TODO add component host tokens?
+// ProviderUtils.addComponentHostTokens(tokensForSubstitution, amState);
+
+ // create config file on hdfs and add local resource
+ ProviderUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
+ component, tokensForSubstitution, instance, context);
+
+ // substitute launch command
+ String launchCommand = ProviderUtils
+ .substituteStrWithTokens(component.getLaunchCommand(),
+ tokensForSubstitution);
+ CommandLineBuilder operation = new CommandLineBuilder();
+ operation.add(launchCommand);
+ operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
+ launcher.addCommand(operation.build());
+
+ // By default retry forever every 30 seconds
+ launcher.setRetryContext(YarnServiceConf
+ .getInt(CONTAINER_RETRY_MAX, -1, service.getConfiguration(),
+ yarnConf), YarnServiceConf
+ .getInt(CONTAINER_RETRY_INTERVAL, 30000, service.getConfiguration(),
+ yarnConf));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
new file mode 100644
index 0000000..0f949e0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
+import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for factories.
+ */
+public abstract class ProviderFactory {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(ProviderFactory.class);
+
+ protected ProviderFactory() {}
+
+ public abstract AbstractClientProvider createClientProvider();
+
+ public abstract ProviderService createServerProvider();
+
+ public static synchronized ProviderService getProviderService(Artifact
+ artifact) {
+ return createServiceProviderFactory(artifact).createServerProvider();
+ }
+
+ public static synchronized AbstractClientProvider getClientProvider(Artifact
+ artifact) {
+ return createServiceProviderFactory(artifact).createClientProvider();
+ }
+
+ /**
+ * Create a provider for a specific service
+ * @param artifact artifact
+ * @return provider factory
+ */
+ public static synchronized ProviderFactory createServiceProviderFactory(
+ Artifact artifact) {
+ if (artifact == null || artifact.getType() == null) {
+ LOG.debug("Loading service provider type default");
+ return DefaultProviderFactory.getInstance();
+ }
+ LOG.debug("Loading service provider type {}", artifact.getType());
+ switch (artifact.getType()) {
+ // TODO add handling for custom types?
+ // TODO handle service
+ case DOCKER:
+ return DockerProviderFactory.getInstance();
+ case TARBALL:
+ return TarballProviderFactory.getInstance();
+ default:
+ throw new IllegalArgumentException(String.format("Resolution error, " +
+ "%s should not be passed to createServiceProviderFactory",
+ artifact.getType()));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
new file mode 100644
index 0000000..eb721b4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+
+import java.io.IOException;
+
+public interface ProviderService {
+
+ /**
+ * Set up the entire container launch context
+ */
+ void buildContainerLaunchContext(AbstractLauncher containerLauncher,
+ Service service, ComponentInstance instance,
+ SliderFileSystem sliderFileSystem, Configuration yarnConf)
+ throws IOException, SliderException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
new file mode 100644
index 0000000..ec0c2ca
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.utils.PublishedConfiguration;
+import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.*;
+
+/**
+ * This is a factoring out of methods handy for providers. It's bonded to a log
+ * at construction time.
+ */
+public class ProviderUtils implements YarnServiceConstants {
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(ProviderUtils.class);
+
+
+ /**
+ * Add oneself to the classpath. This does not work
+ * on minicluster test runs where the JAR is not built up.
+ * @param providerResources map of provider resources to add these entries to
+ * @param providerClass provider to add
+ * @param jarName name of the jar to use
+ * @param sliderFileSystem target filesystem
+ * @param tempPath path in the cluster FS for temp files
+ * @param libdir relative directory to place resources
+ * @param miniClusterTestRun true if minicluster is being used
+ * @return true if the class was found in a JAR
+ *
+ * @throws FileNotFoundException if the JAR was not found and this is NOT
+ * a mini cluster test run
+ * @throws IOException IO problems
+ * @throws SliderException any Slider problem
+ */
+ public static boolean addProviderJar(
+ Map<String, LocalResource> providerResources,
+ Class providerClass,
+ String jarName,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libdir,
+ boolean miniClusterTestRun) throws
+ IOException,
+ SliderException {
+ try {
+ SliderUtils.putJar(providerResources,
+ sliderFileSystem,
+ providerClass,
+ tempPath,
+ libdir,
+ jarName);
+ return true;
+ } catch (FileNotFoundException e) {
+ if (miniClusterTestRun) {
+ return false;
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Loads all dependency jars from the default path.
+ * @param providerResources map of provider resources to add these entries to
+ * @param sliderFileSystem target filesystem
+ * @param tempPath path in the cluster FS for temp files
+ * @param libDir relative directory to place resources
+ * @param libLocalSrcDir explicitly supplied local libs dir
+ * @throws IOException trouble copying to HDFS
+ * @throws SliderException trouble copying to HDFS
+ */
+ public static void addAllDependencyJars(
+ Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libDir,
+ String libLocalSrcDir)
+ throws IOException, SliderException {
+ if (SliderUtils.isSet(libLocalSrcDir)) {
+ File file = new File(libLocalSrcDir);
+ if (!file.exists() || !file.isDirectory()) {
+ throw new BadCommandArgumentsException(
+ "Supplied lib src dir %s is not valid", libLocalSrcDir);
+ }
+ }
+ SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
+ libDir, libLocalSrcDir);
+ }
+
+ public static String substituteStrWithTokens(String content,
+ Map<String, String> tokensForSubstitution) {
+ for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
+ content =
+ content.replaceAll(Pattern.quote(token.getKey()), token.getValue());
+ }
+ return content;
+ }
+
+ // configs will be substituted by corresponding env in tokenMap
+ public static void substituteMapWithTokens(Map<String, String> configs,
+ Map<String, String> tokenMap) {
+ for (Map.Entry<String, String> entry : configs.entrySet()) {
+ String value = entry.getValue();
+ if (tokenMap != null) {
+ for (Map.Entry<String, String> token : tokenMap.entrySet()) {
+ value =
+ value.replaceAll(Pattern.quote(token.getKey()), token.getValue());
+ }
+ }
+ entry.setValue(value);
+ }
+ }
+
+ /**
+ * Localize the service keytabs for the service.
+ * @param launcher container launcher
+ * @param fileSystem file system
+ * @throws IOException trouble uploading to HDFS
+ */
+ public void localizeServiceKeytabs(AbstractLauncher launcher,
+ SliderFileSystem fileSystem, Service service) throws IOException {
+
+ Configuration conf = service.getConfiguration();
+ String keytabPathOnHost =
+ conf.getProperty(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
+ if (SliderUtils.isUnset(keytabPathOnHost)) {
+ String amKeytabName =
+ conf.getProperty(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
+ String keytabDir =
+ conf.getProperty(YarnServiceConf.KEY_HDFS_KEYTAB_DIR);
+ // we need to localize the keytab files in the directory
+ Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
+ service.getName());
+ boolean serviceKeytabsDeployed = false;
+ if (fileSystem.getFileSystem().exists(keytabDirPath)) {
+ FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(
+ keytabDirPath);
+ LocalResource keytabRes;
+ for (FileStatus keytab : keytabs) {
+ if (!amKeytabName.equals(keytab.getPath().getName())
+ && keytab.getPath().getName().endsWith(".keytab")) {
+ serviceKeytabsDeployed = true;
+ log.info("Localizing keytab {}", keytab.getPath().getName());
+ keytabRes = fileSystem.createAmResource(keytab.getPath(),
+ LocalResourceType.FILE);
+ launcher.addLocalResource(KEYTAB_DIR + "/" +
+ keytab.getPath().getName(),
+ keytabRes);
+ }
+ }
+ }
+ if (!serviceKeytabsDeployed) {
+ log.warn("No service keytabs for the service have been localized. "
+ + "If the service requires keytabs for secure operation, "
+ + "please ensure that the required keytabs have been uploaded "
+ + "to the folder {}", keytabDirPath);
+ }
+ }
+ }
+
+ // 1. Create all config files for a component on hdfs for localization
+ // 2. Add the config file to localResource
+ public static synchronized void createConfigFileAndAddLocalResource(
+ AbstractLauncher launcher, SliderFileSystem fs, Component component,
+ Map<String, String> tokensForSubstitution, ComponentInstance instance,
+ ServiceContext context) throws IOException {
+ Path compDir =
+ new Path(new Path(fs.getAppDir(), "components"), component.getName());
+ Path compInstanceDir =
+ new Path(compDir, instance.getCompInstanceName());
+ if (!fs.getFileSystem().exists(compInstanceDir)) {
+ log.info(instance.getCompInstanceId() + ": Creating dir on hdfs: " + compInstanceDir);
+ fs.getFileSystem().mkdirs(compInstanceDir,
+ new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
+ instance.setCompInstanceDir(compInstanceDir);
+ } else {
+ log.info("Component instance conf dir already exists: " + compInstanceDir);
+ }
+
+ if (log.isDebugEnabled()) {
+ log.debug("Tokens substitution for component instance: " + instance
+ .getCompInstanceName() + System.lineSeparator()
+ + tokensForSubstitution);
+ }
+
+ for (ConfigFile originalFile : component.getConfiguration().getFiles()) {
+ ConfigFile configFile = originalFile.copy();
+ String fileName = new Path(configFile.getDestFile()).getName();
+
+ // substitute file name
+ for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
+ configFile.setDestFile(configFile.getDestFile()
+ .replaceAll(Pattern.quote(token.getKey()), token.getValue()));
+ }
+
+ Path remoteFile = new Path(compInstanceDir, fileName);
+ if (!fs.getFileSystem().exists(remoteFile)) {
+ log.info("Saving config file on hdfs for component " + instance
+ .getCompInstanceName() + ": " + configFile);
+
+ if (configFile.getSrcFile() != null) {
+ // Load config file template
+ switch (configFile.getType()) {
+ case HADOOP_XML:
+ // Hadoop_xml_template
+ resolveHadoopXmlTemplateAndSaveOnHdfs(fs.getFileSystem(),
+ tokensForSubstitution, configFile, remoteFile, context);
+ break;
+ case TEMPLATE:
+ // plain-template
+ resolvePlainTemplateAndSaveOnHdfs(fs.getFileSystem(),
+ tokensForSubstitution, configFile, remoteFile, context);
+ break;
+ default:
+ log.info("Not supporting loading src_file for " + configFile);
+ break;
+ }
+ } else {
+ // non-template
+ resolveNonTemplateConfigsAndSaveOnHdfs(fs, tokensForSubstitution,
+ instance, configFile, fileName, remoteFile);
+ }
+ }
+
+ // Add resource for localization
+ LocalResource configResource =
+ fs.createAmResource(remoteFile, LocalResourceType.FILE);
+ File destFile = new File(configFile.getDestFile());
+ String symlink = APP_CONF_DIR + "/" + fileName;
+ if (destFile.isAbsolute()) {
+ launcher.addLocalResource(symlink, configResource,
+ configFile.getDestFile());
+ log.info("Add config file for localization: " + symlink + " -> "
+ + configResource.getResource().getFile() + ", dest mount path: "
+ + configFile.getDestFile());
+ } else {
+ launcher.addLocalResource(symlink, configResource);
+ log.info("Add config file for localization: " + symlink + " -> "
+ + configResource.getResource().getFile());
+ }
+ }
+ }
+
+ private static void resolveNonTemplateConfigsAndSaveOnHdfs(SliderFileSystem fs,
+ Map<String, String> tokensForSubstitution, ComponentInstance instance,
+ ConfigFile configFile, String fileName, Path remoteFile)
+ throws IOException {
+ // substitute non-template configs
+ substituteMapWithTokens(configFile.getProps(), tokensForSubstitution);
+
+ // write configs onto hdfs
+ PublishedConfiguration publishedConfiguration =
+ new PublishedConfiguration(fileName,
+ configFile.getProps().entrySet());
+ if (!fs.getFileSystem().exists(remoteFile)) {
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(
+ ConfigFormat.resolve(configFile.getType().toString()),
+ publishedConfiguration);
+ try (FSDataOutputStream os = fs.getFileSystem().create(remoteFile)) {
+ configurationOutputter.save(os);
+ os.flush();
+ }
+ } else {
+ log.info("Component instance = " + instance.getCompInstanceName()
+ + ", config file already exists: " + remoteFile);
+ }
+ }
+
+ // 1. substitute config template - only handle hadoop_xml format
+ // 2. save on hdfs
+ @SuppressWarnings("unchecked")
+ private static void resolveHadoopXmlTemplateAndSaveOnHdfs(FileSystem fs,
+ Map<String, String> tokensForSubstitution, ConfigFile configFile,
+ Path remoteFile, ServiceContext context) throws IOException {
+ Map<String, String> conf;
+ try {
+ conf = (Map<String, String>) context.configCache.get(configFile);
+ } catch (ExecutionException e) {
+ log.info("Failed to load config file: " + configFile, e);
+ return;
+ }
+ // make a copy for substitution
+ org.apache.hadoop.conf.Configuration confCopy =
+ new org.apache.hadoop.conf.Configuration(false);
+ for (Map.Entry<String, String> entry : conf.entrySet()) {
+ confCopy.set(entry.getKey(), entry.getValue());
+ }
+ // substitute properties
+ for (Map.Entry<String, String> entry : configFile.getProps().entrySet()) {
+ confCopy.set(entry.getKey(), entry.getValue());
+ }
+ // substitute env variables
+ for (Map.Entry<String, String> entry : confCopy) {
+ String val = entry.getValue();
+ if (val != null) {
+ for (Map.Entry<String, String> token : tokensForSubstitution
+ .entrySet()) {
+ val = val.replaceAll(Pattern.quote(token.getKey()), token.getValue());
+ confCopy.set(entry.getKey(), val);
+ }
+ }
+ }
+ // save on hdfs
+ try (OutputStream output = fs.create(remoteFile)) {
+ confCopy.writeXml(output);
+ log.info("Reading config from: " + configFile.getSrcFile()
+ + ", writing to: " + remoteFile);
+ }
+ }
+
+ // 1) read the template as a string
+ // 2) do token substitution
+ // 3) save on hdfs
+ private static void resolvePlainTemplateAndSaveOnHdfs(FileSystem fs,
+ Map<String, String> tokensForSubstitution, ConfigFile configFile,
+ Path remoteFile, ServiceContext context) {
+ String content;
+ try {
+ content = (String) context.configCache.get(configFile);
+ } catch (ExecutionException e) {
+ log.info("Failed to load config file: " + configFile, e);
+ return;
+ }
+ // substitute tokens
+ content = substituteStrWithTokens(content, tokensForSubstitution);
+
+ try (OutputStream output = fs.create(remoteFile)) {
+ org.apache.commons.io.IOUtils.write(content, output);
+ } catch (IOException e) {
+ log.info("Failed to create " + remoteFile);
+ }
+ }
+
+ /**
+ * Get initial component token map to be substituted into config values.
+ * @return tokens to replace
+ */
+ public static Map<String, String> initCompTokensForSubstitute(
+ ComponentInstance instance) {
+ Map<String, String> tokens = new HashMap<>();
+ tokens.put(COMPONENT_NAME, instance.getCompSpec().getName());
+ tokens
+ .put(COMPONENT_NAME_LC, instance.getCompSpec().getName().toLowerCase());
+ tokens.put(COMPONENT_INSTANCE_NAME, instance.getCompInstanceName());
+ tokens.put(CONTAINER_ID, instance.getContainer().getId().toString());
+ tokens.put(COMPONENT_ID,
+ String.valueOf(instance.getCompInstanceId().getId()));
+ return tokens;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
new file mode 100644
index 0000000..0920a9c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.defaultImpl;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+public class DefaultClientProvider extends AbstractClientProvider {
+
+ public DefaultClientProvider() {
+ }
+
+ @Override
+ public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
+ }
+
+ @Override
+ protected void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException {
+ // validate dest_file is not absolute
+ if (Paths.get(configFile.getDestFile()).isAbsolute()) {
+ throw new IllegalArgumentException(
+ "Dest_file must not be absolute path: " + configFile.getDestFile());
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
new file mode 100644
index 0000000..868bba8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.defaultImpl;
+
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+
+public final class DefaultProviderFactory extends ProviderFactory {
+ private static final ProviderFactory FACTORY = new
+ DefaultProviderFactory();
+
+ private DefaultProviderFactory() {}
+
+ private static class Client {
+ static final AbstractClientProvider PROVIDER = new DefaultClientProvider();
+ }
+
+ private static class Server {
+ static final ProviderService PROVIDER = new DefaultProviderService();
+ }
+
+ @Override
+ public AbstractClientProvider createClientProvider() {
+ return Client.PROVIDER;
+ }
+
+ @Override
+ public ProviderService createServerProvider() {
+ return Server.PROVIDER;
+ }
+
+ public static ProviderFactory getInstance() {
+ return FACTORY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
new file mode 100644
index 0000000..a3a0c1f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.defaultImpl;
+
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+
+import java.io.IOException;
+
+public class DefaultProviderService extends AbstractProviderService {
+
+ @Override
+ public void processArtifact(AbstractLauncher launcher,
+ ComponentInstance compInstance, SliderFileSystem fileSystem,
+ Service service)
+ throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
new file mode 100644
index 0000000..d4a2254
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+
+import java.io.IOException;
+
+public class DockerClientProvider extends AbstractClientProvider
+ implements YarnServiceConstants {
+
+ public DockerClientProvider() {
+ super();
+ }
+
+ @Override
+ public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
+ if (artifact == null) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
+ }
+ if (StringUtils.isEmpty(artifact.getId())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+ }
+ }
+
+ @Override
+ protected void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
new file mode 100644
index 0000000..f30c002
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+public interface DockerKeys {
+ String PROVIDER_DOCKER = "docker";
+ String DOCKER_PREFIX = "docker.";
+ String DOCKER_IMAGE = DOCKER_PREFIX + "image";
+ String DOCKER_NETWORK = DOCKER_PREFIX + "network";
+ String DOCKER_USE_PRIVILEGED = DOCKER_PREFIX + "usePrivileged";
+ String DOCKER_START_COMMAND = DOCKER_PREFIX + "startCommand";
+
+ String DEFAULT_DOCKER_NETWORK = "bridge";
+ Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
new file mode 100644
index 0000000..57330ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+
+public class DockerProviderFactory extends ProviderFactory {
+ private static final ProviderFactory FACTORY = new
+ DockerProviderFactory();
+
+ private DockerProviderFactory() {
+ }
+
+ private static class Client {
+ static final AbstractClientProvider PROVIDER = new DockerClientProvider();
+ }
+
+ private static class Server {
+ static final ProviderService PROVIDER = new DockerProviderService();
+ }
+
+ @Override
+ public AbstractClientProvider createClientProvider() {
+ return Client.PROVIDER;
+ }
+
+ @Override
+ public ProviderService createServerProvider() {
+ return Server.PROVIDER;
+ }
+
+ public static ProviderFactory getInstance() {
+ return FACTORY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
new file mode 100644
index 0000000..0741947
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+
+import java.io.IOException;
+import java.text.MessageFormat;
+
+public class DockerProviderService extends AbstractProviderService
+ implements DockerKeys {
+
+ public void processArtifact(AbstractLauncher launcher,
+ ComponentInstance compInstance, SliderFileSystem fileSystem,
+ Service service) throws IOException{
+ launcher.setYarnDockerMode(true);
+ launcher.setDockerImage(compInstance.getCompSpec().getArtifact().getId());
+ launcher.setDockerNetwork(compInstance.getCompSpec().getConfiguration()
+ .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK));
+ String domain = compInstance.getComponent().getScheduler().getConfig()
+ .get(RegistryConstants.KEY_DNS_DOMAIN);
+ String hostname;
+ if (domain == null || domain.isEmpty()) {
+ hostname = MessageFormat
+ .format("{0}.{1}.{2}", compInstance.getCompInstanceName(),
+ service.getName(), RegistryUtils.currentUser());
+ } else {
+ hostname = MessageFormat
+ .format("{0}.{1}.{2}.{3}", compInstance.getCompInstanceName(),
+ service.getName(), RegistryUtils.currentUser(), domain);
+ }
+ launcher.setDockerHostname(hostname);
+ launcher.setRunPrivilegedContainer(
+ compInstance.getCompSpec().getRunPrivilegedContainer());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
new file mode 100644
index 0000000..01f7b20
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.tarball;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+public class TarballClientProvider extends AbstractClientProvider
+ implements YarnServiceConstants {
+
+ public TarballClientProvider() {
+ }
+
+ @Override
+ public void validateArtifact(Artifact artifact, FileSystem fs)
+ throws IOException {
+ if (artifact == null) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
+ }
+ if (StringUtils.isEmpty(artifact.getId())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+ }
+ Path p = new Path(artifact.getId());
+ if (!fs.exists(p)) {
+ throw new IllegalArgumentException( "Artifact tarball does not exist "
+ + artifact.getId());
+ }
+ }
+
+ @Override
+ protected void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException {
+ // validate dest_file is not absolute
+ if (Paths.get(configFile.getDestFile()).isAbsolute()) {
+ throw new IllegalArgumentException(
+ "Dest_file must not be absolute path: " + configFile.getDestFile());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
new file mode 100644
index 0000000..9d81f66
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.tarball;
+
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+
+public class TarballProviderFactory extends ProviderFactory {
+ private static final ProviderFactory FACTORY = new
+ TarballProviderFactory();
+
+ private TarballProviderFactory() {
+ }
+
+ private static class Client {
+ static final AbstractClientProvider PROVIDER = new TarballClientProvider();
+ }
+
+ private static class Server {
+ static final ProviderService PROVIDER = new TarballProviderService();
+ }
+
+ @Override
+ public AbstractClientProvider createClientProvider() {
+ return Client.PROVIDER;
+ }
+
+ @Override
+ public ProviderService createServerProvider() {
+ return Server.PROVIDER;
+ }
+
+ public static ProviderFactory getInstance() {
+ return FACTORY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
new file mode 100644
index 0000000..9f29c8b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.tarball;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+
+import java.io.IOException;
+
+public class TarballProviderService extends AbstractProviderService {
+
+ @Override
+ public void processArtifact(AbstractLauncher launcher,
+ ComponentInstance instance, SliderFileSystem fileSystem,
+ Service service)
+ throws IOException {
+ Path artifact = new Path(instance.getCompSpec().getArtifact().getId());
+ if (!fileSystem.isFile(artifact)) {
+ throw new IOException(
+ "Package doesn't exist as a resource: " + artifact.toString());
+ }
+ log.info("Adding resource {}", artifact.toString());
+ LocalResourceType type = LocalResourceType.ARCHIVE;
+ LocalResource packageResource = fileSystem.createAmResource(artifact, type);
+ launcher.addLocalResource(APP_LIB_DIR, packageResource);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
new file mode 100644
index 0000000..56634f6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.registry;
+
+/**
+ * These are constants unique to the Slider AM
+ */
+public class CustomRegistryConstants {
+
+ public static final String MANAGEMENT_REST_API =
+ "classpath:org.apache.slider.management";
+
+ public static final String REGISTRY_REST_API =
+ "classpath:org.apache.slider.registry";
+
+ public static final String PUBLISHER_REST_API =
+ "classpath:org.apache.slider.publisher";
+
+ public static final String PUBLISHER_CONFIGURATIONS_API =
+ "classpath:org.apache.slider.publisher.configurations";
+
+ public static final String PUBLISHER_EXPORTS_API =
+ "classpath:org.apache.slider.publisher.exports";
+
+ public static final String PUBLISHER_DOCUMENTS_API =
+ "classpath:org.apache.slider.publisher.documents";
+
+ public static final String AGENT_SECURE_REST_API =
+ "classpath:org.apache.slider.agents.secure";
+
+ public static final String AGENT_ONEWAY_REST_API =
+ "classpath:org.apache.slider.agents.oneway";
+
+ public static final String AM_IPC_PROTOCOL =
+ "classpath:org.apache.slider.appmaster.ipc";
+
+ public static final String AM_REST_BASE =
+ "classpath:org.apache.slider.client.rest";
+
+ public static final String WEB_UI = "http://";
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
new file mode 100644
index 0000000..e310f45
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_UPDATE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE)
+
+public class ActionUpdateArgs extends AbstractClusterBuildingActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_UPDATE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
index f7b7349..00151f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.yarn.service.client.params;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
index d8d8ab4..204149b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
@@ -26,59 +26,38 @@ package org.apache.hadoop.yarn.service.client.params;
* so it is easier to see what arguments are there
*/
public interface Arguments {
- String ARG_ADDON = "--addon";
- String ARG_ALL = "--all";
- String ARG_ALIAS = "--alias";
- String ARG_APPLICATION = "--application";
+
String ARG_APPDEF = "--appdef";
- String ARG_APP_HOME = "--apphome";
String ARG_BASE_PATH = "--basepath";
- String ARG_CLIENT = "--client";
- String ARG_CONFDIR = "--appconf";
String ARG_COMPONENT = "--component";
- String ARG_COUNT = "--count";
String ARG_COMPONENT_SHORT = "--comp";
String ARG_COMPONENTS = "--components";
String ARG_COMP_OPT= "--compopt";
String ARG_COMP_OPT_SHORT = "--co";
String ARG_CONFIG = "--config";
String ARG_CONTAINERS = "--containers";
- String ARG_CREDENTIALS = "--credentials";
String ARG_DEBUG = "--debug";
String ARG_DEFINE = "-D";
String ARG_DELETE = "--delete";
String ARG_DEST = "--dest";
String ARG_DESTDIR = "--destdir";
- String ARG_DESTFILE = "--destfile";
- String ARG_EXITCODE = "--exitcode";
- String ARG_FAIL = "--fail";
- /**
- filesystem-uri: {@value}
- */
String ARG_FILESYSTEM = "--fs";
String ARG_FILESYSTEM_LONG = "--filesystem";
String ARG_FOLDER = "--folder";
String ARG_FORCE = "--force";
String ARG_FORMAT = "--format";
- String ARG_GETCERTSTORE = "--getcertstore";
String ARG_GETCONF = "--getconf";
String ARG_GETEXP = "--getexp";
String ARG_GETFILES = "--getfiles";
- String ARG_HEALTHY= "--healthy";
String ARG_HELP = "--help";
- String ARG_HOSTNAME = "--hostname";
- String ARG_ID = "--id";
String ARG_IMAGE = "--image";
String ARG_INSTALL = "--install";
String ARG_INTERNAL = "--internal";
String ARG_KEYLEN = "--keylen";
String ARG_KEYTAB = "--keytab";
- String ARG_KEYSTORE = "--keystore";
String ARG_KEYTABINSTALL = ARG_INSTALL;
String ARG_KEYTABDELETE = ARG_DELETE;
String ARG_KEYTABLIST = "--list";
- String ARG_LABEL = "--label";
- String ARG_LEVEL = "--level";
String ARG_LIST = "--list";
String ARG_LISTCONF = "--listconf";
String ARG_LISTEXP = "--listexp";
@@ -87,8 +66,6 @@ public interface Arguments {
String ARG_MANAGER = "--manager";
String ARG_MANAGER_SHORT = "--m";
String ARG_MESSAGE = "--message";
- String ARG_METAINFO = "--metainfo";
- String ARG_METAINFO_JSON = "--metainfojson";
String ARG_NAME = "--name";
String ARG_OPTION = "--option";
String ARG_OPTION_SHORT = "-O";
@@ -96,35 +73,23 @@ public interface Arguments {
String ARG_OUTPUT_SHORT = "-o";
String ARG_OVERWRITE = "--overwrite";
String ARG_PACKAGE = "--package";
- String ARG_PASSWORD = "--password";
String ARG_PATH = "--path";
- String ARG_PKGDELETE = ARG_DELETE;
- String ARG_PKGINSTANCES = "--instances";
- String ARG_PKGLIST = ARG_LIST;
String ARG_PRINCIPAL = "--principal";
- String ARG_PROVIDER = "--provider";
String ARG_QUEUE = "--queue";
String ARG_LIFETIME = "--lifetime";
- String ARG_REPLACE_PKG = "--replacepkg";
String ARG_RESOURCE = "--resource";
String ARG_RESOURCE_MANAGER = "--rm";
String ARG_SECURE = "--secure";
String ARG_SERVICETYPE = "--servicetype";
String ARG_SERVICES = "--services";
- String ARG_SLIDER = "--slider";
String ARG_SOURCE = "--source";
String ARG_STATE = "--state";
String ARG_SYSPROP = "-S";
- String ARG_TRUSTSTORE = "--truststore";
String ARG_USER = "--user";
String ARG_UPLOAD = "--upload";
String ARG_VERBOSE = "--verbose";
String ARG_VERSION = "--version";
String ARG_WAIT = "--wait";
- String ARG_YARN = "--yarn";
- String ARG_ZKHOSTS = "--zkhosts";
- String ARG_ZKPATH = "--zkpath";
- String ARG_ZKPORT = "--zkport";
/*
STOP: DO NOT ADD YOUR ARGUMENTS HERE. GO BACK AND INSERT THEM IN THE
RIGHT PLACE IN THE LIST
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
index e85db58..7b957fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
@@ -20,78 +20,36 @@ package org.apache.hadoop.yarn.service.client.params;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
-import org.apache.slider.common.params.ActionAMSuicideArgs;
-import org.apache.slider.common.params.ActionClientArgs;
-import org.apache.slider.common.params.ActionDiagnosticArgs;
-import org.apache.slider.common.params.ActionExistsArgs;
-import org.apache.slider.common.params.ActionFreezeArgs;
-import org.apache.slider.common.params.ActionHelpArgs;
-import org.apache.slider.common.params.ActionKDiagArgs;
-import org.apache.slider.common.params.ActionKeytabArgs;
-import org.apache.slider.common.params.ActionKillContainerArgs;
-import org.apache.slider.common.params.ActionListArgs;
-import org.apache.slider.common.params.ActionLookupArgs;
-import org.apache.slider.common.params.ActionNodesArgs;
-import org.apache.slider.common.params.ActionRegistryArgs;
-import org.apache.slider.common.params.ActionResolveArgs;
-import org.apache.slider.common.params.ActionResourceArgs;
-import org.apache.slider.common.params.ActionStatusArgs;
-import org.apache.slider.common.params.ActionThawArgs;
-import org.apache.slider.common.params.ActionTokensArgs;
-import org.apache.slider.common.params.ActionUpdateArgs;
-import org.apache.slider.common.params.ActionUpgradeArgs;
-import org.apache.slider.common.params.ActionVersionArgs;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
import java.util.Collection;
/**
- * Slider Client CLI Args
+ * Client CLI Args
*/
public class ClientArgs extends CommonArgs {
- /*
-
- All the arguments for specific actions
-
- */
- /**
- * This is not bonded to jcommander, it is set up
- * after the construction to point to the relevant
- * entry
- *
- * KEEP IN ALPHABETICAL ORDER
- */
- private AbstractClusterBuildingActionArgs buildingActionArgs;
-
// =========================================================
// Keep all of these in alphabetical order. Thanks.
// =========================================================
- private final ActionAMSuicideArgs actionAMSuicideArgs = new ActionAMSuicideArgs();
private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
private final ActionClientArgs actionClientArgs = new ActionClientArgs();
private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
private final ActionDependencyArgs actionDependencyArgs = new ActionDependencyArgs();
private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs();
- private final ActionDiagnosticArgs actionDiagnosticArgs = new ActionDiagnosticArgs();
private final ActionExistsArgs actionExistsArgs = new ActionExistsArgs();
private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs();
private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
private final ActionKDiagArgs actionKDiagArgs = new ActionKDiagArgs();
private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs();
- private final ActionKillContainerArgs actionKillContainerArgs =
- new ActionKillContainerArgs();
private final ActionListArgs actionListArgs = new ActionListArgs();
- private final ActionLookupArgs actionLookupArgs = new ActionLookupArgs();
- private final ActionNodesArgs actionNodesArgs = new ActionNodesArgs();
private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
private final ActionResourceArgs actionResourceArgs = new ActionResourceArgs();
@@ -99,8 +57,6 @@ public class ClientArgs extends CommonArgs {
private final ActionThawArgs actionThawArgs = new ActionThawArgs();
private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs();
private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
- private final ActionUpgradeArgs actionUpgradeArgs = new ActionUpgradeArgs();
- private final ActionVersionArgs actionVersionArgs = new ActionVersionArgs();
public ClientArgs(String[] args) {
super(args);
@@ -114,32 +70,15 @@ public class ClientArgs extends CommonArgs {
protected void addActionArguments() {
addActions(
- actionAMSuicideArgs,
actionBuildArgs,
- actionClientArgs,
actionCreateArgs,
actionDependencyArgs,
actionDestroyArgs,
- actionDiagnosticArgs,
- actionExistsArgs,
actionFlexArgs,
actionFreezeArgs,
actionHelpArgs,
- actionKDiagArgs,
- actionKeytabArgs,
- actionKillContainerArgs,
- actionListArgs,
- actionLookupArgs,
- actionNodesArgs,
- actionRegistryArgs,
- actionResolveArgs,
- actionResourceArgs,
actionStatusArgs,
- actionThawArgs,
- actionTokenArgs,
- actionUpdateArgs,
- actionUpgradeArgs,
- actionVersionArgs
+ actionThawArgs
);
}
@@ -154,43 +93,20 @@ public class ClientArgs extends CommonArgs {
}
if (getBasePath() != null) {
log.debug("Setting basePath to {}", getBasePath());
- conf.set(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH,
+ conf.set(YarnServiceConf.YARN_SERVICE_BASE_PATH,
getBasePath().toString());
}
}
- public ActionDiagnosticArgs getActionDiagnosticArgs() {
- return actionDiagnosticArgs;
- }
-
- public AbstractClusterBuildingActionArgs getBuildingActionArgs() {
- return buildingActionArgs;
- }
-
- public ActionAMSuicideArgs getActionAMSuicideArgs() {
- return actionAMSuicideArgs;
- }
public ActionBuildArgs getActionBuildArgs() {
return actionBuildArgs;
}
- public ActionClientArgs getActionClientArgs() { return actionClientArgs; }
-
- public ActionKDiagArgs getActionKDiagArgs() {
- return actionKDiagArgs;
- }
-
- public ActionKeytabArgs getActionKeytabArgs() { return actionKeytabArgs; }
-
public ActionUpdateArgs getActionUpdateArgs() {
return actionUpdateArgs;
}
- public ActionUpgradeArgs getActionUpgradeArgs() {
- return actionUpgradeArgs;
- }
-
public ActionCreateArgs getActionCreateArgs() {
return actionCreateArgs;
}
@@ -215,21 +131,10 @@ public class ClientArgs extends CommonArgs {
return actionFreezeArgs;
}
- public ActionKillContainerArgs getActionKillContainerArgs() {
- return actionKillContainerArgs;
- }
-
public ActionListArgs getActionListArgs() {
return actionListArgs;
}
- public ActionNodesArgs getActionNodesArgs() {
- return actionNodesArgs;
- }
-
- public ActionLookupArgs getActionLookupArgs() {
- return actionLookupArgs;
- }
public ActionRegistryArgs getActionRegistryArgs() {
return actionRegistryArgs;
@@ -268,14 +173,10 @@ public class ClientArgs extends CommonArgs {
switch (action) {
case ACTION_BUILD:
bindCoreAction(actionBuildArgs);
- //its a builder, so set those actions too
- buildingActionArgs = actionBuildArgs;
break;
case ACTION_CREATE:
bindCoreAction(actionCreateArgs);
- //its a builder, so set those actions too
- buildingActionArgs = actionCreateArgs;
break;
case ACTION_STOP:
@@ -286,14 +187,6 @@ public class ClientArgs extends CommonArgs {
bindCoreAction(actionThawArgs);
break;
- case ACTION_AM_SUICIDE:
- bindCoreAction(actionAMSuicideArgs);
- break;
-
- case ACTION_CLIENT:
- bindCoreAction(actionClientArgs);
- break;
-
case ACTION_DEPENDENCY:
bindCoreAction(actionDependencyArgs);
break;
@@ -302,10 +195,6 @@ public class ClientArgs extends CommonArgs {
bindCoreAction(actionDestroyArgs);
break;
- case ACTION_DIAGNOSTICS:
- bindCoreAction(actionDiagnosticArgs);
- break;
-
case ACTION_EXISTS:
bindCoreAction(actionExistsArgs);
break;
@@ -326,22 +215,10 @@ public class ClientArgs extends CommonArgs {
bindCoreAction(actionKeytabArgs);
break;
- case ACTION_KILL_CONTAINER:
- bindCoreAction(actionKillContainerArgs);
- break;
-
case ACTION_LIST:
bindCoreAction(actionListArgs);
break;
- case ACTION_LOOKUP:
- bindCoreAction(actionLookupArgs);
- break;
-
- case ACTION_NODES:
- bindCoreAction(actionNodesArgs);
- break;
-
case ACTION_REGISTRY:
bindCoreAction(actionRegistryArgs);
break;
@@ -366,14 +243,6 @@ public class ClientArgs extends CommonArgs {
bindCoreAction(actionUpdateArgs);
break;
- case ACTION_UPGRADE:
- bindCoreAction(actionUpgradeArgs);
- break;
-
- case ACTION_VERSION:
- bindCoreAction(actionVersionArgs);
- break;
-
default:
throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
+ " " + action);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
index 3160512..d44412a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
@@ -25,11 +25,11 @@ import com.beust.jcommander.ParameterException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.exceptions.UsageException;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,7 +112,7 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
if (commandOfInterest == null) {
// JCommander.usage is too verbose for a command with many options like
// slider no short version of that is found Instead, we compose our msg by
- helperMessage.append("\nUsage: slider COMMAND [options]\n");
+ helperMessage.append("\nUsage: service COMMAND [options]\n");
helperMessage.append("where COMMAND is one of\n");
for (String jcommand : serviceArgs.commander.getCommands().keySet()) {
helperMessage.append(String.format("\t%-"
@@ -123,7 +123,7 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
.append("Most commands print help when invoked without parameters or with --help");
result = helperMessage.toString();
} else {
- helperMessage.append("\nUsage: slider ").append(commandOfInterest);
+ helperMessage.append("\nUsage: service ").append(commandOfInterest);
helperMessage.append(serviceArgs.coreAction.getMinParams() > 0 ? " <application>" : "");
helperMessage.append("\n");
for (ParameterDescription paramDesc : serviceArgs.commander.getCommands()
@@ -224,14 +224,6 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
}
/**
- * Get the core action -type depends on the action
- * @return the action class
- */
- public AbstractActionArgs getCoreAction() {
- return coreAction;
- }
-
- /**
* Validate the arguments against the action requested
*/
public void validate() throws BadCommandArgumentsException, UsageException {
@@ -244,7 +236,8 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
coreAction.validate();
} catch (BadCommandArgumentsException e) {
String badArgMsgBuilder =
- e.toString() + "\n" + usage(this, coreAction.getActionName());
+ e.getMessage() + System.lineSeparator() + usage(this,
+ coreAction.getActionName());
throw new BadCommandArgumentsException(badArgMsgBuilder);
}
}
@@ -286,9 +279,4 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
public String getAction() {
return commander.getParsedCommand();
}
-
- public List<String> getActionArgs() {
- return coreAction.parameters;
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
index 0bdf58e..b6cd0a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
@@ -19,9 +19,7 @@
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
-import org.apache.slider.common.params.AbstractArgsDelegate;
-import org.apache.slider.common.params.DontSplitArguments;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import java.util.ArrayList;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
new file mode 100644
index 0000000..85de615
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.converters.IParameterSplitter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DontSplitArguments implements IParameterSplitter {
+
+ @Override
+ public List<String> split(String value) {
+ List<String> list = new ArrayList<>(1);
+ list.add(value);
+ return list;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
new file mode 100644
index 0000000..bf194b6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import java.io.File;
+
+/**
+ * Launch args for create and start and anything else that can start something
+ */
+public interface LaunchArgsAccessor extends WaitTimeAccessor {
+ String getRmAddress();
+
+ File getOutputFile();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
new file mode 100644
index 0000000..157fb61
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+
+import java.io.File;
+
+/**
+ * Any launch-time args
+ */
+public class LaunchArgsDelegate extends WaitArgsDelegate implements
+ LaunchArgsAccessor {
+
+
+ //TODO: do we need this?
+ @Parameter(names = ARG_RESOURCE_MANAGER,
+ description = "Resource manager hostname:port ",
+ required = false)
+ private String rmAddress;
+
+ @Override
+ public String getRmAddress() {
+ return rmAddress;
+ }
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for any application report")
+ public File outputFile;
+
+ @Override
+ public File getOutputFile() {
+ return outputFile;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
new file mode 100644
index 0000000..7972716
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Delegate for application and resource options.
+ */
+public class OptionArgsDelegate extends AbstractArgsDelegate {
+
+ /**
+ * Options key value.
+ */
+ @Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
+ description = ARG_OPTION + "<name> <value>",
+ splitter = DontSplitArguments.class)
+ public List<String> optionTuples = new ArrayList<>(0);
+
+
+ /**
+ * All the app component option triples.
+ */
+ @Parameter(names = {ARG_COMP_OPT, ARG_COMP_OPT_SHORT}, arity = 3,
+ description = "Component option " + ARG_COMP_OPT +
+ " <component> <name> <option>",
+ splitter = DontSplitArguments.class)
+ public List<String> compOptTriples = new ArrayList<>(0);
+
+ public Map<String, String> getOptionsMap() throws
+ BadCommandArgumentsException {
+ return convertTupleListToMap(ARG_OPTION, optionTuples);
+ }
+
+ /**
+ * Get the role heap mapping (may be empty, but never null).
+ * @return role heap mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, Map<String, String>> getCompOptionMap()
+ throws BadCommandArgumentsException {
+ return convertTripleListToMaps(ARG_COMP_OPT, compOptTriples);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
new file mode 100644
index 0000000..040ac64
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.converters.BaseConverter;
+import org.apache.hadoop.fs.Path;
+
+public class PathArgumentConverter extends BaseConverter<Path> {
+
+ public PathArgumentConverter(String optionName) {
+ super(optionName);
+ }
+
+ @Override
+ public Path convert(String value) {
+ return new Path(value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
index 1853229..a446665 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
@@ -21,10 +21,6 @@ package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.common.params.LaunchArgsAccessor;
-import org.apache.slider.common.params.LaunchArgsDelegate;
import java.io.File;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
index 3ea6f67..fc3c5a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
@@ -24,16 +24,13 @@ package org.apache.hadoop.yarn.service.client.params;
* are listed here to ensure the names are consistent
*/
public interface SliderActions {
- String ACTION_AM_SUICIDE = "am-suicide";
String ACTION_BUILD = "build";
String ACTION_CLIENT = "client";
String ACTION_CREATE = "create";
- String ACTION_DIAGNOSTICS = "diagnostics";
String ACTION_DEPENDENCY = "dependency";
String ACTION_UPDATE = "update";
String ACTION_UPGRADE = "upgrade";
String ACTION_DESTROY = "destroy";
- String ACTION_ECHO = "echo";
String ACTION_EXISTS = "exists";
String ACTION_FLEX = "flex";
String ACTION_STOP = "stop";
@@ -41,12 +38,8 @@ public interface SliderActions {
String ACTION_INSTALL_KEYTAB = "install-keytab";
String ACTION_KDIAG = "kdiag";
String ACTION_KEYTAB = "keytab";
- String ACTION_KILL_CONTAINER = "kill-container";
String ACTION_LIST = "list";
- String ACTION_LOOKUP = "lookup";
- String ACTION_NODES = "nodes";
- String ACTION_PREFLIGHT = "preflight";
- String ACTION_RECONFIGURE = "reconfigure";
+
String ACTION_REGISTRY = "registry";
String ACTION_RESOLVE = "resolve";
String ACTION_RESOURCE = "resource";
@@ -54,52 +47,35 @@ public interface SliderActions {
String ACTION_START = "start";
String ACTION_TOKENS = "tokens";
- String ACTION_VERSION = "version";
- String DESCRIBE_ACTION_AM_SUICIDE =
- "Tell the Slider Application Master to simulate a process failure by terminating itself";
String DESCRIBE_ACTION_BUILD =
- "Build a Slider cluster specification, but do not start it";
+ "Build a service specification, but do not start it";
String DESCRIBE_ACTION_CREATE =
- "Create a live Slider application";
+ "Build and start a service, it's equivalent to first invoke build and then start";
String DESCRIBE_ACTION_DEPENDENCY =
- "Slider AM and agent dependency (libraries) management";
+ "Yarn service framework dependency (libraries) management";
String DESCRIBE_ACTION_UPDATE =
- "Update template for a Slider application";
+ "Update template for service";
String DESCRIBE_ACTION_UPGRADE =
"Rolling upgrade/downgrade the component/containerto a newer/previous version";
String DESCRIBE_ACTION_DESTROY =
- "Destroy a stopped Slider application";
+ "Destroy a stopped service, service must be stopped first before destroying.";
String DESCRIBE_ACTION_EXISTS =
"Probe for an application running";
- String DESCRIBE_ACTION_FLEX = "Flex a Slider application";
+ String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
String DESCRIBE_ACTION_FREEZE =
- "Stop a running application";
- String DESCRIBE_ACTION_GETCONF =
- "Get the configuration of an application";
+ "Stop a running service";
String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems";
- String DESCRIBE_ACTION_KILL_CONTAINER =
- "Kill a container in the application";
String DESCRIBE_ACTION_HELP = "Print help information";
String DESCRIBE_ACTION_LIST =
- "List running Slider applications";
- String DESCRIBE_ACTION_LOOKUP =
- "look up a YARN application";
- String DESCRIBE_ACTION_NODES = "List the node information for the YARN cluster or a running application";
- String DESCRIBE_ACTION_MONITOR =
- "Monitor a running application";
+ "List running services";
String DESCRIBE_ACTION_REGISTRY =
- "Query the registry of a YARN application";
- String DESCRIBE_ACTION_RESOLVE =
- "Resolve or list records in the YARN registry";
+ "Query the registry of a service";
String DESCRIBE_ACTION_STATUS =
- "Get the status of an application";
+ "Get the status of a service";
String DESCRIBE_ACTION_THAW =
- "Start a stopped application";
- String DESCRIBE_ACTION_VERSION =
- "Print the Slider version information";
+ "Start a service with pre-built specification or a previously stopped service";
String DESCRIBE_ACTION_CLIENT = "Install the application client in the specified directory or obtain a client keystore or truststore";
String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
- String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client";
String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
new file mode 100644
index 0000000..86f3709
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+
+public class WaitArgsDelegate extends AbstractArgsDelegate implements
+ WaitTimeAccessor {
+
+
+ //--wait [timeout]
+ @Parameter(names = {ARG_WAIT},
+ description = "time to wait for an action to complete")
+ public int waittime = 0;
+
+
+ @Override
+ public int getWaittime() {
+ return waittime;
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ this.waittime = waittime;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
new file mode 100644
index 0000000..f6afae6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+public interface WaitTimeAccessor {
+ int getWaittime();
+ void setWaittime(int waittime);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
index dcb455f..982a114 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
@@ -36,17 +36,17 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.service.ServiceScheduler;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
import org.apache.hadoop.yarn.service.component.Component;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.BoundedAppender;
-import org.apache.slider.api.resource.ContainerState;
-import org.apache.slider.common.tools.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import org.apache.slider.server.servicemonitor.ProbeStatus;
-import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
+import org.apache.hadoop.yarn.service.servicemonitor.probe.ProbeStatus;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -88,7 +88,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
private volatile ContainerStatus status;
private long containerStartedTime = 0;
// This container object is used for rest API query
- private org.apache.slider.api.resource.Container containerSpec;
+ private org.apache.hadoop.yarn.service.api.records.Container containerSpec;
private static final StateMachineFactory<ComponentInstance,
ComponentInstanceState, ComponentInstanceEventType, ComponentInstanceEvent>
@@ -142,11 +142,11 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
compInstance.getContainerId(), compInstance), 0, 1,
TimeUnit.SECONDS);
- org.apache.slider.api.resource.Container container =
- new org.apache.slider.api.resource.Container();
+ org.apache.hadoop.yarn.service.api.records.Container container =
+ new org.apache.hadoop.yarn.service.api.records.Container();
container.setId(compInstance.getContainerId().toString());
container.setLaunchTime(new Date());
- container.setState(org.apache.slider.api.resource.ContainerState.RUNNING_BUT_UNREADY);
+ container.setState(ContainerState.RUNNING_BUT_UNREADY);
container.setBareHost(compInstance.container.getNodeId().getHost());
container.setComponentName(compInstance.getCompInstanceName());
if (compInstance.containerSpec != null) {
@@ -290,7 +290,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
public void updateContainerStatus(ContainerStatus status) {
this.status = status;
- org.apache.slider.api.resource.Container container =
+ org.apache.hadoop.yarn.service.api.records.Container container =
getCompSpec().getContainer(getContainerId().toString());
if (container != null) {
container.setIp(StringUtils.join(",", status.getIPs()));
@@ -330,7 +330,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
return this.container.getNodeId();
}
- public org.apache.slider.api.resource.Component getCompSpec() {
+ public org.apache.hadoop.yarn.service.api.records.Component getCompSpec() {
return component.getComponentSpec();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index bfe40c0..331871a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -39,9 +39,9 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Apps;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.server.servicemonitor.MonitorUtils;
-import org.apache.slider.server.servicemonitor.Probe;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.servicemonitor.probe.MonitorUtils;
+import org.apache.hadoop.yarn.service.servicemonitor.probe.Probe;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,12 +61,12 @@ import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.STARTED;
import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.STOP;
import static org.apache.hadoop.yarn.service.component.ComponentState.*;
-import static org.apache.slider.api.ResourceKeys.CONTAINER_FAILURE_THRESHOLD;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_THRESHOLD;
public class Component implements EventHandler<ComponentEvent> {
private static final Logger LOG = LoggerFactory.getLogger(Component.class);
- private org.apache.slider.api.resource.Component componentSpec;
+ private org.apache.hadoop.yarn.service.api.records.Component componentSpec;
private long allocateId;
private Priority priority;
private ServiceMetrics componentMetrics;
@@ -124,7 +124,8 @@ public class Component implements EventHandler<ComponentEvent> {
FLEX, new FlexComponentTransition())
.installTopology();
- public Component(org.apache.slider.api.resource.Component component,
+ public Component(
+ org.apache.hadoop.yarn.service.api.records.Component component,
long allocateId, ServiceContext context) {
this.allocateId = allocateId;
this.priority = Priority.newInstance((int) allocateId);
@@ -418,7 +419,7 @@ public class Component implements EventHandler<ComponentEvent> {
return compInstances;
}
- public org.apache.slider.api.resource.Component getComponentSpec() {
+ public org.apache.hadoop.yarn.service.api.records.Component getComponentSpec() {
return this.componentSpec;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
new file mode 100644
index 0000000..cbbb206
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+public interface RestApiConstants {
+
+ // Rest endpoints
+ String CONTEXT_ROOT = "/services/v1";
+ String VERSION = "/version";
+ String APP_ROOT_PATH = "/applications";
+ String APP_PATH = "/applications/{app_name}";
+ String COMPONENT_PATH = "/applications/{app_name}/components/{component_name}";
+
+ // Query param
+ String APP_NAME = "app_name";
+ String COMPONENT_NAME = "component_name";
+
+ String DEFAULT_COMPONENT_NAME = "default";
+
+ String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST";
+ String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT";
+ Long DEFAULT_UNLIMITED_LIFETIME = -1l;
+
+ Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001;
+ Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002;
+ Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003;
+ Integer ERROR_CODE_APP_NAME_INVALID = 404004;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
index d63c1a4..bdef600 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.yarn.service.conf;
-import org.apache.slider.core.main.LauncherExitCodes;
+import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
public interface SliderExitCodes extends LauncherExitCodes {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderKeys.java
deleted file mode 100644
index e1687d2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderKeys.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-/**
- * Keys and various constants for Slider
- */
-public interface SliderKeys extends SliderXmlConfKeys {
-
- /**
- * This is the name of the slider appmaster in configurations :{@value}
- */
- String COMPONENT_AM = "slider-appmaster";
-
- /**
- * Slider role is "special":{@value}
- */
- int ROLE_AM_PRIORITY_INDEX = 0;
-
-
- /**
- * The path under which cluster and temp data are stored
- * {@value}
- */
- String SLIDER_BASE_DIRECTORY = ".slider";
-
- /**
- * The paths under which Slider AM dependency libraries are stored
- */
- String SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK = "slider_dep";
- String SLIDER_DEPENDENCY_HDP_PARENT_DIR = "/hdp";
- String SLIDER_DEPENDENCY_DIR = "/apps/%s/slider";
- String SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME = "slider-dep";
- String SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT = ".tar.gz";
- String SLIDER_DEPENDENCY_DIR_PERMISSIONS = "755";
-
- /**
- *
- */
- String HDP_VERSION_PROP_NAME = "HDP_VERSION";
-
- /**
- * name of the relative path to expaned an image into: {@value}.
- * The title of this path is to help people understand it when
- * they see it in their error messages
- */
- String LOCAL_TARBALL_INSTALL_SUBDIR = "expandedarchive";
-
-
- /**
- * Application type for YARN {@value}
- */
- String APP_TYPE = "org-apache-slider";
-
- /**
- * A component type for an external app that has been predefined using the
- * slider build command
- */
- String COMPONENT_SEPARATOR = "-";
-
- /**
- * A component type for a client component
- */
- String COMPONENT_TYPE_CLIENT = "client";
-
- /**
- * Key for application version.
- */
- String APP_VERSION_UNKNOWN = "awaiting heartbeat...";
-
- /**
- * Keys for application container specific properties, like release timeout
- */
- String APP_CONTAINER_RELEASE_TIMEOUT = "site.global.app_container.release_timeout_secs";
-
- /**
- * Subdirectories of HDFS cluster dir.
- */
- String DATA_DIR_NAME = "data";
- String HISTORY_DIR_NAME = "history";
- String HISTORY_FILENAME_SUFFIX = "json";
- String HISTORY_FILENAME_PREFIX = "rolehistory-";
- String KEYTAB_DIR = "keytabs";
- String RESOURCE_DIR = "resources";
-
- /**
- * Filename pattern is required to save in strict temporal order.
- * Important: older files must sort less-than newer files when using
- * case-sensitive name sort.
- */
- String HISTORY_FILENAME_CREATION_PATTERN = HISTORY_FILENAME_PREFIX +"%016x."+
- HISTORY_FILENAME_SUFFIX;
- /**
- * The posix regexp used to locate this
- */
- String HISTORY_FILENAME_MATCH_PATTERN = HISTORY_FILENAME_PREFIX +"[0-9a-f]+\\."+
- HISTORY_FILENAME_SUFFIX;
- /**
- * The posix regexp used to locate this
- */
- String HISTORY_FILENAME_GLOB_PATTERN = HISTORY_FILENAME_PREFIX +"*."+
- HISTORY_FILENAME_SUFFIX;
-
- String CLUSTER_DIRECTORY = "cluster";
-
- /**
- * JVM property to define the slider lib directory;
- * this is set by the slider script: {@value}
- */
- String PROPERTY_LIB_DIR = "slider.libdir";
-
- /**
- * name of generated dir for this conf: {@value}
- */
- String SUBMITTED_CONF_DIR = "conf";
-
- /**
- * Slider AM log4j file name : {@value}
- */
- String LOG4J_SERVER_PROP_FILENAME = "slideram-log4j.properties";
-
- /**
- * Log4j sysprop to name the resource :{@value}
- */
- String SYSPROP_LOG4J_CONFIGURATION = "log4j.configuration";
-
- /**
- * sysprop for Slider AM log4j directory :{@value}
- */
- String SYSPROP_LOG_DIR = "LOG_DIR";
-
- /**
- * name of the Slider client resource
- * loaded when the service is loaded.
- */
- String SLIDER_CLIENT_XML = "slider-client.xml";
-
- /**
- * The name of the resource to put on the classpath
- */
- String SLIDER_SERVER_XML = "slider-server.xml";
-
- String TMP_DIR_PREFIX = "tmp";
-
- /**
- * Store the default app definition, e.g. metainfo file or content of a folder
- */
- String APP_DEF_DIR = "appdef";
- /**
- * Store additional app defs - co-processors
- */
- String ADDONS_DIR = "addons";
-
- String SLIDER_JAR = "slider-core.jar";
-
- String STDOUT_AM = "slider-out.txt";
- String STDERR_AM = "slider-err.txt";
-
- String HADOOP_USER_NAME = "HADOOP_USER_NAME";
-
- /**
- * Name of the AM filter to use: {@value}
- */
- String AM_FILTER_NAME =
- "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer";
-
- String YARN_CONTAINER_PATH = "/node/container/";
-
- String APP_CONF_DIR = "conf";
-
- String APP_LIB_DIR = "lib";
-
- String OUT_FILE = "stdout.txt";
- String ERR_FILE = "stderr.txt";
-
- String QUICK_LINKS = "quicklinks";
-
- String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderXmlConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderXmlConfKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderXmlConfKeys.java
deleted file mode 100644
index 523e08d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderXmlConfKeys.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-/**
- * These are the keys that can be added to <code>conf/slider-client.xml</code>.
- */
-public interface SliderXmlConfKeys {
- String PREFIX_PROVIDER = "slider.provider";
- /**
- * pattern to identify a provider
- * {@value}
- */
- String KEY_PROVIDER = PREFIX_PROVIDER + ".%s";
-
- /**
- * conf option set to point to where the config came from
- * {@value}
- */
- String KEY_TEMPLATE_ORIGIN = "slider.template.origin";
-
- /**
- * Original name for the default FS. This is still
- * expected by applications deployed
- */
- String FS_DEFAULT_NAME_CLASSIC = "fs.default.name";
-
- /**
- * Slider principal
- */
- String KEY_KERBEROS_PRINCIPAL = "slider.kerberos.principal";
-
- /**
- * Name of the property for ACLs for Slider AM.
- * {@value}
- */
- String KEY_PROTOCOL_ACL = "slider.security.protocol.acl";
-
- /**
- * Limit on restarts for the AM
- * {@value}
- */
- String KEY_AM_RESTART_LIMIT = "slider.yarn.restart.limit";
-
- /**
- * queue name, by default let YARN pick the queue
- */
- String KEY_YARN_QUEUE = "slider.yarn.queue";
- String DEFAULT_YARN_QUEUE = null;
-
- /**
- * default priority
- */
- String KEY_YARN_QUEUE_PRIORITY = "slider.yarn.queue.priority";
- int DEFAULT_YARN_QUEUE_PRIORITY = 1;
-
-
- String KEY_AM_RESOURCE_MEM = "slider.am.resource.memory";
- long DEFAULT_KEY_AM_RESOURCE_MEM = 1024;
-
- /**
- * The slider base path: {@value}
- * Defaults to HomeDir/.slider
- */
- String KEY_SLIDER_BASE_PATH = "slider.base.path";
-
-
- /**
- * Option for the permissions for the cluster directory itself: {@value}
- */
- String CLUSTER_DIRECTORY_PERMISSIONS =
- "slider.cluster.directory.permissions";
-
- /**
- * Default value for the permissions :{@value}
- */
- String DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS = "750";
-
- /**
- *
- * Option for the permissions for the data directory itself: {@value}
- */
- String DATA_DIRECTORY_PERMISSIONS = "slider.data.directory.permissions";
-
- /**
- * Default value for the data directory permissions: {@value}
- */
- String DEFAULT_DATA_DIRECTORY_PERMISSIONS = "750";
-
- String IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH =
- "ipc.client.fallback-to-simple-auth-allowed";
- String HADOOP_HTTP_FILTER_INITIALIZERS =
- "hadoop.http.filter.initializers";
- String KEY_KEYSTORE_LOCATION = "ssl.server.keystore.location";
- String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
- /** Declare that a keytab must be provided */
- String KEY_AM_LOGIN_KEYTAB_REQUIRED = "slider.am.login.keytab.required";
- String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir";
- String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
- String KEY_KEYTAB_PRINCIPAL = "slider.keytab.principal.name";
- String KEY_SECURITY_ENABLED = "site.global.security_enabled";
-
- /**
- * Set to disable server-side checks for python, openssl &c.
- * This should only be set for testing
- */
- String KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED =
- "slider.am.dependency.checks.disabled";
-
- /**
- * The path to the python executable utilized to launch the agent.
- */
- String PYTHON_EXECUTABLE_PATH = "agent.python.exec.path";
-
- /**
- * Flag to enable the insecure AM filter: {@value}
- */
- String X_DEV_INSECURE_WS = "slider.feature.ws.insecure";
-
- /**
- * Flag to indicate the insecure AM filter is enabled by default: {@value}.
- */
- boolean X_DEV_INSECURE_DEFAULT = false;
-
-
- /**
- * Flag to indicate the insecure AM filter is required for
- * complex REST Verbs: {@value}.
- * When Slider switches to being Hadoop 2.7+ only, this flag
- * can be set to false
- */
- boolean X_DEV_INSECURE_REQUIRED = true;
-
- /**
- *
- */
- String KEY_IPC_CLIENT_RETRY_POLICY_ENABLED =
- "slider.ipc.client.retry.enabled";
- boolean IPC_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = true;
- String KEY_IPC_CLIENT_RETRY_POLICY_SPEC =
- "slider.ipc.client.retry.policy.spec";
- String IPC_CLIENT_RETRY_POLICY_SPEC_DEFAULT =
- "10000,6,60000,10"; //t1,n1,t2,n2,...
-
- String KEY_AM_LAUNCH_ENV = "slider.am.launch.env";
-
- /**
- * From {@code DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY}
- */
- String DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY = "dfs.namenode.kerberos.principal";
-
- String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY = "dfs.datanode.kerberos.principal";
-
- //Delegation token related keys
- String DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY
- = "dfs.namenode.delegation.key.update-interval";
- long DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 24 * 60 * 60 *
- 1000; // 1 day
- String DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY
- = "dfs.namenode.delegation.token.renew-interval";
- long DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24 * 60 * 60 *
- 1000; // 1 day
- String DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY
- = "dfs.namenode.delegation.token.max-lifetime";
- long DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60 *
- 1000; // 7 days
- String DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY
- = "dfs.namenode.delegation.token.always-use"; // for tests
- boolean DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT = false;
- String DFS_NAMENODE_KEYTAB_FILE_KEY = "dfs.namenode.keytab.file";
- String DFS_NAMENODE_DU_RESERVED_KEY = "dfs.namenode.resource.du.reserved";
-
- String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary";
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
index 9225570..33fc671 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.yarn.service.conf;
-import org.apache.slider.api.resource.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
public class YarnServiceConf {
@@ -30,8 +30,54 @@ public class YarnServiceConf {
public static final String CONTAINER_RETRY_MAX = "yarn.service.container-failure.retry.max";
public static final String CONTAINER_RETRY_INTERVAL = "yarn.service.container-failure.retry-interval";
+ public static final String AM_RESTART_MAX = "yarn.service.am-restart.max-attempts";
+ public static final String AM_RESOURCE_MEM = "yarn.service.am-resource.memory";
+ public static final long DEFAULT_KEY_AM_RESOURCE_MEM = 1024;
+
+ public static final String YARN_QUEUE = "yarn.service.queue";
+
+ /**
+ * The yarn service base path:
+ * Defaults to HomeDir/.yarn/
+ */
+ public static final String YARN_SERVICE_BASE_PATH = "yarn.service.base.path";
+
+ //TODO rename
+ /** Declare that a keytab must be provided */
+ public static final String KEY_AM_LOGIN_KEYTAB_REQUIRED = "slider.am.login.keytab.required";
+ public static final String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
+ public static final String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir";
+ public static final String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
+
+ /**
+ * maximum number of failed containers (in a single component)
+ * before the app exits
+ */
+ public static final String CONTAINER_FAILURE_THRESHOLD =
+ "yarn.service.container-failure-per-component.threshold";
+ /**
+ * Maximum number of container failures on a node before the node is blacklisted
+ */
+ public static final String NODE_BLACKLIST_THRESHOLD =
+ "yarn.service.node-blacklist.threshold";
+
+ /**
+ * The failure count for CONTAINER_FAILURE_THRESHOLD and NODE_BLACKLIST_THRESHOLD
+ * gets reset periodically, the unit is seconds.
+ */
+ public static final String CONTAINER_FAILURE_WINDOW =
+ "yarn.service.failure-count-reset.window";
+
+ /**
+ * interval between readiness checks.
+ */
+ public static final String READINESS_CHECK_INTERVAL = "yarn.service.readiness-check-interval.seconds";
+ public static final int DEFAULT_READINESS_CHECK_INTERVAL = 30; // seconds
+
/**
- * Get long value for the property
+ * Get long value for the property. First get from the userConf, if not
+ * present, get from systemConf.
+ *
* @param name name of the property
* @param defaultValue default value of the property, if it is not defined in
* userConf and systemConf.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
new file mode 100644
index 0000000..cbcba82
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+public interface YarnServiceConstants {
+
+ /**
+ * The path under which cluster and temp data are stored
+ */
+ String SERVICE_BASE_DIRECTORY = ".yarn";
+
+ /**
+ * The paths under which Service AM dependency libraries are stored
+ */
+ String DEPENDENCY_LOCALIZED_DIR_LINK = "service_dep";
+ String DEPENDENCY_DIR = "/yarn-services/%s/";
+ String DEPENDENCY_TAR_GZ_FILE_NAME = "service-dep";
+ String DEPENDENCY_TAR_GZ_FILE_EXT = ".tar.gz";
+ String DEPENDENCY_DIR_PERMISSIONS = "755";
+
+ /**
+ * Application type for YARN service
+ */
+ String APP_TYPE = "yarn-service";
+
+ String KEYTAB_DIR = "keytabs";
+ String RESOURCE_DIR = "resources";
+
+
+ String SERVICES_DIRECTORY = "services";
+
+ /**
+ * JVM property to define the service lib directory;
+ * this is set by the yarn.sh script
+ */
+ String PROPERTY_LIB_DIR = "service.libdir";
+
+ /**
+ * name of generated dir for this conf
+ */
+ String SUBMITTED_CONF_DIR = "conf";
+
+ /**
+ * Service AM log4j file name
+ */
+ String YARN_SERVICE_LOG4J_FILENAME = "yarnservice-log4j.properties";
+
+ /**
+ * Log4j sysprop to name the resource
+ */
+ String SYSPROP_LOG4J_CONFIGURATION = "log4j.configuration";
+
+ /**
+ * sysprop for Service AM log4j directory
+ */
+ String SYSPROP_LOG_DIR = "LOG_DIR";
+
+ String TMP_DIR_PREFIX = "tmp";
+
+
+ String SERVICE_CORE_JAR = "yarn-service-core.jar";
+
+ String STDOUT_AM = "serviceam-out.txt";
+ String STDERR_AM = "serviceam-err.txt";
+
+ String HADOOP_USER_NAME = "HADOOP_USER_NAME";
+
+ String APP_CONF_DIR = "conf";
+
+ String APP_LIB_DIR = "lib";
+
+ String OUT_FILE = "stdout.txt";
+ String ERR_FILE = "stderr.txt";
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[71/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f8fe178
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f8fe178
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f8fe178
Branch: refs/heads/yarn-native-services
Commit: 4f8fe178e6faa567391d26358680e52fc0232bab
Parents: 538c63e
Author: Billie Rinaldi <bi...@apache.org>
Authored: Mon Aug 28 09:59:55 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 25 16:37:22 2017 -0700
----------------------------------------------------------------------
.../resources/assemblies/hadoop-yarn-dist.xml | 8 +-
.../assemblies/hadoop-yarn-services-dist.xml | 2 +-
hadoop-project/pom.xml | 2 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 14 +-
.../hadoop-yarn/conf/yarn-env.sh | 2 +-
.../hadoop-yarn-services-api/pom.xml | 2 +-
.../hadoop/yarn/service/webapp/ApiServer.java | 274 +++
.../yarn/service/webapp/ApiServerWebApp.java | 159 ++
.../service/webapp/ApplicationApiService.java | 275 ---
.../service/webapp/ApplicationApiWebApp.java | 123 --
...RN-Simplified-V1-API-Layer-For-Services.yaml | 185 +-
.../src/main/scripts/run_rest_service.sh | 28 -
.../dev-support/findbugs-exclude.xml | 48 +
.../conf/yarnservice-log4j.properties | 62 +
.../hadoop-yarn-services-core/pom.xml | 408 +++++
.../hadoop/yarn/service/ClientAMProtocol.java | 40 +
.../hadoop/yarn/service/ClientAMService.java | 132 ++
.../yarn/service/ContainerFailureTracker.java | 89 +
.../hadoop/yarn/service/ServiceContext.java | 41 +
.../hadoop/yarn/service/ServiceMaster.java | 156 ++
.../hadoop/yarn/service/ServiceMetrics.java | 98 +
.../hadoop/yarn/service/ServiceScheduler.java | 654 +++++++
.../yarn/service/api/ServiceApiConstants.java | 69 +
.../yarn/service/api/records/Artifact.java | 160 ++
.../yarn/service/api/records/BaseResource.java | 52 +
.../yarn/service/api/records/Component.java | 412 +++++
.../yarn/service/api/records/ConfigFile.java | 225 +++
.../yarn/service/api/records/ConfigFormat.java | 67 +
.../yarn/service/api/records/Configuration.java | 225 +++
.../yarn/service/api/records/Container.java | 297 +++
.../service/api/records/ContainerState.java | 30 +
.../hadoop/yarn/service/api/records/Error.java | 129 ++
.../service/api/records/PlacementPolicy.java | 102 ++
.../service/api/records/ReadinessCheck.java | 175 ++
.../yarn/service/api/records/Resource.java | 159 ++
.../yarn/service/api/records/Service.java | 466 +++++
.../yarn/service/api/records/ServiceState.java | 33 +
.../yarn/service/api/records/ServiceStatus.java | 148 ++
.../yarn/service/client/ClientAMProxy.java | 57 +
.../hadoop/yarn/service/client/ServiceCLI.java | 112 ++
.../yarn/service/client/ServiceClient.java | 892 +++++++++
.../client/params/AbstractActionArgs.java | 158 ++
.../client/params/AbstractArgsDelegate.java | 28 +
.../AbstractClusterBuildingActionArgs.java | 58 +
.../service/client/params/ActionBuildArgs.java | 31 +
.../service/client/params/ActionClientArgs.java | 71 +
.../service/client/params/ActionCreateArgs.java | 33 +
.../client/params/ActionDependencyArgs.java | 65 +
.../client/params/ActionDestroyArgs.java | 37 +
.../service/client/params/ActionExistsArgs.java | 49 +
.../service/client/params/ActionFlexArgs.java | 50 +
.../service/client/params/ActionFreezeArgs.java | 56 +
.../service/client/params/ActionHelpArgs.java | 44 +
.../service/client/params/ActionKDiagArgs.java | 76 +
.../service/client/params/ActionKeytabArgs.java | 76 +
.../service/client/params/ActionListArgs.java | 76 +
.../client/params/ActionRegistryArgs.java | 218 +++
.../client/params/ActionResolveArgs.java | 153 ++
.../client/params/ActionResourceArgs.java | 70 +
.../service/client/params/ActionStatusArgs.java | 51 +
.../service/client/params/ActionThawArgs.java | 67 +
.../service/client/params/ActionTokensArgs.java | 78 +
.../service/client/params/ActionUpdateArgs.java | 32 +
.../yarn/service/client/params/ArgOps.java | 156 ++
.../yarn/service/client/params/Arguments.java | 103 ++
.../yarn/service/client/params/ClientArgs.java | 252 +++
.../yarn/service/client/params/CommonArgs.java | 282 +++
.../client/params/ComponentArgsDelegate.java | 52 +
.../client/params/DontSplitArguments.java | 34 +
.../client/params/LaunchArgsAccessor.java | 30 +
.../client/params/LaunchArgsDelegate.java | 51 +
.../client/params/OptionArgsDelegate.java | 66 +
.../client/params/PathArgumentConverter.java | 34 +
.../service/client/params/SliderAMArgs.java | 57 +
.../client/params/SliderAMCreateAction.java | 73 +
.../service/client/params/SliderActions.java | 82 +
.../service/client/params/WaitArgsDelegate.java | 42 +
.../service/client/params/WaitTimeAccessor.java | 24 +
.../yarn/service/component/Component.java | 494 +++++
.../yarn/service/component/ComponentEvent.java | 83 +
.../service/component/ComponentEventType.java | 26 +
.../yarn/service/component/ComponentState.java | 25 +
.../component/instance/ComponentInstance.java | 493 +++++
.../instance/ComponentInstanceEvent.java | 58 +
.../instance/ComponentInstanceEventType.java | 26 +
.../component/instance/ComponentInstanceId.java | 91 +
.../instance/ComponentInstanceState.java | 26 +
.../yarn/service/conf/RestApiConstants.java | 43 +
.../yarn/service/conf/SliderExitCodes.java | 88 +
.../yarn/service/conf/YarnServiceConf.java | 101 ++
.../yarn/service/conf/YarnServiceConstants.java | 90 +
.../containerlaunch/AbstractLauncher.java | 271 +++
.../containerlaunch/ClasspathConstructor.java | 172 ++
.../containerlaunch/CommandLineBuilder.java | 86 +
.../containerlaunch/ContainerLaunchService.java | 101 ++
.../containerlaunch/CredentialUtils.java | 319 ++++
.../containerlaunch/JavaCommandLineBuilder.java | 181 ++
.../exceptions/BadClusterStateException.java | 36 +
.../BadCommandArgumentsException.java | 30 +
.../service/exceptions/BadConfigException.java | 39 +
.../yarn/service/exceptions/ErrorStrings.java | 42 +
.../service/exceptions/ExitCodeProvider.java | 32 +
.../service/exceptions/LauncherExitCodes.java | 196 ++
.../exceptions/RestApiErrorMessages.java | 92 +
.../exceptions/ServiceLaunchException.java | 73 +
.../service/exceptions/SliderException.java | 66 +
.../yarn/service/exceptions/UsageException.java | 34 +
.../pb/client/ClientAMProtocolPBClientImpl.java | 91 +
.../impl/pb/service/ClientAMProtocolPB.java | 29 +
.../service/ClientAMProtocolPBServiceImpl.java | 70 +
.../yarn/service/monitor/ServiceMonitor.java | 147 ++
.../yarn/service/monitor/probe/HttpProbe.java | 110 ++
.../service/monitor/probe/LogEntryBuilder.java | 76 +
.../yarn/service/monitor/probe/MonitorKeys.java | 66 +
.../service/monitor/probe/MonitorUtils.java | 84 +
.../yarn/service/monitor/probe/PortProbe.java | 98 +
.../yarn/service/monitor/probe/Probe.java | 100 ++
.../yarn/service/monitor/probe/ProbeStatus.java | 160 ++
.../provider/AbstractClientProvider.java | 122 ++
.../provider/AbstractProviderService.java | 109 ++
.../yarn/service/provider/ProviderFactory.java | 76 +
.../yarn/service/provider/ProviderService.java | 39 +
.../yarn/service/provider/ProviderUtils.java | 402 +++++
.../defaultImpl/DefaultClientProvider.java | 46 +
.../defaultImpl/DefaultProviderFactory.java | 51 +
.../defaultImpl/DefaultProviderService.java | 36 +
.../provider/docker/DockerClientProvider.java | 53 +
.../service/provider/docker/DockerKeys.java | 30 +
.../provider/docker/DockerProviderFactory.java | 52 +
.../provider/docker/DockerProviderService.java | 57 +
.../provider/tarball/TarballClientProvider.java | 65 +
.../tarball/TarballProviderFactory.java | 52 +
.../tarball/TarballProviderService.java | 48 +
.../registry/CustomRegistryConstants.java | 57 +
.../registry/YarnRegistryViewForProviders.java | 225 +++
.../timelineservice/ServiceMetricsSink.java | 102 ++
.../ServiceTimelineEntityType.java | 39 +
.../timelineservice/ServiceTimelineEvent.java | 34 +
.../ServiceTimelineMetricsConstants.java | 92 +
.../ServiceTimelinePublisher.java | 368 ++++
.../service/timelineservice/package-info.java | 27 +
.../utils/ApplicationReportSerDeser.java | 56 +
.../service/utils/ClientRegistryBinder.java | 201 +++
.../hadoop/yarn/service/utils/Comparators.java | 62 +
.../hadoop/yarn/service/utils/ConfigHelper.java | 157 ++
.../hadoop/yarn/service/utils/ConfigUtils.java | 97 +
.../yarn/service/utils/CoreFileSystem.java | 521 ++++++
.../hadoop/yarn/service/utils/Duration.java | 109 ++
.../hadoop/yarn/service/utils/JsonSerDeser.java | 249 +++
.../yarn/service/utils/KerberosDiags.java | 680 +++++++
.../yarn/service/utils/PatternValidator.java | 58 +
.../hadoop/yarn/service/utils/PortScanner.java | 113 ++
.../service/utils/PublishedConfiguration.java | 196 ++
.../utils/PublishedConfigurationOutputter.java | 212 +++
.../utils/SerializedApplicationReport.java | 98 +
.../yarn/service/utils/ServiceApiUtil.java | 446 +++++
.../service/utils/ServiceRegistryUtils.java | 71 +
.../yarn/service/utils/SliderFileSystem.java | 51 +
.../hadoop/yarn/service/utils/SliderUtils.java | 1654 +++++++++++++++++
.../yarn/service/utils/ZookeeperUtils.java | 146 ++
.../src/main/proto/ClientAMProtocol.proto | 56 +
.../hadoop/yarn/service/MockServiceAM.java | 221 +++
.../hadoop/yarn/service/ServiceTestUtils.java | 59 +
.../hadoop/yarn/service/TestServiceApiUtil.java | 530 ++++++
.../yarn/service/TestYarnNativeServices.java | 472 +++++
.../client/TestBuildExternalComponents.java | 128 ++
.../yarn/service/client/TestServiceCLI.java | 155 ++
.../yarn/service/conf/ExampleAppJson.java | 65 +
.../yarn/service/conf/TestAppJsonResolve.java | 222 +++
.../service/conf/TestLoadExampleAppJson.java | 78 +
.../service/conf/TestValidateServiceNames.java | 126 ++
.../service/monitor/TestServiceMonitor.java | 104 ++
.../providers/TestAbstractClientProvider.java | 118 ++
.../service/providers/TestProviderFactory.java | 76 +
.../TestServiceTimelinePublisher.java | 293 +++
.../src/test/resources/example-app.json | 15 +
.../service/conf/examples/app-override.json | 72 +
.../hadoop/yarn/service/conf/examples/app.json | 48 +
.../yarn/service/conf/examples/default.json | 16 +
.../yarn/service/conf/examples/external0.json | 8 +
.../yarn/service/conf/examples/external1.json | 30 +
.../yarn/service/conf/examples/external2.json | 22 +
.../src/test/resources/yarn-site.xml | 19 +
.../hadoop-yarn-services/pom.xml | 38 +
.../dev-support/findbugs-exclude.xml | 48 -
.../conf/yarnservice-log4j.properties | 62 -
.../hadoop-yarn-slider-core/pom.xml | 409 -----
.../src/assembly/executable-jar.xml | 47 -
.../hadoop/yarn/service/ClientAMProtocol.java | 40 -
.../hadoop/yarn/service/ClientAMService.java | 132 --
.../yarn/service/ContainerFailureTracker.java | 89 -
.../hadoop/yarn/service/ServiceContext.java | 41 -
.../hadoop/yarn/service/ServiceMaster.java | 157 --
.../hadoop/yarn/service/ServiceScheduler.java | 655 -------
.../api/constants/ServiceApiConstants.java | 69 -
.../yarn/service/api/records/Application.java | 466 -----
.../service/api/records/ApplicationState.java | 33 -
.../service/api/records/ApplicationStatus.java | 148 --
.../yarn/service/api/records/Artifact.java | 160 --
.../yarn/service/api/records/BaseResource.java | 52 -
.../yarn/service/api/records/Component.java | 412 -----
.../yarn/service/api/records/ConfigFile.java | 225 ---
.../yarn/service/api/records/ConfigFormat.java | 67 -
.../yarn/service/api/records/Configuration.java | 225 ---
.../yarn/service/api/records/Container.java | 297 ---
.../service/api/records/ContainerState.java | 30 -
.../hadoop/yarn/service/api/records/Error.java | 129 --
.../service/api/records/PlacementPolicy.java | 102 --
.../service/api/records/ReadinessCheck.java | 175 --
.../yarn/service/api/records/Resource.java | 159 --
.../yarn/service/client/ClientAMProxy.java | 49 -
.../hadoop/yarn/service/client/ServiceCLI.java | 104 --
.../yarn/service/client/ServiceClient.java | 872 ---------
.../client/params/AbstractActionArgs.java | 158 --
.../client/params/AbstractArgsDelegate.java | 28 -
.../AbstractClusterBuildingActionArgs.java | 58 -
.../service/client/params/ActionBuildArgs.java | 31 -
.../service/client/params/ActionClientArgs.java | 71 -
.../service/client/params/ActionCreateArgs.java | 33 -
.../client/params/ActionDependencyArgs.java | 65 -
.../client/params/ActionDestroyArgs.java | 37 -
.../service/client/params/ActionExistsArgs.java | 49 -
.../service/client/params/ActionFlexArgs.java | 50 -
.../service/client/params/ActionFreezeArgs.java | 56 -
.../service/client/params/ActionHelpArgs.java | 44 -
.../service/client/params/ActionKDiagArgs.java | 76 -
.../service/client/params/ActionKeytabArgs.java | 76 -
.../service/client/params/ActionListArgs.java | 76 -
.../client/params/ActionRegistryArgs.java | 218 ---
.../client/params/ActionResolveArgs.java | 153 --
.../client/params/ActionResourceArgs.java | 70 -
.../service/client/params/ActionStatusArgs.java | 51 -
.../service/client/params/ActionThawArgs.java | 67 -
.../service/client/params/ActionTokensArgs.java | 78 -
.../service/client/params/ActionUpdateArgs.java | 32 -
.../yarn/service/client/params/ArgOps.java | 156 --
.../yarn/service/client/params/Arguments.java | 103 --
.../yarn/service/client/params/ClientArgs.java | 252 ---
.../yarn/service/client/params/CommonArgs.java | 282 ---
.../client/params/ComponentArgsDelegate.java | 52 -
.../client/params/DontSplitArguments.java | 34 -
.../client/params/LaunchArgsAccessor.java | 30 -
.../client/params/LaunchArgsDelegate.java | 51 -
.../client/params/OptionArgsDelegate.java | 66 -
.../client/params/PathArgumentConverter.java | 34 -
.../service/client/params/SliderAMArgs.java | 57 -
.../client/params/SliderAMCreateAction.java | 73 -
.../service/client/params/SliderActions.java | 82 -
.../service/client/params/WaitArgsDelegate.java | 42 -
.../service/client/params/WaitTimeAccessor.java | 24 -
.../service/compinstance/ComponentInstance.java | 493 -----
.../compinstance/ComponentInstanceEvent.java | 58 -
.../ComponentInstanceEventType.java | 27 -
.../compinstance/ComponentInstanceId.java | 91 -
.../compinstance/ComponentInstanceState.java | 26 -
.../yarn/service/component/Component.java | 493 -----
.../yarn/service/component/ComponentEvent.java | 83 -
.../service/component/ComponentEventType.java | 26 -
.../yarn/service/component/ComponentState.java | 25 -
.../yarn/service/conf/RestApiConstants.java | 43 -
.../yarn/service/conf/SliderExitCodes.java | 88 -
.../yarn/service/conf/YarnServiceConf.java | 97 -
.../yarn/service/conf/YarnServiceConstants.java | 90 -
.../containerlaunch/AbstractLauncher.java | 271 ---
.../containerlaunch/ClasspathConstructor.java | 172 --
.../containerlaunch/CommandLineBuilder.java | 86 -
.../containerlaunch/ContainerLaunchService.java | 101 --
.../containerlaunch/CredentialUtils.java | 319 ----
.../containerlaunch/JavaCommandLineBuilder.java | 181 --
.../exceptions/BadClusterStateException.java | 36 -
.../BadCommandArgumentsException.java | 30 -
.../service/exceptions/BadConfigException.java | 39 -
.../yarn/service/exceptions/ErrorStrings.java | 57 -
.../service/exceptions/ExitCodeProvider.java | 32 -
.../service/exceptions/LauncherExitCodes.java | 196 --
.../exceptions/RestApiErrorMessages.java | 92 -
.../exceptions/ServiceLaunchException.java | 73 -
.../service/exceptions/SliderException.java | 66 -
.../yarn/service/exceptions/UsageException.java | 34 -
.../pb/client/ClientAMProtocolPBClientImpl.java | 91 -
.../impl/pb/service/ClientAMProtocolPB.java | 29 -
.../service/ClientAMProtocolPBServiceImpl.java | 70 -
.../yarn/service/metrics/ServiceMetrics.java | 101 --
.../provider/AbstractClientProvider.java | 122 --
.../provider/AbstractProviderService.java | 109 --
.../yarn/service/provider/ProviderFactory.java | 76 -
.../yarn/service/provider/ProviderService.java | 39 -
.../yarn/service/provider/ProviderUtils.java | 402 -----
.../defaultImpl/DefaultClientProvider.java | 46 -
.../defaultImpl/DefaultProviderFactory.java | 51 -
.../defaultImpl/DefaultProviderService.java | 36 -
.../provider/docker/DockerClientProvider.java | 53 -
.../service/provider/docker/DockerKeys.java | 30 -
.../provider/docker/DockerProviderFactory.java | 52 -
.../provider/docker/DockerProviderService.java | 57 -
.../provider/tarball/TarballClientProvider.java | 65 -
.../tarball/TarballProviderFactory.java | 52 -
.../tarball/TarballProviderService.java | 48 -
.../registry/CustomRegistryConstants.java | 57 -
.../registry/YarnRegistryViewForProviders.java | 225 ---
.../yarn/service/rest/BaseRestClient.java | 149 --
.../yarn/service/rest/ExceptionConverter.java | 128 --
.../hadoop/yarn/service/rest/HttpVerb.java | 57 -
.../rest/SliderURLConnectionFactory.java | 176 --
.../yarn/service/rest/UgiJerseyBinding.java | 153 --
.../service/rest/UrlConnectionOperations.java | 83 -
.../service/servicemonitor/ServiceMonitor.java | 148 --
.../service/servicemonitor/probe/HttpProbe.java | 110 --
.../servicemonitor/probe/LogEntryBuilder.java | 76 -
.../servicemonitor/probe/MonitorKeys.java | 66 -
.../servicemonitor/probe/MonitorUtils.java | 84 -
.../service/servicemonitor/probe/PortProbe.java | 98 -
.../service/servicemonitor/probe/Probe.java | 100 --
.../servicemonitor/probe/ProbeStatus.java | 160 --
.../timelineservice/ServiceMetricsSink.java | 102 --
.../ServiceTimelineEntityType.java | 39 -
.../timelineservice/ServiceTimelineEvent.java | 34 -
.../ServiceTimelineMetricsConstants.java | 92 -
.../ServiceTimelinePublisher.java | 368 ----
.../service/timelineservice/package-info.java | 27 -
.../utils/ApplicationReportSerDeser.java | 56 -
.../service/utils/ClientRegistryBinder.java | 201 ---
.../hadoop/yarn/service/utils/Comparators.java | 62 -
.../hadoop/yarn/service/utils/ConfigHelper.java | 157 --
.../hadoop/yarn/service/utils/ConfigUtils.java | 97 -
.../yarn/service/utils/CoreFileSystem.java | 521 ------
.../hadoop/yarn/service/utils/Duration.java | 109 --
.../hadoop/yarn/service/utils/JsonSerDeser.java | 249 ---
.../yarn/service/utils/KerberosDiags.java | 680 -------
.../yarn/service/utils/PatternValidator.java | 58 -
.../hadoop/yarn/service/utils/PortScanner.java | 113 --
.../service/utils/PublishedConfiguration.java | 196 --
.../utils/PublishedConfigurationOutputter.java | 212 ---
.../utils/SerializedApplicationReport.java | 98 -
.../yarn/service/utils/ServiceApiUtil.java | 443 -----
.../service/utils/ServiceRegistryUtils.java | 71 -
.../yarn/service/utils/SliderFileSystem.java | 51 -
.../hadoop/yarn/service/utils/SliderUtils.java | 1699 ------------------
.../yarn/service/utils/ZookeeperUtils.java | 146 --
.../src/main/proto/ClientAMProtocol.proto | 56 -
.../hadoop/yarn/service/MockServiceAM.java | 221 ---
.../hadoop/yarn/service/ServiceTestUtils.java | 59 -
.../hadoop/yarn/service/TestServiceApiUtil.java | 529 ------
.../yarn/service/TestYarnNativeServices.java | 472 -----
.../client/TestBuildExternalComponents.java | 128 --
.../yarn/service/client/TestServiceCLI.java | 139 --
.../yarn/service/conf/ExampleAppJson.java | 65 -
.../yarn/service/conf/TestAppJsonResolve.java | 224 ---
.../service/conf/TestLoadExampleAppJson.java | 78 -
.../service/conf/TestValidateServiceNames.java | 123 --
.../providers/TestAbstractClientProvider.java | 118 --
.../service/providers/TestProviderFactory.java | 76 -
.../servicemonitor/TestServiceMonitor.java | 104 --
.../TestServiceTimelinePublisher.java | 293 ---
.../src/test/resources/example-app.json | 15 -
.../src/test/resources/log4j.properties | 66 -
.../service/conf/examples/app-override.json | 72 -
.../hadoop/yarn/service/conf/examples/app.json | 47 -
.../yarn/service/conf/examples/default.json | 16 -
.../yarn/service/conf/examples/external0.json | 8 -
.../yarn/service/conf/examples/external1.json | 30 -
.../yarn/service/conf/examples/external2.json | 22 -
.../yarn/service/provider/docker/appConfig.json | 42 -
.../yarn/service/provider/docker/resources.json | 16 -
.../yarn/service/provider/docker/test.template | 16 -
.../src/test/resources/yarn-site.xml | 19 -
.../hadoop-yarn-slider/pom.xml | 39 -
.../hadoop-yarn-applications/pom.xml | 2 +-
.../native-services/NativeServicesDiscovery.md | 14 +
.../native-services/NativeServicesIntro.md | 13 +
hadoop-yarn-project/hadoop-yarn/pom.xml | 2 +-
371 files changed, 23798 insertions(+), 24702 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 4483fa8..8aeeabd 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -87,19 +87,19 @@
</includes>
</fileSet>
<fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target</directory>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
<includes>
<include>*-sources.jar</include>
</includes>
</fileSet>
<fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf</directory>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf</directory>
<outputDirectory>etc/hadoop</outputDirectory>
</fileSet>
<fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target/hadoop-yarn-slider-core-${project.version}</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/lib/slider</outputDirectory>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target/hadoop-yarn-services-core-${project.version}</directory>
+ <outputDirectory>/share/hadoop/${hadoop.component}/lib/services</outputDirectory>
</fileSet>
<fileSet>
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml
index 5de45a9..1b81f98 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml
@@ -17,7 +17,7 @@
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
- <id>hadoop-yarn-slider-dist</id>
+ <id>hadoop-yarn-services-dist</id>
<formats>
<format>dir</format>
</formats>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 3c8e1c3..426dd62 100755
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -426,7 +426,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-slider-core</artifactId>
+ <artifactId>hadoop-yarn-services-core</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index c455b29..c93ed41 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -48,7 +48,7 @@ function hadoop_usage
hadoop_add_subcommand "rmadmin" admin "admin tools"
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
- hadoop_add_subcommand "servicesapi" "run yarn-native-service rest server"
+ hadoop_add_subcommand "apiserver" "run yarn-native-service rest server"
hadoop_add_subcommand "sharedcachemanager" admin "run the SharedCacheManager daemon"
hadoop_add_subcommand "service" "run a service"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
@@ -147,14 +147,14 @@ function yarncmd_case
scmadmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
;;
- servicesapi)
+ apiserver)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services"'/*'
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*'
- HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApplicationApiWebApp'
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApiServerWebApp'
local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
-${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
+${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services,\
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
@@ -167,11 +167,11 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
;;
service)
- hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services"'/*'
HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.client.ServiceCLI'
local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
-${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
+${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services,\
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index 02aec3b..90a87bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -159,4 +159,4 @@
#
# See ResourceManager for some examples
#
-#export YARN_SERVICESAPI_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-servicesapi.log-$(date +'%Y%m%d%H%M')"
+#export YARN_APISERVER_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-apiserver.log-$(date +'%Y%m%d%H%M')"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index fb2f2ba..7d9f15c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -91,7 +91,7 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-slider-core</artifactId>
+ <artifactId>hadoop-yarn-services-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
new file mode 100644
index 0000000..f55e3f1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.webapp;
+
+import com.google.inject.Singleton;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
+
+/**
+ * The rest API endpoints for users to manage services on YARN.
+ */
+@Singleton
+@Path(CONTEXT_ROOT)
+public class ApiServer {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ApiServer.class);
+ private static Configuration YARN_CONFIG = new YarnConfiguration();
+ private static ServiceClient SERVICE_CLIENT;
+
+ static {
+ init();
+ }
+
+ // initialize all the common resources - order is important
+ private static void init() {
+ SERVICE_CLIENT = new ServiceClient();
+ SERVICE_CLIENT.init(YARN_CONFIG);
+ SERVICE_CLIENT.start();
+ }
+
+ @GET
+ @Path(VERSION)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
+ public Response getVersion() {
+ String version = VersionInfo.getBuildVersion();
+ LOG.info(version);
+ return Response.ok(version).build();
+ }
+
+ @POST
+ @Path(SERVICE_ROOT_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response createService(Service service) {
+ LOG.info("POST: createService = {}", service);
+ ServiceStatus serviceStatus = new ServiceStatus();
+ try {
+ ApplicationId applicationId = SERVICE_CLIENT.actionCreate(service);
+ LOG.info("Successfully created service " + service.getName()
+ + " applicationId = " + applicationId);
+ serviceStatus.setState(ServiceState.ACCEPTED);
+ serviceStatus.setUri(
+ CONTEXT_ROOT + SERVICE_ROOT_PATH + "/" + service
+ .getName());
+ return Response.status(Status.CREATED).entity(serviceStatus).build();
+ } catch (IllegalArgumentException e) {
+ serviceStatus.setDiagnostics(e.getMessage());
+ return Response.status(Status.BAD_REQUEST).entity(serviceStatus)
+ .build();
+ } catch (Exception e) {
+ String message = "Failed to create service " + service.getName();
+ LOG.error(message, e);
+ serviceStatus.setDiagnostics(message + ": " + e.getMessage());
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(serviceStatus).build();
+ }
+ }
+
+ @GET
+ @Path(SERVICE_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response getService(@PathParam(SERVICE_NAME) String appName) {
+ LOG.info("GET: getService for appName = {}", appName);
+ ServiceStatus serviceStatus = new ServiceStatus();
+ try {
+ Service app = SERVICE_CLIENT.getStatus(appName);
+ return Response.ok(app).build();
+ } catch (IllegalArgumentException e) {
+ serviceStatus.setDiagnostics(e.getMessage());
+ serviceStatus.setCode(ERROR_CODE_APP_NAME_INVALID);
+ return Response.status(Status.NOT_FOUND).entity(serviceStatus)
+ .build();
+ } catch (Exception e) {
+ LOG.error("Get service failed", e);
+ serviceStatus
+ .setDiagnostics("Failed to retrieve service: " + e.getMessage());
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(serviceStatus).build();
+ }
+ }
+
+ @DELETE
+ @Path(SERVICE_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response deleteService(@PathParam(SERVICE_NAME) String appName) {
+ LOG.info("DELETE: deleteService for appName = {}", appName);
+ return stopService(appName, true);
+ }
+
+ private Response stopService(String appName, boolean destroy) {
+ try {
+ SERVICE_CLIENT.actionStop(appName, destroy);
+ if (destroy) {
+ SERVICE_CLIENT.actionDestroy(appName);
+ LOG.info("Successfully deleted service {}", appName);
+ } else {
+ LOG.info("Successfully stopped service {}", appName);
+ }
+ return Response.status(Status.NO_CONTENT).build();
+ } catch (ApplicationNotFoundException e) {
+ ServiceStatus serviceStatus = new ServiceStatus();
+ serviceStatus.setDiagnostics(
+ "Service " + appName + " not found " + e.getMessage());
+ return Response.status(Status.NOT_FOUND).entity(serviceStatus)
+ .build();
+ } catch (Exception e) {
+ ServiceStatus serviceStatus = new ServiceStatus();
+ serviceStatus.setDiagnostics(e.getMessage());
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(serviceStatus).build();
+ }
+ }
+
+ @PUT
+ @Path(COMPONENT_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
+ public Response updateComponent(@PathParam(SERVICE_NAME) String appName,
+ @PathParam(COMPONENT_NAME) String componentName, Component component) {
+
+ if (component.getNumberOfContainers() < 0) {
+ return Response.status(Status.BAD_REQUEST).entity(
+ "Service = " + appName + ", Component = " + component.getName()
+ + ": Invalid number of containers specified " + component
+ .getNumberOfContainers()).build();
+ }
+ try {
+ Map<String, Long> original = SERVICE_CLIENT.flexByRestService(appName,
+ Collections.singletonMap(component.getName(),
+ component.getNumberOfContainers()));
+ return Response.ok().entity(
+ "Updating component " + componentName + " size from " + original
+ .get(componentName) + " to " + component.getNumberOfContainers())
+ .build();
+ } catch (YarnException | IOException e) {
+ ServiceStatus status = new ServiceStatus();
+ status.setDiagnostics(e.getMessage());
+ return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status)
+ .build();
+ }
+ }
+
+ @PUT
+ @Path(SERVICE_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response updateService(@PathParam(SERVICE_NAME) String appName,
+ Service updateServiceData) {
+ LOG.info("PUT: updateService for app = {} with data = {}", appName,
+ updateServiceData);
+
+ // Ignore the app name provided in updateServiceData and always use appName
+ // path param
+ updateServiceData.setName(appName);
+
+ // For STOP the app should be running. If already stopped then this
+ // operation will be a no-op. For START it should be in stopped state.
+ // If already running then this operation will be a no-op.
+ if (updateServiceData.getState() != null
+ && updateServiceData.getState() == ServiceState.STOPPED) {
+ return stopService(appName, false);
+ }
+
+ // If a START is requested
+ if (updateServiceData.getState() != null
+ && updateServiceData.getState() == ServiceState.STARTED) {
+ return startService(appName);
+ }
+
+ // If new lifetime value specified then update it
+ if (updateServiceData.getLifetime() != null
+ && updateServiceData.getLifetime() > 0) {
+ return updateLifetime(appName, updateServiceData);
+ }
+
+ // flex a single component app
+ if (updateServiceData.getNumberOfContainers() != null && !ServiceApiUtil
+ .hasComponent(updateServiceData)) {
+ Component defaultComp = ServiceApiUtil.createDefaultComponent(updateServiceData);
+ return updateComponent(updateServiceData.getName(), defaultComp.getName(),
+ defaultComp);
+ }
+
+ // If nothing happens consider it a no-op
+ return Response.status(Status.NO_CONTENT).build();
+ }
+
+ private Response updateLifetime(String appName, Service updateAppData) {
+ try {
+ String newLifeTime =
+ SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime());
+ return Response.ok("Service " + appName + " lifeTime is successfully updated to "
+ + updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build();
+ } catch (Exception e) {
+ String message =
+ "Failed to update service (" + appName + ") lifetime ("
+ + updateAppData.getLifetime() + ")";
+ LOG.error(message, e);
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(message + " : " + e.getMessage()).build();
+ }
+ }
+
+ private Response startService(String appName) {
+ try {
+ SERVICE_CLIENT.actionStart(appName);
+ LOG.info("Successfully started service " + appName);
+ return Response.ok("Service " + appName + " is successfully started").build();
+ } catch (Exception e) {
+ String message = "Failed to start service " + appName;
+ LOG.info(message, e);
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(message + ": " + e.getMessage()).build();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
new file mode 100644
index 0000000..b226df7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.webapp;
+
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.eclipse.jetty.webapp.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
+
+/**
+ * This class launches the web service using Hadoop HttpServer2 (which uses
+ * an embedded Jetty container). This is the entry point to your service.
+ * The Java command used to launch this app should call the main method.
+ */
+public class ApiServerWebApp extends AbstractService {
+ private static final Logger logger = LoggerFactory
+ .getLogger(ApiServerWebApp.class);
+ private static final String SEP = ";";
+
+ // REST API server for YARN native services
+ private HttpServer2 apiServer;
+ private InetSocketAddress bindAddress;
+
+ public static void main(String[] args) throws IOException {
+ ApiServerWebApp apiWebApp = new ApiServerWebApp();
+ try {
+ apiWebApp.startWebApp();
+ } catch (Exception e) {
+ apiWebApp.close();
+ }
+ }
+
+ public ApiServerWebApp() {
+ super(ApiServerWebApp.class.getName());
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ bindAddress = getConfig().getSocketAddr(API_SERVER_ADDRESS,
+ DEFAULT_API_SERVER_ADDRESS , DEFAULT_API_SERVER_PORT);
+ logger.info("YARN API server running on " + bindAddress);
+ if (UserGroupInformation.isSecurityEnabled()) {
+ doSecureLogin(getConfig());
+ }
+ startWebApp();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (apiServer != null) {
+ apiServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ private void doSecureLogin(org.apache.hadoop.conf.Configuration conf)
+ throws IOException {
+ SecurityUtil.login(conf, YarnConfiguration.RM_KEYTAB,
+ YarnConfiguration.RM_PRINCIPAL, bindAddress.getHostName());
+ addFilters(conf);
+ }
+
+ private void addFilters(org.apache.hadoop.conf.Configuration conf) {
+ // Always load pseudo authentication filter to parse "user.name" in an URL
+ // to identify a HTTP request's user.
+ boolean hasHadoopAuthFilterInitializer = false;
+ String filterInitializerConfKey = "hadoop.http.filter.initializers";
+ Class<?>[] initializersClasses =
+ conf.getClasses(filterInitializerConfKey);
+ List<String> targets = new ArrayList<String>();
+ if (initializersClasses != null) {
+ for (Class<?> initializer : initializersClasses) {
+ if (initializer.getName().equals(
+ AuthenticationFilterInitializer.class.getName())) {
+ hasHadoopAuthFilterInitializer = true;
+ break;
+ }
+ targets.add(initializer.getName());
+ }
+ }
+ if (!hasHadoopAuthFilterInitializer) {
+ targets.add(AuthenticationFilterInitializer.class.getName());
+ conf.set(filterInitializerConfKey, StringUtils.join(",", targets));
+ }
+ }
+
+ private void startWebApp() throws IOException {
+ URI uri = URI.create("http://" + NetUtils.getHostPortString(bindAddress));
+
+ apiServer = new HttpServer2.Builder()
+ .setName("api-server")
+ .setConf(getConfig())
+ .setSecurityEnabled(UserGroupInformation.isSecurityEnabled())
+ .setUsernameConfKey(RM_WEBAPP_SPNEGO_USER_NAME_KEY)
+ .setKeytabConfKey(RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
+ .addEndpoint(uri).build();
+
+ String apiPackages =
+ ApiServer.class.getPackage().getName() + SEP
+ + GenericExceptionHandler.class.getPackage().getName() + SEP
+ + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
+ apiServer.addJerseyResourcePackage(apiPackages, "/*");
+
+ try {
+ logger.info("Service starting up. Logging start...");
+ apiServer.start();
+ logger.info("Server status = {}", apiServer.toString());
+ for (Configuration conf : apiServer.getWebAppContext()
+ .getConfigurations()) {
+ logger.info("Configurations = {}", conf);
+ }
+ logger.info("Context Path = {}", Collections.singletonList(
+ apiServer.getWebAppContext().getContextPath()));
+ logger.info("ResourceBase = {}", Collections.singletonList(
+ apiServer.getWebAppContext().getResourceBase()));
+ logger.info("War = {}", Collections
+ .singletonList(apiServer.getWebAppContext().getWar()));
+ } catch (Exception ex) {
+ logger.error("Hadoop HttpServer2 App **failed**", ex);
+ throw ex;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java
deleted file mode 100644
index 30fc5ef..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.webapp;
-
-import com.google.inject.Singleton;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ApplicationState;
-import org.apache.hadoop.yarn.service.api.records.ApplicationStatus;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-
-import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
-
-@Singleton
-@Path(CONTEXT_ROOT)
-public class ApplicationApiService {
- private static final Logger LOG =
- LoggerFactory.getLogger(ApplicationApiService.class);
- private static Configuration YARN_CONFIG = new YarnConfiguration();
- private static ServiceClient SERVICE_CLIENT;
-
- static {
- init();
- }
-
- // initialize all the common resources - order is important
- private static void init() {
- SERVICE_CLIENT = new ServiceClient();
- SERVICE_CLIENT.init(YARN_CONFIG);
- SERVICE_CLIENT.start();
- }
-
- @GET
- @Path(VERSION)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
- public Response getVersion() {
- String version = VersionInfo.getBuildVersion();
- LOG.info(version);
- return Response.ok(version).build();
- }
-
- @POST
- @Path(APP_ROOT_PATH)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON })
- public Response createApplication(Application application) {
- LOG.info("POST: createApplication = {}", application);
- ApplicationStatus applicationStatus = new ApplicationStatus();
- try {
- ApplicationId applicationId = SERVICE_CLIENT.actionCreate(application);
- LOG.info("Successfully created application " + application.getName()
- + " applicationId = " + applicationId);
- applicationStatus.setState(ApplicationState.ACCEPTED);
- applicationStatus.setUri(
- CONTEXT_ROOT + APP_ROOT_PATH + "/" + application
- .getName());
- return Response.status(Status.CREATED).entity(applicationStatus).build();
- } catch (IllegalArgumentException e) {
- applicationStatus.setDiagnostics(e.getMessage());
- return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
- .build();
- } catch (Exception e) {
- String message = "Failed to create application " + application.getName();
- LOG.error(message, e);
- applicationStatus.setDiagnostics(message + ": " + e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(applicationStatus).build();
- }
- }
-
- @GET
- @Path(APP_PATH)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON })
- public Response getApplication(@PathParam(APP_NAME) String appName) {
- LOG.info("GET: getApplication for appName = {}", appName);
- ApplicationStatus applicationStatus = new ApplicationStatus();
-
- // app name validation
- if (!SliderUtils.isClusternameValid(appName)) {
- applicationStatus.setDiagnostics("Invalid application name: " + appName);
- applicationStatus.setCode(ERROR_CODE_APP_NAME_INVALID);
- return Response.status(Status.NOT_FOUND).entity(applicationStatus)
- .build();
- }
-
- try {
- Application app = SERVICE_CLIENT.getStatus(appName);
- return Response.ok(app).build();
- } catch (Exception e) {
- LOG.error("Get application failed", e);
- applicationStatus
- .setDiagnostics("Failed to retrieve application: " + e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(applicationStatus).build();
- }
- }
-
- @DELETE
- @Path(APP_PATH)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON })
- public Response deleteApplication(@PathParam(APP_NAME) String appName) {
- LOG.info("DELETE: deleteApplication for appName = {}", appName);
- return stopApplication(appName, true);
- }
-
- private Response stopApplication(String appName, boolean destroy) {
- try {
- SERVICE_CLIENT.actionStop(appName, destroy);
- if (destroy) {
- SERVICE_CLIENT.actionDestroy(appName);
- LOG.info("Successfully deleted application {}", appName);
- } else {
- LOG.info("Successfully stopped application {}", appName);
- }
- return Response.status(Status.NO_CONTENT).build();
- } catch (ApplicationNotFoundException e) {
- ApplicationStatus applicationStatus = new ApplicationStatus();
- applicationStatus.setDiagnostics(
- "Application " + appName + " not found " + e.getMessage());
- return Response.status(Status.NOT_FOUND).entity(applicationStatus)
- .build();
- } catch (Exception e) {
- ApplicationStatus applicationStatus = new ApplicationStatus();
- applicationStatus.setDiagnostics(e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(applicationStatus).build();
- }
- }
-
- @PUT
- @Path(COMPONENT_PATH)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
- public Response updateComponent(@PathParam(APP_NAME) String appName,
- @PathParam(COMPONENT_NAME) String componentName, Component component) {
-
- if (component.getNumberOfContainers() < 0) {
- return Response.status(Status.BAD_REQUEST).entity(
- "Application = " + appName + ", Component = " + component.getName()
- + ": Invalid number of containers specified " + component
- .getNumberOfContainers()).build();
- }
- try {
- Map<String, Long> original = SERVICE_CLIENT.flexByRestService(appName,
- Collections.singletonMap(component.getName(),
- component.getNumberOfContainers()));
- return Response.ok().entity(
- "Updating component " + componentName + " size from " + original
- .get(componentName) + " to " + component.getNumberOfContainers())
- .build();
- } catch (YarnException | IOException e) {
- ApplicationStatus status = new ApplicationStatus();
- status.setDiagnostics(e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status)
- .build();
- }
- }
-
- @PUT
- @Path(APP_PATH)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON })
- public Response updateApplication(@PathParam(APP_NAME) String appName,
- Application updateAppData) {
- LOG.info("PUT: updateApplication for app = {} with data = {}", appName,
- updateAppData);
-
- // Ignore the app name provided in updateAppData and always use appName
- // path param
- updateAppData.setName(appName);
-
- // For STOP the app should be running. If already stopped then this
- // operation will be a no-op. For START it should be in stopped state.
- // If already running then this operation will be a no-op.
- if (updateAppData.getState() != null
- && updateAppData.getState() == ApplicationState.STOPPED) {
- return stopApplication(appName, false);
- }
-
- // If a START is requested
- if (updateAppData.getState() != null
- && updateAppData.getState() == ApplicationState.STARTED) {
- return startApplication(appName);
- }
-
- // If new lifetime value specified then update it
- if (updateAppData.getLifetime() != null
- && updateAppData.getLifetime() > 0) {
- return updateLifetime(appName, updateAppData);
- }
-
- // flex a single component app
- if (updateAppData.getNumberOfContainers() != null && !ServiceApiUtil
- .hasComponent(updateAppData)) {
- Component defaultComp = ServiceApiUtil.createDefaultComponent(updateAppData);
- return updateComponent(updateAppData.getName(), defaultComp.getName(),
- defaultComp);
- }
-
- // If nothing happens consider it a no-op
- return Response.status(Status.NO_CONTENT).build();
- }
-
- private Response updateLifetime(String appName, Application updateAppData) {
- try {
- String newLifeTime =
- SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime());
- return Response.ok("Application " + appName + " lifeTime is successfully updated to "
- + updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build();
- } catch (Exception e) {
- String message =
- "Failed to update application (" + appName + ") lifetime ("
- + updateAppData.getLifetime() + ")";
- LOG.error(message, e);
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(message + " : " + e.getMessage()).build();
- }
- }
-
- private Response startApplication(String appName) {
- try {
- SERVICE_CLIENT.actionStart(appName);
- LOG.info("Successfully started application " + appName);
- return Response.ok("Application " + appName + " is successfully started").build();
- } catch (Exception e) {
- String message = "Failed to start application " + appName;
- LOG.info(message, e);
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(message + ": " + e.getMessage()).build();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java
deleted file mode 100644
index 7225209..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.webapp;
-
-import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.URI;
-import java.util.Arrays;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.eclipse.jetty.webapp.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class launches the web application using Hadoop HttpServer2 (which uses
- * an embedded Jetty container). This is the entry point to your application.
- * The Java command used to launch this app should call the main method.
- */
-public class ApplicationApiWebApp extends AbstractService {
- private static final Logger logger = LoggerFactory
- .getLogger(ApplicationApiWebApp.class);
- private static final String SEP = ";";
-
- // REST API server for YARN native services
- private HttpServer2 applicationApiServer;
-
- public static void main(String[] args) throws IOException {
- ApplicationApiWebApp apiWebApp = new ApplicationApiWebApp();
- try {
- apiWebApp.startWebApp();
- } catch (Exception e) {
- if (apiWebApp != null) {
- apiWebApp.close();
- }
- }
- }
-
- public ApplicationApiWebApp() {
- super(ApplicationApiWebApp.class.getName());
- }
-
- @Override
- protected void serviceStart() throws Exception {
- startWebApp();
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (applicationApiServer != null) {
- applicationApiServer.stop();
- }
- super.serviceStop();
- }
-
- protected void startWebApp() throws IOException {
- // The port that we should run on can be set into an environment variable
- // Look for that variable and default to 9191 if it isn't there.
- String webPort = System.getenv(PROPERTY_REST_SERVICE_PORT);
- if (StringUtils.isEmpty(webPort)) {
- webPort = "9191";
- }
-
- String webHost = System.getenv(PROPERTY_REST_SERVICE_HOST);
- if (StringUtils.isEmpty(webHost)) {
- webHost = InetAddress.getLocalHost().getHostName();
- }
- logger.info("YARN native services REST API running on host {} and port {}",
- webHost, webPort);
- logger.info("Configuration = {}", getConfig());
-
- applicationApiServer = new HttpServer2.Builder()
- .setName("services-rest-api")
- .addEndpoint(URI.create("http://" + webHost + ":" + webPort)).build();
-
- String apiPackages =
- ApplicationApiService.class.getPackage().getName() + SEP
- + GenericExceptionHandler.class.getPackage().getName() + SEP
- + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
- applicationApiServer.addJerseyResourcePackage(apiPackages, "/*");
-
- try {
- logger.info("Application starting up. Logging start...");
- applicationApiServer.start();
- logger.info("Server status = {}", applicationApiServer.toString());
- for (Configuration conf : applicationApiServer.getWebAppContext()
- .getConfigurations()) {
- logger.info("Configurations = {}", conf);
- }
- logger.info("Context Path = {}", Arrays.asList(applicationApiServer
- .getWebAppContext().getContextPath()));
- logger.info("ResourceBase = {}", Arrays.asList(applicationApiServer
- .getWebAppContext().getResourceBase()));
- logger.info("War = {}",
- Arrays.asList(applicationApiServer.getWebAppContext().getWar()));
- } catch (Exception ex) {
- logger.error("Hadoop HttpServer2 App **failed**", ex);
- throw ex;
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
deleted file mode 100644
index fc57c82..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ /dev/null
@@ -1,2548 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.GlobFilter;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.Slider;
-import org.apache.slider.api.RoleKeys;
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.ClasspathConstructor;
-import org.apache.slider.core.main.LauncherExitCodes;
-import org.apache.slider.server.services.utility.PatternValidator;
-import org.apache.slider.server.services.workflow.ForkedProcessService;
-import org.apache.zookeeper.server.util.KerberosUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintWriter;
-import java.io.Serializable;
-import java.io.StringWriter;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.TimerTask;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.regex.Pattern;
-import java.util.zip.GZIPOutputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
-
-import static org.apache.hadoop.yarn.service.conf.SliderKeys.COMPONENT_SEPARATOR;
-
-/**
- * These are slider-specific Util methods
- */
-public final class SliderUtils {
-
- private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
-
- /**
- * Atomic bool to track whether or not process security has already been
- * turned on (prevents re-entrancy)
- */
- private static final AtomicBoolean processSecurityAlreadyInitialized =
- new AtomicBoolean(false);
- public static final String JAVA_SECURITY_KRB5_REALM =
- "java.security.krb5.realm";
- public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
-
- /**
- * Winutils
- */
- public static final String WINUTILS = "WINUTILS.EXE";
- /**
- * name of openssl program
- */
- public static final String OPENSSL = "openssl";
-
- /**
- * name of python program
- */
- public static final String PYTHON = "python";
-
- /**
- * type of docker standalone application
- */
- public static final String DOCKER = "docker";
- /**
- * type of docker on yarn application
- */
- public static final String DOCKER_YARN = "yarn_docker";
-
- public static final int NODE_LIST_LIMIT = 10;
-
- private SliderUtils() {
- }
-
- /**
- * Implementation of set-ness, groovy definition of true/false for a string
- * @param s string
- * @return true iff the string is neither null nor empty
- */
- public static boolean isUnset(String s) {
- return s == null || s.isEmpty();
- }
-
- public static boolean isSet(String s) {
- return !isUnset(s);
- }
-
- public static boolean isEmpty(Collection l) {
- return l == null || l.isEmpty();
- }
-
- /**
- * Probe for a collection existing and not being empty
- * @param l collection
- * @return true if the reference is valid and it contains entries
- */
-
- public static boolean isNotEmpty(Collection l) {
- return l != null && !l.isEmpty();
- }
-
- /**
- * Probe for a map existing and not being empty
- * @param m map
- * @return true if the reference is valid and it contains map entries
- */
- public static boolean isNotEmpty(Map m) {
- return m != null && !m.isEmpty();
- }
-
- /*
- * Validates whether num is an integer
- * @param num
- * @param msg the message to be shown in exception
- */
- @SuppressWarnings("ResultOfMethodCallIgnored")
- private static void validateNumber(String num, String msg) throws
- BadConfigException {
- try {
- Integer.parseInt(num);
- } catch (NumberFormatException nfe) {
- throw new BadConfigException(msg + num);
- }
- }
-
- /*
- * Translates the trailing JVM heapsize unit: g, G, m, M
- * This assumes designated unit of 'm'
- * @param heapsize
- * @return heapsize in MB
- */
- public static String translateTrailingHeapUnit(String heapsize) throws
- BadConfigException {
- String errMsg = "Bad heapsize: ";
- if (heapsize.endsWith("m") || heapsize.endsWith("M")) {
- String num = heapsize.substring(0, heapsize.length() - 1);
- validateNumber(num, errMsg);
- return num;
- }
- if (heapsize.endsWith("g") || heapsize.endsWith("G")) {
- String num = heapsize.substring(0, heapsize.length() - 1) + "000";
- validateNumber(num, errMsg);
- return num;
- }
- // check if specified heap size is a number
- validateNumber(heapsize, errMsg);
- return heapsize;
- }
-
- /**
- * recursive directory delete
- * @param dir dir to delete
- * @throws IOException on any problem
- */
- public static void deleteDirectoryTree(File dir) throws IOException {
- if (dir.exists()) {
- if (dir.isDirectory()) {
- log.info("Cleaning up {}", dir);
- //delete the children
- File[] files = dir.listFiles();
- if (files == null) {
- throw new IOException("listfiles() failed for " + dir);
- }
- for (File file : files) {
- log.info("deleting {}", file);
- if (!file.delete()) {
- log.warn("Unable to delete " + file);
- }
- }
- if (!dir.delete()) {
- log.warn("Unable to delete " + dir);
- }
- } else {
- throw new IOException("Not a directory " + dir);
- }
- } else {
- //not found, do nothing
- log.debug("No output dir yet");
- }
- }
-
- /**
- * Find a containing JAR
- * @param clazz class to find
- * @return the file
- * @throws IOException any IO problem, including the class not having a
- * classloader
- * @throws FileNotFoundException if the class did not resolve to a file
- */
- public static File findContainingJarOrFail(Class clazz) throws IOException {
- File localFile = SliderUtils.findContainingJar(clazz);
- if (null == localFile) {
- throw new FileNotFoundException("Could not find JAR containing " + clazz);
- }
- return localFile;
- }
-
-
- /**
- * Find a containing JAR
- * @param my_class class to find
- * @return the file or null if it is not found
- * @throws IOException any IO problem, including the class not having a
- * classloader
- */
- public static File findContainingJar(Class my_class) throws IOException {
- ClassLoader loader = my_class.getClassLoader();
- if (loader == null) {
- throw new IOException(
- "Class " + my_class + " does not have a classloader!");
- }
- String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
- Enumeration<URL> urlEnumeration = loader.getResources(class_file);
- for (; urlEnumeration.hasMoreElements(); ) {
- URL url = urlEnumeration.nextElement();
- if ("jar".equals(url.getProtocol())) {
- String toReturn = url.getPath();
- if (toReturn.startsWith("file:")) {
- toReturn = toReturn.substring("file:".length());
- }
- // URLDecoder is a misnamed class, since it actually decodes
- // x-www-form-urlencoded MIME type rather than actual
- // URL encoding (which the file path has). Therefore it would
- // decode +s to ' 's which is incorrect (spaces are actually
- // either unencoded or encoded as "%20"). Replace +s first, so
- // that they are kept sacred during the decoding process.
- toReturn = toReturn.replaceAll("\\+", "%2B");
- toReturn = URLDecoder.decode(toReturn, "UTF-8");
- String jarFilePath = toReturn.replaceAll("!.*$", "");
- return new File(jarFilePath);
- } else {
- log.info("could not locate JAR containing {} URL={}", my_class, url);
- }
- }
- return null;
- }
-
- public static void checkPort(String hostname, int port, int connectTimeout)
- throws IOException {
- InetSocketAddress addr = new InetSocketAddress(hostname, port);
- checkPort(hostname, addr, connectTimeout);
- }
-
- @SuppressWarnings("SocketOpenedButNotSafelyClosed")
- public static void checkPort(String name,
- InetSocketAddress address,
- int connectTimeout)
- throws IOException {
- try(Socket socket = new Socket()) {
- socket.connect(address, connectTimeout);
- } catch (Exception e) {
- throw new IOException("Failed to connect to " + name
- + " at " + address
- + " after " + connectTimeout + "milliseconds"
- + ": " + e,
- e);
- }
- }
-
- public static void checkURL(String name, String url, int timeout) throws
- IOException {
- InetSocketAddress address = NetUtils.createSocketAddr(url);
- checkPort(name, address, timeout);
- }
-
- /**
- * A required file
- * @param role role of the file (for errors)
- * @param filename the filename
- * @throws ExitUtil.ExitException if the file is missing
- * @return the file
- */
- public static File requiredFile(String filename, String role) throws
- IOException {
- if (filename.isEmpty()) {
- throw new ExitUtil.ExitException(-1, role + " file not defined");
- }
- File file = new File(filename);
- if (!file.exists()) {
- throw new ExitUtil.ExitException(-1,
- role + " file not found: " +
- file.getCanonicalPath());
- }
- return file;
- }
-
- private static final PatternValidator clusternamePattern
- = new PatternValidator("[a-z][a-z0-9_-]*");
-
- private static final PatternValidator compNamePattern
- = new PatternValidator("[a-z][a-z0-9-]*");
-
- public static void validateCompName(String compName) {
- compNamePattern.validate(compName);
- }
-
- /**
- * Normalize a cluster name then verify that it is valid
- * @param name proposed cluster name
- * @return true iff it is valid
- */
- public static boolean isClusternameValid(String name) {
- return name != null && clusternamePattern.matches(name);
- }
-
- public static boolean oldIsClusternameValid(String name) {
- if (name == null || name.isEmpty()) {
- return false;
- }
- int first = name.charAt(0);
- if (0 == (Character.getType(first) & Character.LOWERCASE_LETTER)) {
- return false;
- }
-
- for (int i = 0; i < name.length(); i++) {
- int elt = (int) name.charAt(i);
- int t = Character.getType(elt);
- if (0 == (t & Character.LOWERCASE_LETTER)
- && 0 == (t & Character.DECIMAL_DIGIT_NUMBER)
- && elt != '-'
- && elt != '_') {
- return false;
- }
- if (!Character.isLetterOrDigit(elt) && elt != '-' && elt != '_') {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Copy a directory to a new FS -both paths must be qualified. If
- * a directory needs to be created, supplied permissions can override
- * the default values. Existing directories are not touched
- * @param conf conf file
- * @param srcDirPath src dir
- * @param destDirPath dest dir
- * @param permission permission for the dest directory; null means "default"
- * @return # of files copies
- */
- public static int copyDirectory(Configuration conf,
- Path srcDirPath,
- Path destDirPath,
- FsPermission permission) throws
- IOException,
- BadClusterStateException {
- FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf);
- FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf);
- //list all paths in the src.
- if (!srcFS.exists(srcDirPath)) {
- throw new FileNotFoundException("Source dir not found " + srcDirPath);
- }
- if (!srcFS.isDirectory(srcDirPath)) {
- throw new FileNotFoundException(
- "Source dir not a directory " + srcDirPath);
- }
- GlobFilter dotFilter = new GlobFilter("[!.]*");
- FileStatus[] entries = srcFS.listStatus(srcDirPath, dotFilter);
- int srcFileCount = entries.length;
- if (srcFileCount == 0) {
- return 0;
- }
- if (permission == null) {
- permission = FsPermission.getDirDefault();
- }
- if (!destFS.exists(destDirPath)) {
- new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath,
- permission);
- }
- Path[] sourcePaths = new Path[srcFileCount];
- for (int i = 0; i < srcFileCount; i++) {
- FileStatus e = entries[i];
- Path srcFile = e.getPath();
- if (srcFS.isDirectory(srcFile)) {
- String msg = "Configuration dir " + srcDirPath
- + " contains a directory " + srcFile;
- log.warn(msg);
- throw new IOException(msg);
- }
- log.debug("copying src conf file {}", srcFile);
- sourcePaths[i] = srcFile;
- }
- log.debug("Copying {} files from {} to dest {}", srcFileCount,
- srcDirPath,
- destDirPath);
- FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf);
- return srcFileCount;
- }
-
- /**
- * Copy a file to a new FS -both paths must be qualified.
- * @param conf conf file
- * @param srcFile src file
- * @param destFile dest file
- */
- public static void copy(Configuration conf,
- Path srcFile,
- Path destFile) throws
- IOException,
- BadClusterStateException {
- FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
- //list all paths in the src.
- if (!srcFS.exists(srcFile)) {
- throw new FileNotFoundException("Source file not found " + srcFile);
- }
- if (!srcFS.isFile(srcFile)) {
- throw new FileNotFoundException(
- "Source file not a file " + srcFile);
- }
- FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
- FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
- }
-
- public static String stringify(Throwable t) {
- StringWriter sw = new StringWriter();
- sw.append(t.toString()).append('\n');
- t.printStackTrace(new PrintWriter(sw));
- return sw.toString();
- }
-
- /**
- * Extract the first line of a multi-line string. This is typically used to
- * prune the stack trace appended to the end of exception messages returned by
- * YARN in AMRMClientAsync callbacks.
- *
- * @param msg
- * message string (most likely multi-lines)
- * @return the first line of a multi-line string or the original string if it
- * is a null, empty or single-line
- */
- public static String extractFirstLine(String msg) {
- if (StringUtils.isNotBlank(msg)) {
- int newlineIndex = msg.indexOf(System.lineSeparator());
- if (newlineIndex != -1) {
- msg = msg.substring(0, newlineIndex);
- }
- }
- return msg;
- }
-
- /**
- * Create a configuration with Slider-specific tuning.
- * This is done rather than doing custom configs.
- * @return the config
- */
- public static YarnConfiguration createConfiguration() {
- YarnConfiguration conf = new YarnConfiguration();
- patchConfiguration(conf);
- return conf;
- }
-
- /**
- * Take an existing conf and patch it for Slider's needs. Useful
- * in Service.init & RunService methods where a shared config is being
- * passed in
- * @param conf configuration
- * @return the patched configuration
- */
- public static Configuration patchConfiguration(Configuration conf) {
-
- //if the fallback option is NOT set, enable it.
- //if it is explicitly set to anything -leave alone
- if (conf.get(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH) == null) {
- conf.set(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH, "true");
- }
- return conf;
- }
-
- /**
- * Take a collection, return a list containing the string value of every
- * element in the collection.
- * @param c collection
- * @return a stringified list
- */
- public static List<String> collectionToStringList(Collection c) {
- List<String> l = new ArrayList<>(c.size());
- for (Object o : c) {
- l.add(o.toString());
- }
- return l;
- }
-
- /**
- * Join an collection of objects with a separator that appears after every
- * instance in the list -including at the end
- * @param collection collection to call toString() on each element
- * @param separator separator string
- * @return the joined entries
- */
- public static String join(Collection collection, String separator) {
- return join(collection, separator, true);
- }
-
- /**
- * Join an collection of objects with a separator that appears after every
- * instance in the list -optionally at the end
- * @param collection collection to call toString() on each element
- * @param separator separator string
- * @param trailing add a trailing entry or not
- * @return the joined entries
- */
- public static String join(Collection collection,
- String separator,
- boolean trailing) {
- StringBuilder b = new StringBuilder();
- // fast return on empty collection
- if (collection.isEmpty()) {
- return trailing ? separator : "";
- }
- for (Object o : collection) {
- b.append(o);
- b.append(separator);
- }
- int length = separator.length();
- String s = b.toString();
- return (trailing || s.isEmpty()) ?
- s : (b.substring(0, b.length() - length));
- }
-
- /**
- * Join an array of strings with a separator that appears after every
- * instance in the list -including at the end
- * @param collection strings
- * @param separator separator string
- * @return the joined entries
- */
- public static String join(String[] collection, String separator) {
- return join(collection, separator, true);
-
-
- }
-
- /**
- * Join an array of strings with a separator that appears after every
- * instance in the list -optionally at the end
- * @param collection strings
- * @param separator separator string
- * @param trailing add a trailing entry or not
- * @return the joined entries
- */
- public static String join(String[] collection, String separator,
- boolean trailing) {
- return join(Arrays.asList(collection), separator, trailing);
- }
-
- /**
- * Join an array of strings with a separator that appears after every
- * instance in the list -except at the end
- * @param collection strings
- * @param separator separator string
- * @return the list
- */
- public static String joinWithInnerSeparator(String separator,
- Object... collection) {
- StringBuilder b = new StringBuilder();
- boolean first = true;
-
- for (Object o : collection) {
- if (first) {
- first = false;
- } else {
- b.append(separator);
- }
- b.append(o.toString());
- b.append(separator);
- }
- return b.toString();
- }
-
- /**
- * Resolve a mandatory environment variable
- * @param key env var
- * @return the resolved value
- * @throws BadClusterStateException
- */
- public static String mandatoryEnvVariable(String key) throws
- BadClusterStateException {
- String v = System.getenv(key);
- if (v == null) {
- throw new BadClusterStateException("Missing Environment variable " + key);
- }
- return v;
- }
-
- public static String appReportToString(ApplicationReport r,
- String separator) {
- StringBuilder builder = new StringBuilder(512);
- builder.append("application ")
- .append(
- r.getName())
- .append("/")
- .append(r.getApplicationType())
- .append(separator);
- Set<String> tags = r.getApplicationTags();
- if (!tags.isEmpty()) {
- for (String tag : tags) {
- builder.append(tag).append(separator);
- }
- }
- DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
- dateFormat.setTimeZone(TimeZone.getDefault());
- builder.append("state: ").append(r.getYarnApplicationState());
- String trackingUrl = r.getTrackingUrl();
- if (isSet(trackingUrl)) {
- builder.append(separator).append("URL: ").append(trackingUrl);
- }
- builder.append(separator)
- .append("Started: ")
- .append(dateFormat.format(new Date(r.getStartTime())));
- long finishTime = r.getFinishTime();
- if (finishTime > 0) {
- builder.append(separator)
- .append("Finished: ")
- .append(dateFormat.format(new Date(finishTime)));
- }
- String rpcHost = r.getHost();
- if (!isSet(rpcHost)) {
- builder.append(separator)
- .append("RPC :")
- .append(rpcHost)
- .append(':')
- .append(r.getRpcPort());
- }
- String diagnostics = r.getDiagnostics();
- if (!isSet(diagnostics)) {
- builder.append(separator).append("Diagnostics :").append(diagnostics);
- }
- return builder.toString();
- }
-
- /**
- * Convert the instance details of an application to a string
- * @param name instance name
- * @param report the application report
- * @param verbose verbose output
- * @return a string
- */
- public static String instanceDetailsToString(String name,
- ApplicationReport report,
- List<ContainerInformation> containers,
- String version,
- Set<String> components,
- boolean verbose) {
- // format strings
- String staticf = "%-30s";
- String reportedf = staticf + " %10s %-42s";
- String livef = reportedf + " %s";
- StringBuilder builder = new StringBuilder(200);
- if (report == null) {
- builder.append(String.format(staticf, name));
- } else {
- // there's a report to look at
- String appId = report.getApplicationId().toString();
- String state = report.getYarnApplicationState().toString();
- if (report.getYarnApplicationState() == YarnApplicationState.RUNNING) {
- // running: there's a URL
- builder.append(
- String.format(livef, name, state, appId, report.getTrackingUrl()));
- } else {
- builder.append(String.format(reportedf, name, state, appId));
- }
- if (verbose) {
- builder.append('\n');
- builder.append(SliderUtils.appReportToString(report, "\n "));
- }
- if (containers != null) {
- builder.append('\n');
- builder.append(SliderUtils.containersToString(containers, version,
- components));
- }
- }
-
- builder.append('\n');
- return builder.toString();
- }
-
- public static String containersToString(
- List<ContainerInformation> containers, String version,
- Set<String> components) {
- String containerf = " %-28s %30s %45s %s%n";
- StringBuilder builder = new StringBuilder(512);
- builder.append("Containers:%n");
- builder.append(String.format(" %-28s %30s %45s %s%n", "Component Name",
- "App Version", "Container Id", "Container Info/Logs"));
- for (ContainerInformation container : containers) {
- if (filter(container.appVersion, version)
- || filter(container.component, components)) {
- continue;
- }
- builder.append(String.format(containerf, container.component,
- container.appVersion, container.containerId, container.host
- + SliderKeys.YARN_CONTAINER_PATH + container.containerId));
- }
- return builder.toString();
- }
-
- /**
- * Filter a string value given a single filter
- *
- * @param value
- * the string value to check
- * @param filter
- * a single string filter
- * @return return true if value should be trapped, false if it should be let
- * through
- */
- public static boolean filter(String value, String filter) {
- return !(StringUtils.isEmpty(filter) || filter.equals(value));
- }
-
- /**
- * Filter a string value given a set of filters
- *
- * @param value
- * the string value to check
- * @param filters
- * a set of string filters
- * @return return true if value should be trapped, false if it should be let
- * through
- */
- public static boolean filter(String value, Set<String> filters) {
- return !(filters.isEmpty() || filters.contains(value));
- }
-
- /**
- * Sorts the given list of application reports, most recently started
- * or finished instance first.
- *
- * @param instances list of instances
- */
- public static void sortApplicationsByMostRecent(List<ApplicationReport> instances) {
- Collections.sort(instances, new MostRecentlyStartedOrFinishedFirst());
- }
-
- /**
- * Sorts the given list of application reports
- * Finished instances are ordered by finished time and running/accepted instances are
- * ordered by start time
- * Finally Instance are order by finished instances coming after running instances
- *
- * @param instances list of instances
- */
- public static void sortApplicationReport(List<ApplicationReport> instances) {
- if (instances.size() <= 1) {
- return;
- }
- List<ApplicationReport> nonLiveInstance =
- new ArrayList<>(instances.size());
- List<ApplicationReport> liveInstance =
- new ArrayList<>(instances.size());
-
- for (ApplicationReport report : instances) {
- if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
- ||
- report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
- liveInstance.add(report);
- } else {
- nonLiveInstance.add(report);
- }
- }
-
- if (liveInstance.size() > 1) {
- Collections.sort(liveInstance, new MostRecentlyStartedAppFirst());
- }
- if (nonLiveInstance.size() > 1) {
- Collections.sort(nonLiveInstance, new MostRecentAppFinishFirst());
- }
- instances.clear();
- instances.addAll(liveInstance);
- instances.addAll(nonLiveInstance);
- }
-
- /**
- * Built a (sorted) map of application reports, mapped to the instance name
- * The list is sorted, and the addition process does not add a report
- * if there is already one that exists. If the list handed in is sorted,
- * those that are listed first form the entries returned
- * @param instances list of intances
- * @param minState minimum YARN state to be included
- * @param maxState maximum YARN state to be included
- * @return all reports in the list whose state >= minimum and <= maximum
- */
- public static Map<String, ApplicationReport> buildApplicationReportMap(
- List<ApplicationReport> instances,
- YarnApplicationState minState, YarnApplicationState maxState) {
- TreeMap<String, ApplicationReport> map = new TreeMap<>();
- for (ApplicationReport report : instances) {
- YarnApplicationState state = report.getYarnApplicationState();
- if (state.ordinal() >= minState.ordinal() &&
- state.ordinal() <= maxState.ordinal() &&
- map.get(report.getName()) == null) {
- map.put(report.getName(), report);
- }
- }
- return map;
- }
-
- /**
- * Take a map and produce a sorted equivalent
- * @param source source map
- * @return a map whose iterator returns the string-sorted ordering of entries
- */
- public static Map<String, String> sortedMap(Map<String, String> source) {
- Map<String, String> out = new TreeMap<>(source);
- return out;
- }
-
- /**
- * Convert a properties instance to a string map.
- * @param properties source property object
- * @return a string map
- */
- public static Map<String, String> toMap(Properties properties) {
- Map<String, String> out = new HashMap<>(properties.size());
- for (Map.Entry<Object, Object> entry : properties.entrySet()) {
- out.put(entry.getKey().toString(), entry.getValue().toString());
- }
- return out;
- }
-
- /**
- * Merge in one map to another -all entries in the second map are
- * merged into the first -overwriting any duplicate keys.
- * @param first first map -the updated one.
- * @param second the map that is merged in
- * @return the first map
- */
- public static Map<String, String> mergeMap(Map<String, String> first,
- Map<String, String> second) {
- first.putAll(second);
- return first;
- }
-
- /**
- * Merge a set of entries into a map. This will take the entryset of
- * a map, or a Hadoop collection itself
- * @param dest destination
- * @param entries entries
- * @return dest -with the entries merged in
- */
- public static Map<String, String> mergeEntries(Map<String, String> dest,
- Iterable<Map.Entry<String, String>> entries) {
- for (Map.Entry<String, String> entry : entries) {
- dest.put(entry.getKey(), entry.getValue());
- }
- return dest;
- }
-
- /**
- * Generic map merge logic
- * @param first first map
- * @param second second map
- * @param <T1> key type
- * @param <T2> value type
- * @return 'first' merged with the second
- */
- public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
- Map<T1, T2> second) {
- first.putAll(second);
- return first;
- }
-
- /**
- * Generic map merge logic
- * @param first first map
- * @param second second map
- * @param <T1> key type
- * @param <T2> value type
- * @return 'first' merged with the second
- */
- public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
- Map<T1, T2> second) {
- Preconditions.checkArgument(first != null, "Null 'first' value");
- Preconditions.checkArgument(second != null, "Null 'second' value");
- for (Map.Entry<T1, T2> entry : second.entrySet()) {
- T1 key = entry.getKey();
- if (!first.containsKey(key)) {
- first.put(key, entry.getValue());
- }
- }
- return first;
- }
-
- /**
- * Merge string maps excluding prefixes
- * @param first first map
- * @param second second map
- * @param prefixes prefixes to ignore
- * @return 'first' merged with the second
- */
- public static Map<String, String> mergeMapsIgnoreDuplicateKeysAndPrefixes(
- Map<String, String> first, Map<String, String> second,
- List<String> prefixes) {
- Preconditions.checkArgument(first != null, "Null 'first' value");
- Preconditions.checkArgument(second != null, "Null 'second' value");
- Preconditions.checkArgument(prefixes != null, "Null 'prefixes' value");
- for (Map.Entry<String, String> entry : second.entrySet()) {
- String key = entry.getKey();
- boolean hasPrefix = false;
- for (String prefix : prefixes) {
- if (key.startsWith(prefix)) {
- hasPrefix = true;
- break;
- }
- }
- if (hasPrefix) {
- continue;
- }
- if (!first.containsKey(key)) {
- first.put(key, entry.getValue());
- }
- }
- return first;
- }
-
- /**
- * Convert a map to a multi-line string for printing
- * @param map map to stringify
- * @return a string representation of the map
- */
- public static String stringifyMap(Map<String, String> map) {
- StringBuilder builder = new StringBuilder();
- for (Map.Entry<String, String> entry : map.entrySet()) {
- builder.append(entry.getKey())
- .append("=\"")
- .append(entry.getValue())
- .append("\"\n");
-
- }
- return builder.toString();
- }
-
- /**
- * Get the int value of a role
- * @param roleMap map of role key->val entries
- * @param key key the key to look for
- * @param defVal default value to use if the key is not in the map
- * @param min min value or -1 for do not check
- * @param max max value or -1 for do not check
- * @return the int value the integer value
- * @throws BadConfigException if the value could not be parsed
- */
- public static int getIntValue(Map<String, String> roleMap,
- String key,
- int defVal,
- int min,
- int max
- ) throws BadConfigException {
- String valS = roleMap.get(key);
- return parseAndValidate(key, valS, defVal, min, max);
-
- }
-
- /**
- * Parse an int value, replacing it with defval if undefined;
- * @param errorKey key to use in exceptions
- * @param defVal default value to use if the key is not in the map
- * @param min min value or -1 for do not check
- * @param max max value or -1 for do not check
- * @return the int value the integer value
- * @throws BadConfigException if the value could not be parsed
- */
- public static int parseAndValidate(String errorKey,
- String valS,
- int defVal,
- int min, int max) throws
- BadConfigException {
- if (valS == null) {
- valS = Integer.toString(defVal);
- }
- String trim = valS.trim();
- int val;
- try {
- val = Integer.decode(trim);
- } catch (NumberFormatException e) {
- throw new BadConfigException("Failed to parse value of "
- + errorKey + ": \"" + trim + "\"");
- }
- if (min >= 0 && val < min) {
- throw new BadConfigException("Value of "
- + errorKey + ": " + val + ""
- + "is less than the minimum of " + min);
- }
- if (max >= 0 && val > max) {
- throw new BadConfigException("Value of "
- + errorKey + ": " + val + ""
- + "is more than the maximum of " + max);
- }
- return val;
- }
-
- public static InetSocketAddress getRmAddress(Configuration conf) {
- return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_PORT);
- }
-
- public static InetSocketAddress getRmSchedulerAddress(Configuration conf) {
- return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
- }
-
- /**
- * probe to see if the RM scheduler is defined
- * @param conf config
- * @return true if the RM scheduler address is set to
- * something other than 0.0.0.0
- */
- public static boolean isRmSchedulerAddressDefined(Configuration conf) {
- InetSocketAddress address = getRmSchedulerAddress(conf);
- return isAddressDefined(address);
- }
-
- /**
- * probe to see if the address
- * @param address network address
- * @return true if the scheduler address is set to
- * something other than 0.0.0.0
- */
- public static boolean isAddressDefined(InetSocketAddress address) {
- if (address == null || address.getHostString() == null) {
- return false;
- }
- return !(address.getHostString().equals("0.0.0.0"));
- }
-
- public static void setRmAddress(Configuration conf, String rmAddr) {
- conf.set(YarnConfiguration.RM_ADDRESS, rmAddr);
- }
-
- public static void setRmSchedulerAddress(Configuration conf, String rmAddr) {
- conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);
- }
-
- public static boolean hasAppFinished(ApplicationReport report) {
- return report == null ||
- report.getYarnApplicationState().ordinal() >=
- YarnApplicationState.FINISHED.ordinal();
- }
-
- public static String containerToString(Container container) {
- if (container == null) {
- return "null container";
- }
- return String.format(Locale.ENGLISH,
- "ContainerID=%s nodeID=%s http=%s priority=%s resource=%s",
- container.getId(),
- container.getNodeId(),
- container.getNodeHttpAddress(),
- container.getPriority(),
- container.getResource());
- }
-
- /**
- * convert an AM report to a string for diagnostics
- * @param report the report
- * @return the string value
- */
- public static String reportToString(ApplicationReport report) {
- if (report == null) {
- return "Null application report";
- }
-
- return "App " + report.getName() + "/" + report.getApplicationType() +
- "# " +
- report.getApplicationId() + " user " + report.getUser() +
- " is in state " + report.getYarnApplicationState() +
- " RPC: " + report.getHost() + ":" + report.getRpcPort() +
- " URL: " + report.getOriginalTrackingUrl();
- }
-
- /**
- * Convert a YARN URL into a string value of a normal URL
- * @param url URL
- * @return string representatin
- */
- public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
- StringBuilder builder = new StringBuilder();
- builder.append(url.getScheme()).append("://");
- if (url.getHost() != null) {
- builder.append(url.getHost()).append(":").append(url.getPort());
- }
- builder.append(url.getFile());
- return builder.toString();
- }
-
- public static int findFreePort(int start, int limit) {
- if (start == 0) {
- //bail out if the default is "dont care"
- return 0;
- }
- int found = 0;
- int port = start;
- int finish = start + limit;
- while (found == 0 && port < finish) {
- if (isPortAvailable(port)) {
- found = port;
- } else {
- port++;
- }
- }
- return found;
- }
-
- /**
- * Get a random open port
- * @return true if the port was available for listening on
- */
- public static int getOpenPort() throws IOException {
- ServerSocket socket = null;
- try {
- socket = new ServerSocket(0);
- return socket.getLocalPort();
- } finally {
- if (socket != null) {
- socket.close();
- }
- }
- }
-
- /**
- * See if a port is available for listening on by trying to listen
- * on it and seeing if that works or fails.
- * @param port port to listen to
- * @return true if the port was available for listening on
- */
- public static boolean isPortAvailable(int port) {
- try {
- ServerSocket socket = new ServerSocket(port);
- socket.close();
- return true;
- } catch (IOException e) {
- return false;
- }
- }
-
- /**
- * Build the environment map from a role option map, finding all entries
- * beginning with "env.", adding them to a map of (prefix-removed)
- * env vars
- * @param roleOpts role options. This can be null, meaning the
- * role is undefined
- * @return a possibly empty map of environment variables.
- */
- public static Map<String, String> buildEnvMap(Map<String, String> roleOpts) {
- return buildEnvMap(roleOpts, null);
- }
-
-
- // Build env map: key -> value;
- // value will be replaced by the corresponding value in tokenMap, if any.
- public static Map<String, String> buildEnvMap(
- org.apache.slider.api.resource.Configuration conf,
- Map<String,String> tokenMap) {
- if (tokenMap == null) {
- return conf.getEnv();
- }
- Map<String, String> env = new HashMap<>();
- for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) {
- String key = entry.getKey();
- String val = entry.getValue();
- for (Map.Entry<String,String> token : tokenMap.entrySet()) {
- val = val.replaceAll(Pattern.quote(token.getKey()),
- token.getValue());
- }
- env.put(key,val);
- }
- return env;
- }
-
-
- public static Map<String, String> buildEnvMap(Map<String, String> roleOpts,
- Map<String,String> tokenMap) {
- Map<String, String> env = new HashMap<>();
- if (roleOpts != null) {
- for (Map.Entry<String, String> entry : roleOpts.entrySet()) {
- String key = entry.getKey();
- if (key.startsWith(RoleKeys.ENV_PREFIX)) {
- String envName = key.substring(RoleKeys.ENV_PREFIX.length());
- if (!envName.isEmpty()) {
- String value = entry.getValue();
- if (tokenMap != null) {
- for (Map.Entry<String,String> token : tokenMap.entrySet()) {
- value = value.replaceAll(Pattern.quote(token.getKey()),
- token.getValue());
- }
- }
- env.put(envName, value);
- }
- }
- }
- }
- return env;
- }
-
- /**
- * Apply a set of command line options to a cluster role map
- * @param clusterRoleMap cluster role map to merge onto
- * @param commandOptions command opts
- */
- public static void applyCommandLineRoleOptsToRoleMap(
- Map<String, Map<String, String>> clusterRoleMap,
- Map<String, Map<String, String>> commandOptions) {
- for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
- String key = entry.getKey();
- Map<String, String> optionMap = entry.getValue();
- Map<String, String> existingMap = clusterRoleMap.get(key);
- if (existingMap == null) {
- existingMap = new HashMap<String, String>();
- }
- log.debug("Overwriting role options with command line values {}",
- stringifyMap(optionMap));
- mergeMap(existingMap, optionMap);
- //set or overwrite the role
- clusterRoleMap.put(key, existingMap);
- }
- }
-
- /**
- * verify that the supplied cluster name is valid
- * @param clustername cluster name
- * @throws BadCommandArgumentsException if it is invalid
- */
- public static void validateClusterName(String clustername)
- throws BadCommandArgumentsException {
- if (!isClusternameValid(clustername)) {
- throw new BadCommandArgumentsException(
- "Illegal cluster name: " + clustername);
- }
- }
-
- /**
- * Verify that a Kerberos principal has been set -if not fail
- * with an error message that actually tells you what is missing
- * @param conf configuration to look at
- * @param principal key of principal
- * @throws BadConfigException if the key is not set
- */
- public static void verifyPrincipalSet(Configuration conf,
- String principal) throws
- BadConfigException {
- String principalName = conf.get(principal);
- if (principalName == null) {
- throw new BadConfigException("Unset Kerberos principal : %s",
- principal);
- }
- log.debug("Kerberos princial {}={}", principal, principalName);
- }
-
- /**
- * Flag to indicate whether the cluster is in secure mode
- * @param conf configuration to look at
- * @return true if the slider client/service should be in secure mode
- */
- public static boolean isHadoopClusterSecure(Configuration conf) {
- return SecurityUtil.getAuthenticationMethod(conf) !=
- UserGroupInformation.AuthenticationMethod.SIMPLE;
- }
-
- /**
- * Init security if the cluster configuration declares the cluster is secure
- * @param conf configuration to look at
- * @return true if the cluster is secure
- * @throws IOException cluster is secure
- * @throws SliderException the configuration/process is invalid
- */
- public static boolean maybeInitSecurity(Configuration conf) throws
- IOException,
- SliderException {
- boolean clusterSecure = isHadoopClusterSecure(conf);
- if (clusterSecure) {
- log.debug("Enabling security");
- initProcessSecurity(conf);
- }
- return clusterSecure;
- }
-
- /**
- * Turn on security. This is setup to only run once.
- * @param conf configuration to build up security
- * @return true if security was initialized in this call
- * @throws IOException IO/Net problems
- * @throws BadConfigException the configuration and system state are inconsistent
- */
- public static boolean initProcessSecurity(Configuration conf) throws
- IOException,
- SliderException {
-
- if (processSecurityAlreadyInitialized.compareAndSet(true, true)) {
- //security is already inited
- return false;
- }
-
- log.info("JVM initialized into secure mode with kerberos realm {}",
- SliderUtils.getKerberosRealm());
- //this gets UGI to reset its previous world view (i.e simple auth)
- //security
- log.debug("java.security.krb5.realm={}",
- System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
- log.debug("java.security.krb5.kdc={}",
- System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
- log.debug("hadoop.security.authentication={}",
- conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION));
- log.debug("hadoop.security.authorization={}",
- conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION));
- UserGroupInformation.setConfiguration(conf);
- UserGroupInformation authUser = UserGroupInformation.getCurrentUser();
- log.debug("Authenticating as {}", authUser);
- log.debug("Login user is {}", UserGroupInformation.getLoginUser());
- if (!UserGroupInformation.isSecurityEnabled()) {
- throw new SliderException(LauncherExitCodes.EXIT_UNAUTHORIZED,
- "Although secure mode is enabled," +
- "the application has already set up its user as an insecure entity %s",
- authUser);
- }
- if (authUser.getAuthenticationMethod() ==
- UserGroupInformation.AuthenticationMethod.SIMPLE) {
- throw new BadConfigException("Auth User is not Kerberized %s" +
- " -security has already been set up with the wrong authentication method. "
- + "This can occur if a file system has already been created prior to the loading of "
- + "the security configuration.",
- authUser);
-
- }
-
- SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
- SliderUtils.verifyPrincipalSet(conf, SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
- return true;
- }
-
- /**
- * Force an early login: This catches any auth problems early rather than
- * in RPC operations
- * @throws IOException if the login fails
- */
- public static void forceLogin() throws IOException {
- if (UserGroupInformation.isSecurityEnabled()) {
- if (UserGroupInformation.isLoginKeytabBased()) {
- UserGroupInformation.getLoginUser().reloginFromKeytab();
- } else {
- UserGroupInformation.getLoginUser().reloginFromTicketCache();
- }
- }
- }
-
- public static String getLibDir() {
- String[] libDirs = getLibDirs();
- if (libDirs == null || libDirs.length == 0) {
- return null;
- }
- return libDirs[0];
- }
-
- public static String[] getLibDirs() {
- String libDirStr = System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
- if (isUnset(libDirStr)) {
- return ArrayUtils.EMPTY_STRING_ARRAY;
- }
- return StringUtils.split(libDirStr, ',');
- }
-
- /**
- * Submit a JAR containing a specific class and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- * @param clazz class to look for
- * @param libdir lib directory
- * @param jarName <i>At the destination</i>
- * @return the local resource ref
- * @throws IOException trouble copying to HDFS
- */
- public static LocalResource putJar(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Class clazz,
- Path tempPath,
- String libdir,
- String jarName
- )
- throws IOException, SliderException {
- LocalResource res = sliderFileSystem.submitJarWithClass(
- clazz,
- tempPath,
- libdir,
- jarName);
- providerResources.put(libdir + "/" + jarName, res);
- return res;
- }
-
- /**
- * Submit a JAR containing and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- * @param libDir lib directory
- * @param srcPath copy jars from
- * @throws IOException, SliderException trouble copying to HDFS
- */
- public static void putAllJars(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Path tempPath,
- String libDir,
- String srcPath) throws IOException, SliderException {
- log.info("Loading all dependencies from {}", srcPath);
- if (SliderUtils.isSet(srcPath)) {
- File srcFolder = new File(srcPath);
- FilenameFilter jarFilter = createJarFilter();
- File[] listOfJars = srcFolder.listFiles(jarFilter);
- if (listOfJars == null || listOfJars.length == 0) {
- return;
- }
- for (File jarFile : listOfJars) {
- LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
- providerResources.put(libDir + "/" + jarFile.getName(), res);
- }
- }
- }
-
- /**
- * Accept all filenames ending with {@code .jar}
- * @return a filename filter
- */
- public static FilenameFilter createJarFilter() {
- return new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
- }
- };
- }
-
- /**
- * Submit the AM tar.gz containing all dependencies and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- * @throws IOException, SliderException trouble copying to HDFS
- */
- public static void putAmTarGzipAndUpdate(
- Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem
- ) throws IOException, SliderException {
- log.info("Loading all dependencies from {}{}",
- SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME,
- SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT);
- sliderFileSystem.submitTarGzipAndUpdate(providerResources);
- }
-
- public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
- Map<String, Map<String, String>> dest = new HashMap<>();
- for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) {
- dest.put(entry.getKey(), stringMapClone(entry.getValue()));
- }
- return dest;
- }
-
- public static Map<String, String> stringMapClone(Map<String, String> src) {
- Map<String, String> dest = new HashMap<>();
- return mergeEntries(dest, src.entrySet());
- }
-
- /**
- * List a directory in the local filesystem
- * @param dir directory
- * @return a listing, one to a line
- */
- public static String listDir(File dir) {
- if (dir == null) {
- return "";
- }
- String[] confDirEntries = dir.list();
- if (confDirEntries == null) {
- return "";
- }
- StringBuilder builder = new StringBuilder();
- for (String entry : confDirEntries) {
- builder.append(entry).append("\n");
- }
- return builder.toString();
- }
-
- /**
- * Create a file:// path from a local file
- * @param file file to point the path
- * @return a new Path
- */
- public static Path createLocalPath(File file) {
- return new Path(file.toURI());
- }
-
- /**
- * Get the current user -relays to
- * {@link UserGroupInformation#getCurrentUser()}
- * with any Slider-specific post processing and exception handling
- * @return user info
- * @throws IOException on a failure to get the credentials
- */
- public static UserGroupInformation getCurrentUser() throws IOException {
-
- try {
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
- return currentUser;
- } catch (IOException e) {
- log.info("Failed to get user info", e);
- throw e;
- }
- }
-
- public static String getKerberosRealm() {
- try {
- return KerberosUtil.getDefaultRealm();
- } catch (Exception e) {
- log.debug("introspection into JVM internals failed", e);
- return "(unknown)";
-
- }
- }
-
- /**
- * Register the client resource in
- * {@link SliderKeys#SLIDER_CLIENT_XML}
- * for Configuration instances.
- *
- * @return true if the resource could be loaded
- */
- public static URL registerClientResource() {
- return ConfigHelper.registerDefaultResource(SliderKeys.SLIDER_CLIENT_XML);
- }
-
- /**
- * Attempt to load the slider client resource. If the
- * resource is not on the CP an empty config is returned.
- * @return a config
- */
- public static Configuration loadSliderClientXML() {
- return ConfigHelper.loadFromResource(SliderKeys.SLIDER_CLIENT_XML);
- }
-
- /**
- * Convert a char sequence to a string.
- * This ensures that comparisons work
- * @param charSequence source
- * @return the string equivalent
- */
- public static String sequenceToString(CharSequence charSequence) {
- StringBuilder stringBuilder = new StringBuilder(charSequence);
- return stringBuilder.toString();
- }
-
- /**
- * Build up the classpath for execution
- * -behaves very differently on a mini test cluster vs a production
- * production one.
- *
- * @param sliderConfDir relative path to the dir containing slider config
- * options to put on the classpath -or null
- * @param libdir directory containing the JAR files
- * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
- * (and hence the current classpath should be used, not anything built up)
- * @return a classpath
- */
- public static ClasspathConstructor buildClasspath(String sliderConfDir,
- String libdir,
- SliderFileSystem sliderFileSystem,
- boolean usingMiniMRCluster) {
-
- ClasspathConstructor classpath = new ClasspathConstructor();
- classpath.append(SliderKeys.LOG4J_SERVER_PROP_FILENAME);
-
- // add the runtime classpath needed for tests to work
- if (usingMiniMRCluster) {
- // for mini cluster we pass down the java CP properties
- // and nothing else
- classpath.appendAll(classpath.localJVMClasspath());
- } else {
- if (sliderConfDir != null) {
- classpath.addClassDirectory(sliderConfDir);
- }
- classpath.addLibDir(libdir);
- if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
- classpath.addLibDir(SliderKeys.SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK);
- } else {
- log.info(
- "For faster submission of apps, upload dependencies using cmd {} {}",
- SliderActions.ACTION_DEPENDENCY, Arguments.ARG_UPLOAD);
- }
- classpath.addRemoteClasspathEnvVar();
- classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
- }
- return classpath;
- }
-
- /**
- * Verify that a path refers to a directory. If not
- * logs the parent dir then throws an exception
- * @param dir the directory
- * @param errorlog log for output on an error
- * @throws FileNotFoundException if it is not a directory
- */
- public static void verifyIsDir(File dir, Logger errorlog) throws
- FileNotFoundException {
- if (!dir.exists()) {
- errorlog.warn("contents of {}: {}", dir,
- listDir(dir.getParentFile()));
- throw new FileNotFoundException(dir.toString());
- }
- if (!dir.isDirectory()) {
- errorlog.info("contents of {}: {}", dir,
- listDir(dir.getParentFile()));
- throw new FileNotFoundException(
- "Not a directory: " + dir);
- }
- }
-
- /**
- * Verify that a file exists
- * @param file file
- * @param errorlog log for output on an error
- * @throws FileNotFoundException
- */
- public static void verifyFileExists(File file, Logger errorlog) throws
- FileNotFoundException {
- if (!file.exists()) {
- errorlog.warn("contents of {}: {}", file,
- listDir(file.getParentFile()));
- throw new FileNotFoundException(file.toString());
- }
- if (!file.isFile()) {
- throw new FileNotFoundException("Not a file: " + file.toString());
- }
- }
-
- /**
- * verify that a config option is set
- * @param configuration config
- * @param key key
- * @return the value, in case it needs to be verified too
- * @throws BadConfigException if the key is missing
- */
- public static String verifyOptionSet(Configuration configuration, String key,
- boolean allowEmpty) throws BadConfigException {
- String val = configuration.get(key);
- if (val == null) {
- throw new BadConfigException(
- "Required configuration option \"%s\" not defined ", key);
- }
- if (!allowEmpty && val.isEmpty()) {
- throw new BadConfigException(
- "Configuration option \"%s\" must not be empty", key);
- }
- return val;
- }
-
- /**
- * Verify that a keytab property is defined and refers to a non-empty file
- *
- * @param siteConf configuration
- * @param prop property to look for
- * @return the file referenced
- * @throws BadConfigException on a failure
- */
- public static File verifyKeytabExists(Configuration siteConf,
- String prop) throws
- BadConfigException {
- String keytab = siteConf.get(prop);
- if (keytab == null) {
- throw new BadConfigException("Missing keytab property %s",
- prop);
-
- }
- File keytabFile = new File(keytab);
- if (!keytabFile.exists()) {
- throw new BadConfigException("Missing keytab file %s defined in %s",
- keytabFile,
- prop);
- }
- if (keytabFile.length() == 0 || !keytabFile.isFile()) {
- throw new BadConfigException("Invalid keytab file %s defined in %s",
- keytabFile,
- prop);
- }
- return keytabFile;
- }
-
- /**
- * Convert an epoch time to a GMT time. This
- * uses the deprecated Date.toString() operation,
- * so is in one place to reduce the number of deprecation warnings.
- * @param time timestamp
- * @return string value as ISO-9601
- */
- @SuppressWarnings({"CallToDateToString", "deprecation"})
- public static String toGMTString(long time) {
- return new Date(time).toGMTString();
- }
-
- /**
- * Add the cluster build information; this will include Hadoop details too
- * @param info cluster info
- * @param prefix prefix for the build info
- */
- public static void addBuildInfo(Map<String, String> info, String prefix) {
-
- Properties props = SliderVersionInfo.loadVersionProperties();
- info.put(prefix + "." + SliderVersionInfo.APP_BUILD_INFO, props.getProperty(
- SliderVersionInfo.APP_BUILD_INFO));
- info.put(prefix + "." + SliderVersionInfo.HADOOP_BUILD_INFO,
- props.getProperty(SliderVersionInfo.HADOOP_BUILD_INFO));
-
- info.put(prefix + "." + SliderVersionInfo.HADOOP_DEPLOYED_INFO,
- VersionInfo.getBranch() + " @" + VersionInfo.getSrcChecksum());
- }
-
- public static String propertiesToString(Properties props) {
- TreeSet<String> keys = new TreeSet<>(props.stringPropertyNames());
- StringBuilder builder = new StringBuilder();
- for (String key : keys) {
- builder.append(key)
- .append("=")
- .append(props.getProperty(key))
- .append("\n");
- }
- return builder.toString();
- }
-
- /**
- * Add a subpath to an existing URL. This extends
- * the path, inserting a / between all entries
- * if needed.
- * @param base base path/URL
- * @param path subpath
- * @return base+"/"+subpath
- */
- public static String appendToURL(String base, String path) {
- StringBuilder fullpath = new StringBuilder(base);
- if (!base.endsWith("/")) {
- fullpath.append("/");
- }
- if (path.startsWith("/")) {
- fullpath.append(path.substring(1));
- } else {
- fullpath.append(path);
- }
- return fullpath.toString();
- }
-
- /**
- * Append a list of paths, inserting "/" signs as appropriate
- * @param base base path/URL
- * @param paths subpaths
- * @return base+"/"+paths[0]+"/"+paths[1]...
- */
- public static String appendToURL(String base, String... paths) {
- String result = base;
- for (String path : paths) {
- result = appendToURL(result, path);
- }
- return result;
- }
-
-
- /**
- * Truncate the given string to a maximum length provided
- * with a pad (...) added to the end if expected size if more than 10.
- * @param toTruncate string to truncate; may be null
- * @param maxSize maximum size
- * @return the truncated/padded string.
- */
- public static String truncate(String toTruncate, int maxSize) {
- if (toTruncate == null || maxSize < 1
- || toTruncate.length() <= maxSize) {
- return toTruncate;
- }
-
- String pad = "...";
- if (maxSize < 10) {
- pad = "";
- }
- return toTruncate.substring(0, maxSize - pad.length()).concat(pad);
- }
-
- /**
- * Get a string node label value from a node report
- * @param report node report
- * @return a single trimmed label or ""
- */
- public static String extractNodeLabel(NodeReport report) {
- Set<String> newlabels = report.getNodeLabels();
- if (newlabels != null && !newlabels.isEmpty()) {
- return newlabels.iterator().next().trim();
- } else {
- return "";
- }
- }
-
- /**
- * Callable for async/scheduled halt
- */
- public static class DelayedHalt extends TimerTask {
- private final int status;
- private final String text;
-
- public DelayedHalt(int status, String text) {
- this.status = status;
- this.text = text;
- }
-
- @Override
- public void run() {
- try {
- ExitUtil.halt(status, text);
- //this should never be reached
- } catch (ExitUtil.HaltException e) {
- log.info("Halt failed");
- }
- }
- }
-
- /**
- * A compareTo function that converts the result of a long
- * comparision into the integer that <code>Comparable</code>
- * expects.
- * @param left left side
- * @param right right side
- * @return -1, 0, 1 depending on the diff
- */
- public static int compareTo(long left, long right) {
- long diff = left - right;
- if (diff < 0) {
- return -1;
- }
- if (diff > 0) {
- return 1;
- }
- return 0;
- }
-
- /**
- * Given a source folder create zipped file
- *
- * @param srcFolder
- * @param zipFile
- *
- * @throws IOException
- */
- public static void zipFolder(File srcFolder, File zipFile) throws IOException {
- log.info("Zipping folder {} to {}", srcFolder.getAbsolutePath(), zipFile.getAbsolutePath());
- List<String> files = new ArrayList<>();
- generateFileList(files, srcFolder, srcFolder, true);
-
- byte[] buffer = new byte[1024];
-
- try (FileOutputStream fos = new FileOutputStream(zipFile)) {
- try (ZipOutputStream zos = new ZipOutputStream(fos)) {
-
- for (String file : files) {
- ZipEntry ze = new ZipEntry(file);
- zos.putNextEntry(ze);
- try (FileInputStream in = new FileInputStream(srcFolder + File.separator + file)) {
- int len;
- while ((len = in.read(buffer)) > 0) {
- zos.write(buffer, 0, len);
- }
- }
- }
- }
- }
- }
-
- /**
- * Given a source folder create a tar.gz file
- *
- * @param libDirs
- * @param tarGzipFile
- *
- * @throws IOException
- */
- public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
- FilenameFilter filter) throws IOException {
- log.info("Tar-gzipping folders {} to {}", libDirs,
- tarGzipFile.getAbsolutePath());
-
- try(TarArchiveOutputStream taos =
- new TarArchiveOutputStream(new GZIPOutputStream(
- new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
- for (String libDir : libDirs) {
- File srcFolder = new File(libDir);
- List<String> files = new ArrayList<>();
- generateFileList(files, srcFolder, srcFolder, true, filter);
- for (String file : files) {
- File srcFile = new File(srcFolder, file);
- TarArchiveEntry tarEntry = new TarArchiveEntry(
- srcFile, file);
- taos.putArchiveEntry(tarEntry);
- try(FileInputStream in = new FileInputStream(srcFile)) {
- org.apache.commons.io.IOUtils.copy(in, taos);
- }
- taos.flush();
- taos.closeArchiveEntry();
- }
- }
- }
- }
-
- /**
- * Retrieve the HDP version if it is an HDP cluster, or null otherwise. It
- * first checks if system property HDP_VERSION is defined. If not it checks if
- * system env HDP_VERSION is defined.
- *
- * @return HDP version (if defined) or null otherwise
- */
- public static String getHdpVersion() {
- String hdpVersion = System
- .getProperty(SliderKeys.HDP_VERSION_PROP_NAME);
- if (StringUtils.isEmpty(hdpVersion)) {
- hdpVersion = System.getenv(SliderKeys.HDP_VERSION_PROP_NAME);
- }
- return hdpVersion;
- }
-
- /**
- * Query to find if it is an HDP cluster
- *
- * @return true if this is invoked in an HDP cluster or false otherwise
- */
- public static boolean isHdp() {
- return StringUtils.isNotEmpty(getHdpVersion());
- }
-
- /**
- * Retrieve the version of the current Slider install
- *
- * @return the version string of the Slider release
- */
- public static String getSliderVersion() {
- if (isHdp()) {
- return getHdpVersion();
- } else {
- Properties props = SliderVersionInfo.loadVersionProperties();
- return props.getProperty(SliderVersionInfo.APP_VERSION);
- }
- }
-
- private static void generateFileList(List<String> fileList, File node,
- File rootFolder, Boolean relative) {
- generateFileList(fileList, node, rootFolder, relative, null);
- }
-
- private static void generateFileList(List<String> fileList, File node,
- File rootFolder, Boolean relative, FilenameFilter filter) {
- if (node.isFile()) {
- String fileFullPath = node.toString();
- if (relative) {
- fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1,
- fileFullPath.length()));
- } else {
- fileList.add(fileFullPath);
- }
- }
-
- if (node.isDirectory()) {
- String[] subNode = node.list(filter);
- if (subNode == null || subNode.length == 0) {
- return;
- }
- for (String filename : subNode) {
- generateFileList(fileList, new File(node, filename), rootFolder,
- relative, filter);
- }
- }
- }
-
- /**
- * This wraps ApplicationReports and generates a string version
- * iff the toString() operator is invoked
- */
- public static class OnDemandReportStringifier {
- private final ApplicationReport report;
-
- public OnDemandReportStringifier(ApplicationReport report) {
- this.report = report;
- }
-
- @Override
- public String toString() {
- return appReportToString(report, "\n");
- }
- }
-
- /**
- * Check for any needed libraries being present. On Unix none are needed;
- * on windows they must be present
- * @return true if all is well
- */
- public static String checkForRequiredNativeLibraries() {
-
- if (!Shell.WINDOWS) {
- return "";
- }
- StringBuilder errorText = new StringBuilder("");
- if (!NativeIO.isAvailable()) {
- errorText.append("No native IO library. ");
- }
- try {
- String path = Shell.getQualifiedBinPath(WINUTILS);
- log.debug("winutils is at {}", path);
- } catch (IOException e) {
- errorText.append("No " + WINUTILS);
- log.warn("No winutils: {}", e, e);
- }
- try {
- File target = new File("target");
- FileUtil.canRead(target);
- } catch (UnsatisfiedLinkError e) {
- log.warn("Failing to link to native IO methods: {}", e, e);
- errorText.append("No native IO methods");
- }
- return errorText.toString();
- }
-
- /**
- * Strictly verify that windows utils is present.
- * Checks go as far as opening the file and looking for
- * the headers.
- * @throws IOException on any problem reading the file
- * @throws FileNotFoundException if the file is not considered valid
- */
- public static void maybeVerifyWinUtilsValid() throws
- IOException,
- SliderException {
- String errorText = SliderUtils.checkForRequiredNativeLibraries();
- if (!errorText.isEmpty()) {
- throw new BadClusterStateException(errorText);
- }
- }
-
- public static void verifyIsFile(String program, File exe) throws
- FileNotFoundException {
- if (!exe.isFile()) {
- throw new FileNotFoundException(program
- + " at " + exe
- + " is not a file");
-
- }
- }
-
- public static void verifyFileSize(String program,
- File exe,
- int minFileSize) throws FileNotFoundException {
- if (exe.length() < minFileSize) {
- throw new FileNotFoundException(program
- + " at " + exe
- + " is too short to be an executable");
- }
- }
-
- /**
- * Write bytes to a file
- * @param outfile output file
- * @param data data to write
- * @throws IOException on any IO problem
- */
- public static void write(File outfile, byte[] data)
- throws IOException {
- File parentDir = outfile.getCanonicalFile().getParentFile();
- if (parentDir == null) {
- throw new IOException(outfile.getPath() + " has no parent dir");
- }
- if (!parentDir.exists()) {
- if(!parentDir.mkdirs()) {
- throw new IOException("Failed to create parent directory " + parentDir);
- }
- }
- SliderUtils.verifyIsDir(parentDir, log);
- try(FileOutputStream out = new FileOutputStream(outfile)) {
- out.write(data);
- }
- }
-
- /**
- * Execute a command for a test operation
- * @param name name in error
- * @param status status code expected
- * @param timeoutMillis timeout in millis for process to finish
- * @param logger
- * @param outputString optional string to grep for (must not span a line)
- * @param commands commands @return the process
- * @throws IOException on any failure.
- */
- public static ForkedProcessService execCommand(String name,
- int status,
- long timeoutMillis,
- Logger logger,
- String outputString,
- String... commands) throws IOException, SliderException {
- Preconditions.checkArgument(isSet(name), "no name");
- Preconditions.checkArgument(commands.length > 0, "no commands");
- Preconditions.checkArgument(isSet(commands[0]), "empty command");
-
- ForkedProcessService process;
-
-
- process = new ForkedProcessService(
- name,
- new HashMap<String, String>(),
- Arrays.asList(commands));
- process.setProcessLog(logger);
- process.init(new Configuration());
- String errorText = null;
- process.start();
- try {
- if (!process.waitForServiceToStop(timeoutMillis)) {
- throw new TimeoutException(
- "Process did not stop in " + timeoutMillis + "mS");
- }
- int exitCode = process.getExitCode();
- List<String> recentOutput = process.getRecentOutput();
- if (status != exitCode) {
- // error condition
- errorText = "Expected exit code={" + status + "}, "
- + "actual exit code={" + exitCode + "}";
- } else {
- if (isSet(outputString)) {
- boolean found = false;
- for (String line : recentOutput) {
- if (line.contains(outputString)) {
- found = true;
- break;
- }
- }
- if (!found) {
- errorText = "Did not find \"" + outputString + "\""
- + " in output";
- }
- }
- }
- if (errorText == null) {
- return process;
- }
-
- } catch (TimeoutException e) {
- errorText = e.toString();
- }
- // error text: non null ==> operation failed
- log.warn(errorText);
- List<String> recentOutput = process.getRecentOutput();
- for (String line : recentOutput) {
- log.info(line);
- }
- throw new SliderException(LauncherExitCodes.EXIT_OTHER_FAILURE,
- "Process %s failed: %s", name, errorText);
-
- }
-
-
- /**
- * Validate the slider client-side execution environment.
- * This looks for everything felt to be critical for execution, including
- * native binaries and other essential dependencies.
- * @param logger logger to log to on normal execution
- * @throws IOException on IO failures
- * @throws SliderException on validation failures
- */
- public static void validateSliderClientEnvironment(Logger logger) throws
- IOException,
- SliderException {
- maybeVerifyWinUtilsValid();
- }
-
- /**
- * Validate the slider server-side execution environment.
- * This looks for everything felt to be critical for execution, including
- * native binaries and other essential dependencies.
- * @param logger logger to log to on normal execution
- * @param dependencyChecks flag to indicate checks for agent dependencies
- * @throws IOException on IO failures
- * @throws SliderException on validation failures
- */
- public static void validateSliderServerEnvironment(Logger logger,
- boolean dependencyChecks) throws
- IOException,
- SliderException {
- maybeVerifyWinUtilsValid();
- if (dependencyChecks) {
- validatePythonEnv(logger);
- validateOpenSSLEnv(logger);
- }
- }
-
- public static void validateOpenSSLEnv(Logger logger) throws
- IOException,
- SliderException {
- execCommand(OPENSSL, 0, 5000, logger, "OpenSSL", OPENSSL, "version");
- }
-
- public static void validatePythonEnv(Logger logger) throws
- IOException,
- SliderException {
- execCommand(PYTHON, 0, 5000, logger, "Python", PYTHON, "-V");
- }
-
- /**
- * return the path to the currently running slider command
- *
- * @throws NullPointerException
- * - If the pathname argument is null
- * @throws SecurityException
- * - if a security manager exists and its checkPermission method
- * doesn't allow getting the ProtectionDomain
- */
- public static String getCurrentCommandPath() {
- File f = new File(Slider.class.getProtectionDomain().getCodeSource()
- .getLocation().getPath());
- return f.getAbsolutePath();
- }
-
- /**
- * return the path to the slider-client.xml used by the current running
- * slider command
- *
- * @throws SecurityException
- * - if a security manager exists and its checkPermission method
- * denies access to the class loader for the class
- */
- public static String getClientConfigPath() {
- URL path = ConfigHelper.class.getClassLoader().getResource(
- SliderKeys.SLIDER_CLIENT_XML);
- Preconditions.checkNotNull(path, "Failed to locate resource " + SliderKeys.SLIDER_CLIENT_XML);
- return path.toString();
- }
-
- /**
- * validate if slider-client.xml under the path can be opened
- *
- * @throws IOException
- * : the file can't be found or open
- */
- public static void validateClientConfigFile() throws IOException {
- URL resURL = SliderVersionInfo.class.getClassLoader().getResource(
- SliderKeys.SLIDER_CLIENT_XML);
- if (resURL == null) {
- throw new IOException(
- "slider-client.xml doesn't exist on the path: "
- + getClientConfigPath());
- }
-
- try {
- InputStream inStream = resURL.openStream();
- if (inStream == null) {
- throw new IOException("slider-client.xml can't be opened");
- }
- } catch (IOException e) {
- throw new IOException("slider-client.xml can't be opened: "
- + e.toString());
- }
- }
-
- /**
- * validate if a file on HDFS can be open
- *
- * @throws IOException the file can't be found or opened
- * @throws URISyntaxException
- */
- public static void validateHDFSFile(SliderFileSystem sliderFileSystem,
- String pathStr)
- throws IOException, URISyntaxException {
- try(InputStream inputStream =
- sliderFileSystem.getFileSystem().open(new Path(new URI(pathStr)))) {
- if (inputStream == null) {
- throw new IOException("HDFS file " + pathStr + " can't be opened");
- }
- }
- }
-
- /**
- * return the version and path of the JDK invoking the current running
- * slider command
- *
- * @throws SecurityException
- * - if a security manager exists and its checkPropertyAccess
- * method doesn't allow access to the specified system property.
- */
- public static String getJDKInfo() {
- String version = System.getProperty("java.version");
- String javaHome = System.getProperty("java.home");
- return
- "The version of the JDK invoking the current running slider command: "
- + version + "; The path to it is: " + javaHome;
- }
-
- /**
- * Compare the times of two applications: most recent app comes first
- * Specifically: the one whose start time value is greater.
- */
- private static class MostRecentlyStartedAppFirst
- implements Comparator<ApplicationReport>, Serializable {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- long x = r1.getStartTime();
- long y = r2.getStartTime();
- return compareTwoLongsReverse(x, y);
- }
- }
-
- /**
- * Compare the times of two applications: most recent app comes first.
- * "Recent"== the app whose start time <i>or finish time</i> is the greatest.
- */
- private static class MostRecentlyStartedOrFinishedFirst
- implements Comparator<ApplicationReport>, Serializable {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- long started1 = r1.getStartTime();
- long started2 = r2.getStartTime();
- long finished1 = r1.getFinishTime();
- long finished2 = r2.getFinishTime();
- long lastEvent1 = Math.max(started1, finished1);
- long lastEvent2 = Math.max(started2, finished2);
- return compareTwoLongsReverse(lastEvent1, lastEvent2);
- }
- }
-
- /**
- * Compare the times of two applications: most recently finished app comes first
- * Specifically: the one whose finish time value is greater.
- */
- private static class MostRecentAppFinishFirst
- implements Comparator<ApplicationReport>, Serializable {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- long x = r1.getFinishTime();
- long y = r2.getFinishTime();
- return compareTwoLongsReverse(x, y);
- }
- }
-
- /**
- * Compare two long values for sorting. As the return value for
- * comparators must be int, the simple value of <code>x-y</code>
- * is inapplicable
- * @param x x value
- * @param y y value
- * @return +ve if x is less than y, -ve if y is greater than x; 0 for equality
- */
- public static int compareTwoLongsReverse(long x, long y) {
- return (x < y) ? 1 : ((x == y) ? 0 : -1);
- }
-
- public static String getSystemEnv(String property) {
- return System.getenv(property);
- }
-
- public static Map<String, String> getSystemEnv() {
- return System.getenv();
- }
-
- public static String requestToString(AMRMClient.ContainerRequest request) {
- Preconditions.checkArgument(request != null, "Null request");
- StringBuilder buffer = new StringBuilder(request.toString());
- buffer.append("; ");
- buffer.append("relaxLocality=").append(request.getRelaxLocality()).append("; ");
- String labels = request.getNodeLabelExpression();
- if (labels != null) {
- buffer.append("nodeLabels=").append(labels).append("; ");
- }
- List<String> nodes = request.getNodes();
- if (nodes != null) {
- buffer.append("Nodes = [ ");
- int size = nodes.size();
- for (int i = 0; i < Math.min(NODE_LIST_LIMIT, size); i++) {
- buffer.append(nodes.get(i)).append(' ');
- }
- if (size > NODE_LIST_LIMIT) {
- buffer.append(String.format("...(total %d entries)", size));
- }
- buffer.append("]; ");
- }
- List<String> racks = request.getRacks();
- if (racks != null) {
- buffer.append("racks = [")
- .append(join(racks, ", ", false))
- .append("]; ");
- }
- return buffer.toString();
- }
-
- public static String trimPrefix(String prefix) {
- if (prefix != null && prefix.endsWith(COMPONENT_SEPARATOR)) {
- return prefix.substring(0, prefix.length()-1);
- }
- return prefix;
- }
-
- public static String createNameTag(String name) {
- return "Name: " + name;
- }
-
- public static String createVersionTag(String version) {
- return "Version: " + version;
- }
-
- public static String createDescriptionTag(String description) {
- return "Description: " + description;
- }
-
- /**
- * Get all YarnApplicationState values which signify that an application is
- * in RUNNING or pre-RUNNING state.
- *
- * @return all live app states
- */
- public static EnumSet<YarnApplicationState> getAllLiveAppStates() {
- return EnumSet.range(YarnApplicationState.NEW,
- YarnApplicationState.RUNNING);
- }
-
- /**
- * Get all YarnApplicationState values which signify that an application is
- * not live, which means it is in one of the post RUNNING states.
- *
- * @return all non live app states
- */
- public static EnumSet<YarnApplicationState> getAllNonLiveAppStates() {
- return EnumSet.range(YarnApplicationState.FINISHED,
- YarnApplicationState.KILLED);
- }
-
- public static final String DAYS = ".days";
- public static final String HOURS = ".hours";
- public static final String MINUTES = ".minutes";
- public static final String SECONDS = ".seconds";
-
- /**
- * Get the time range of a set of keys.
- * @param conf configuration to read properties from
- * @param basekey base key to which suffix gets applied
- * @param defDays
- * @param defHours
- * @param defMins
- * @param defSecs
- * @return the aggregate time range in seconds
- */
- public static long getTimeRange(org.apache.slider.api.resource
- .Configuration conf,
- String basekey,
- long defDays,
- long defHours,
- long defMins,
- long defSecs) {
- Preconditions.checkArgument(basekey != null);
- long days = conf.getPropertyLong(basekey + DAYS, defDays);
- long hours = conf.getPropertyLong(basekey + HOURS, defHours);
-
- long minutes = conf.getPropertyLong(basekey + MINUTES, defMins);
- long seconds = conf.getPropertyLong(basekey + SECONDS, defSecs);
- // range check
- Preconditions.checkState(days >= 0 && hours >= 0 && minutes >= 0
- && seconds >= 0,
- "Time range for %s has negative time component %s:%s:%s:%s",
- basekey, days, hours, minutes, seconds);
-
- // calculate total time, schedule the reset if expected
- long totalMinutes = days * 24 * 60 + hours * 24 + minutes;
- return totalMinutes * 60 + seconds;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
deleted file mode 100644
index b3fdef9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.rest;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.api.json.JSONConfiguration;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-
-/**
- * Class to bond to a Jersey client, for UGI integration and SPNEGO.
- * <p>
- * Usage: create an instance, then when creating a Jersey <code>Client</code>
- * pass in to the constructor the handler provided by {@link #getHandler()}
- *
- * see <a href="https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html">Jersey docs</a>
- */
-public class UgiJerseyBinding implements
- HttpURLConnectionFactory {
- private static final Logger log =
- LoggerFactory.getLogger(UgiJerseyBinding.class);
-
- private final UrlConnectionOperations operations;
- private final URLConnectionClientHandler handler;
-
- /**
- * Construct an instance
- * @param operations operations instance
- */
- @SuppressWarnings("ThisEscapedInObjectConstruction")
- public UgiJerseyBinding(UrlConnectionOperations operations) {
- Preconditions.checkArgument(operations != null, "Null operations");
- this.operations = operations;
- handler = new URLConnectionClientHandler(this);
- }
-
- /**
- * Create an instance off the configuration. The SPNEGO policy
- * is derived from the current UGI settings.
- * @param conf config
- */
- public UgiJerseyBinding(Configuration conf) {
- this(new UrlConnectionOperations(conf));
- }
-
- /**
- * Get a URL connection.
- * @param url URL to connect to
- * @return the connection
- * @throws IOException any problem. {@link AuthenticationException}
- * errors are wrapped
- */
- @Override
- public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
- try {
- // open a connection handling status codes and so redirections
- // but as it opens a connection, it's less useful than you think.
-
- return operations.openConnection(url);
- } catch (AuthenticationException e) {
- throw new IOException(e);
- }
- }
-
- public UrlConnectionOperations getOperations() {
- return operations;
- }
-
- public URLConnectionClientHandler getHandler() {
- return handler;
- }
-
- /**
- * Get the SPNEGO flag (as found in the operations instance
- * @return the spnego policy
- */
- public boolean isUseSpnego() {
- return operations.isUseSpnego();
- }
-
-
- /**
- * Uprate error codes 400 and up into faults;
- * <p>
- * see {@link ExceptionConverter#convertJerseyException(String, String, UniformInterfaceException)}
- */
- public static IOException uprateFaults(HttpVerb verb, String url,
- UniformInterfaceException ex)
- throws IOException {
- return ExceptionConverter.convertJerseyException(verb.getVerb(),
- url, ex);
- }
-
- /**
- * Create the standard Jersey client Config
- * @return the recommended Jersey Client config
- */
- public ClientConfig createJerseyClientConfig() {
- ClientConfig clientConfig = new DefaultClientConfig();
- clientConfig.getFeatures().put(JSONConfiguration.FEATURE_POJO_MAPPING, true);
- return clientConfig;
- }
-
- /**
- * Create a jersey client bonded to this handler, using the
- * supplied client config
- * @param clientConfig client configuratin
- * @return a new client instance to use
- */
- public Client createJerseyClient(ClientConfig clientConfig) {
- return new Client(getHandler(), clientConfig);
- }
-
- /**
- * Create a jersey client bonded to this handler, using the
- * client config created with {@link #createJerseyClientConfig()}
- * @return a new client instance to use
- */
- public Client createJerseyClient() {
- return createJerseyClient(createJerseyClientConfig());
- }
-
-}
-
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UrlConnectionOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UrlConnectionOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UrlConnectionOperations.java
deleted file mode 100644
index d7f768e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UrlConnectionOperations.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.rest;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-
-/**
- * Operations on the JDK UrlConnection class.
- *
- */
-public class UrlConnectionOperations extends Configured {
- private static final Logger log =
- LoggerFactory.getLogger(UrlConnectionOperations.class);
-
- private SliderURLConnectionFactory connectionFactory;
-
- private boolean useSpnego = false;
-
- /**
- * Create an instance off the configuration. The SPNEGO policy
- * is derived from the current UGI settings.
- * @param conf config
- */
- public UrlConnectionOperations(Configuration conf) {
- super(conf);
- connectionFactory = SliderURLConnectionFactory.newInstance(conf);
- if (UserGroupInformation.isSecurityEnabled()) {
- log.debug("SPNEGO is enabled");
- setUseSpnego(true);
- }
- }
-
-
- public boolean isUseSpnego() {
- return useSpnego;
- }
-
- public void setUseSpnego(boolean useSpnego) {
- this.useSpnego = useSpnego;
- }
-
- /**
- * Opens a url with cache disabled, redirect handled in
- * (JDK) implementation.
- *
- * @param url to open
- * @return URLConnection
- * @throws IOException
- * @throws AuthenticationException authentication failure
- */
- public HttpURLConnection openConnection(URL url) throws
- IOException,
- AuthenticationException {
- Preconditions.checkArgument(url.getPort() != 0, "no port");
- return (HttpURLConnection) connectionFactory.openConnection(url, useSpnego);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/ServiceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/ServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/ServiceMonitor.java
deleted file mode 100644
index 98a76ea..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/ServiceMonitor.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.component.ComponentEvent;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
-import org.apache.hadoop.yarn.service.component.ComponentState;
-import org.apache.hadoop.yarn.service.servicemonitor.probe.ProbeStatus;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.RUNNING_BUT_UNREADY;
-import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.BECOME_NOT_READY;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.BECOME_READY;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.READY;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_WINDOW;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.DEFAULT_READINESS_CHECK_INTERVAL;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.READINESS_CHECK_INTERVAL;
-
-public class ServiceMonitor extends AbstractService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceMonitor.class);
-
- public ScheduledExecutorService executorService;
- private Map<ContainerId, ComponentInstance> liveInstances = null;
- private ServiceContext context;
- private Configuration conf;
-
- public ServiceMonitor(String name, ServiceContext context) {
- super(name);
- liveInstances = context.scheduler.getLiveInstances();
- this.context = context;
- }
-
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- executorService = Executors.newScheduledThreadPool(1);
- this.conf = conf;
- super.serviceInit(conf);
- }
-
- @Override
- public void serviceStart() throws Exception {
- long readinessCheckInterval = YarnServiceConf
- .getLong(READINESS_CHECK_INTERVAL, DEFAULT_READINESS_CHECK_INTERVAL,
- context.application.getConfiguration(), conf);
-
- executorService
- .scheduleAtFixedRate(new ReadinessChecker(), readinessCheckInterval,
- readinessCheckInterval, TimeUnit.SECONDS);
-
- // Default 6 hours.
- long failureResetInterval = YarnServiceConf
- .getLong(CONTAINER_FAILURE_WINDOW, 21600,
- context.application.getConfiguration(), conf);
-
- executorService
- .scheduleAtFixedRate(new ContainerFailureReset(), failureResetInterval,
- failureResetInterval, TimeUnit.SECONDS);
- }
-
- @Override
- public void serviceStop() throws Exception {
- if (executorService != null) {
- executorService.shutdownNow();
- }
- }
-
- private class ReadinessChecker implements Runnable {
-
- @Override
- public void run() {
-
- // check if the comp instance are ready
- for (Map.Entry<ContainerId, ComponentInstance> entry : liveInstances
- .entrySet()) {
- ComponentInstance instance = entry.getValue();
-
- ProbeStatus status = instance.ping();
- if (status.isSuccess()) {
- if (instance.getState() == RUNNING_BUT_UNREADY) {
- // synchronously update the state.
- instance.handle(
- new ComponentInstanceEvent(entry.getKey(), BECOME_READY));
- }
- } else {
- if (instance.getState() == READY) {
- instance.handle(
- new ComponentInstanceEvent(entry.getKey(), BECOME_NOT_READY));
- }
- }
- }
-
- for (Component component : context.scheduler.getAllComponents()
- .values()) {
- // If comp hasn't started yet and its dependencies are satisfied
- if (component.getState() == ComponentState.INIT && component
- .areDependenciesReady()) {
- LOG.info("[COMPONENT {}]: Dependencies satisfied, ramping up.",
- component.getName());
- ComponentEvent event = new ComponentEvent(component.getName(), FLEX)
- .setDesired(component.getComponentSpec().getNumberOfContainers());
- component.handle(event);
- }
- }
- }
- }
-
- private class ContainerFailureReset implements Runnable {
- @Override
- public void run() {
- for (Component component : context.scheduler.getAllComponents().values()) {
- component.resetCompFailureCount();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/HttpProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/HttpProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/HttpProbe.java
deleted file mode 100644
index 10c1160..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/HttpProbe.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor.probe;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.Map;
-
-public class HttpProbe extends Probe {
- protected static final Logger log = LoggerFactory.getLogger(HttpProbe.class);
-
- private static final String HOST_TOKEN = "${THIS_HOST}";
-
- private final String urlString;
- private final int timeout;
- private final int min, max;
-
-
- public HttpProbe(String url, int timeout, int min, int max, Configuration
- conf) {
- super("Http probe of " + url + " [" + min + "-" + max + "]", conf);
- this.urlString = url;
- this.timeout = timeout;
- this.min = min;
- this.max = max;
- }
-
- public static HttpProbe create(Map<String, String> props)
- throws IOException {
- String urlString = getProperty(props, WEB_PROBE_URL, null);
- new URL(urlString);
- int timeout = getPropertyInt(props, WEB_PROBE_CONNECT_TIMEOUT,
- WEB_PROBE_CONNECT_TIMEOUT_DEFAULT);
- int minSuccess = getPropertyInt(props, WEB_PROBE_MIN_SUCCESS,
- WEB_PROBE_MIN_SUCCESS_DEFAULT);
- int maxSuccess = getPropertyInt(props, WEB_PROBE_MAX_SUCCESS,
- WEB_PROBE_MAX_SUCCESS_DEFAULT);
- return new HttpProbe(urlString, timeout, minSuccess, maxSuccess, null);
- }
-
-
- private static HttpURLConnection getConnection(URL url, int timeout) throws
- IOException {
- HttpURLConnection connection = (HttpURLConnection) url.openConnection();
- connection.setInstanceFollowRedirects(true);
- connection.setConnectTimeout(timeout);
- return connection;
- }
-
- @Override
- public ProbeStatus ping(ComponentInstance instance) {
- ProbeStatus status = new ProbeStatus();
- ContainerStatus containerStatus = instance.getContainerStatus();
- if (containerStatus == null || SliderUtils.isEmpty(containerStatus.getIPs())
- || StringUtils.isEmpty(containerStatus.getHost())) {
- status.fail(this, new IOException("IP is not available yet"));
- return status;
- }
-
- String ip = containerStatus.getIPs().get(0);
- HttpURLConnection connection = null;
- try {
- URL url = new URL(urlString.replace(HOST_TOKEN, ip));
- connection = getConnection(url, this.timeout);
- int rc = connection.getResponseCode();
- if (rc < min || rc > max) {
- String error = "Probe " + url + " error code: " + rc;
- log.info(error);
- status.fail(this,
- new IOException(error));
- } else {
- status.succeed(this);
- }
- } catch (Throwable e) {
- String error = "Probe " + urlString + " failed for IP " + ip + ": " + e;
- log.info(error, e);
- status.fail(this,
- new IOException(error, e));
- } finally {
- if (connection != null) {
- connection.disconnect();
- }
- }
- return status;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/LogEntryBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/LogEntryBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/LogEntryBuilder.java
deleted file mode 100644
index b575d69..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/LogEntryBuilder.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor.probe;
-
-/**
- * Build up log entries for ease of splunk
- */
-public class LogEntryBuilder {
-
- private final StringBuilder builder = new StringBuilder();
-
- public LogEntryBuilder() {
- }
-
- public LogEntryBuilder(String text) {
- elt(text);
- }
-
-
- public LogEntryBuilder(String name, Object value) {
- entry(name, value);
- }
-
- public LogEntryBuilder elt(String text) {
- addComma();
- builder.append(text);
- return this;
- }
-
- public LogEntryBuilder elt(String name, Object value) {
- addComma();
- entry(name, value);
- return this;
- }
-
- private void addComma() {
- if (!isEmpty()) {
- builder.append(", ");
- }
- }
-
- private void entry(String name, Object value) {
- builder.append(name).append('=');
- if (value != null) {
- builder.append('"').append(value.toString()).append('"');
- } else {
- builder.append("null");
- }
- }
-
- @Override
- public String toString() {
- return builder.toString();
- }
-
- private boolean isEmpty() {
- return builder.length() == 0;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorKeys.java
deleted file mode 100644
index f5f3d99..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorKeys.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor.probe;
-
-/**
- * Config keys for monitoring
- */
-public interface MonitorKeys {
-
- /**
- * Port probing key : port to attempt to create a TCP connection to {@value}.
- */
- String PORT_PROBE_PORT = "port";
- /**
- * Port probing key : timeout for the the connection attempt {@value}.
- */
- String PORT_PROBE_CONNECT_TIMEOUT = "timeout";
- /**
- * Port probing default : timeout for the connection attempt {@value}.
- */
- int PORT_PROBE_CONNECT_TIMEOUT_DEFAULT = 1000;
-
- /**
- * Web probing key : URL {@value}.
- */
- String WEB_PROBE_URL = "url";
- /**
- * Web probing key : min success code {@value}.
- */
- String WEB_PROBE_MIN_SUCCESS = "min.success";
- /**
- * Web probing key : max success code {@value}.
- */
- String WEB_PROBE_MAX_SUCCESS = "max.success";
- /**
- * Web probing default : min successful response code {@value}.
- */
- int WEB_PROBE_MIN_SUCCESS_DEFAULT = 200;
- /**
- * Web probing default : max successful response code {@value}.
- */
- int WEB_PROBE_MAX_SUCCESS_DEFAULT = 299;
- /**
- * Web probing key : timeout for the connection attempt {@value}
- */
- String WEB_PROBE_CONNECT_TIMEOUT = "timeout";
- /**
- * Port probing default : timeout for the connection attempt {@value}.
- */
- int WEB_PROBE_CONNECT_TIMEOUT_DEFAULT = 1000;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorUtils.java
deleted file mode 100644
index 46d1fdb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorUtils.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor.probe;
-
-import org.apache.hadoop.yarn.service.api.records.ReadinessCheck;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Formatter;
-import java.util.Locale;
-
-/**
- * Various utils to work with the monitor
- */
-public final class MonitorUtils {
- protected static final Logger LOG = LoggerFactory.getLogger(MonitorUtils
- .class);
-
- private MonitorUtils() {
- }
-
- public static String toPlural(int val) {
- return val != 1 ? "s" : "";
- }
-
- /**
- * Convert milliseconds to human time -the exact format is unspecified
- * @param milliseconds a time in milliseconds
- * @return a time that is converted to human intervals
- */
- public static String millisToHumanTime(long milliseconds) {
- StringBuilder sb = new StringBuilder();
- // Send all output to the Appendable object sb
- Formatter formatter = new Formatter(sb, Locale.US);
-
- long s = Math.abs(milliseconds / 1000);
- long m = Math.abs(milliseconds % 1000);
- if (milliseconds > 0) {
- formatter.format("%d.%03ds", s, m);
- } else if (milliseconds == 0) {
- formatter.format("0");
- } else {
- formatter.format("-%d.%03ds", s, m);
- }
- return sb.toString();
- }
-
- public static Probe getProbe(ReadinessCheck readinessCheck) {
- if (readinessCheck == null) {
- return null;
- }
- if (readinessCheck.getType() == null) {
- return null;
- }
- try {
- switch (readinessCheck.getType()) {
- case HTTP:
- return HttpProbe.create(readinessCheck.getProps());
- case PORT:
- return PortProbe.create(readinessCheck.getProps());
- default:
- return null;
- }
- } catch (Throwable t) {
- throw new IllegalArgumentException("Error creating readiness check " +
- t);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/PortProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/PortProbe.java
deleted file mode 100644
index f6cf3ae..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/PortProbe.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor.probe;
-
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.util.Map;
-
-/**
- * Probe for a port being open.
- */
-public class PortProbe extends Probe {
- protected static final Logger log = LoggerFactory.getLogger(PortProbe.class);
- private final int port;
- private final int timeout;
-
- public PortProbe(int port, int timeout) {
- super("Port probe of " + port + " for " + timeout + "ms", null);
- this.port = port;
- this.timeout = timeout;
- }
-
- public static PortProbe create(Map<String, String> props)
- throws IOException {
- int port = getPropertyInt(props, PORT_PROBE_PORT, null);
-
- if (port >= 65536) {
- throw new IOException(PORT_PROBE_PORT + " " + port + " is out of " +
- "range");
- }
-
- int timeout = getPropertyInt(props, PORT_PROBE_CONNECT_TIMEOUT,
- PORT_PROBE_CONNECT_TIMEOUT_DEFAULT);
-
- return new PortProbe(port, timeout);
- }
-
- /**
- * Try to connect to the (host,port); a failure to connect within
- * the specified timeout is a failure.
- * @param instance role instance
- * @return the outcome
- */
- @Override
- public ProbeStatus ping(ComponentInstance instance) {
- ProbeStatus status = new ProbeStatus();
-
- if (instance.getContainerStatus() == null || SliderUtils
- .isEmpty(instance.getContainerStatus().getIPs())) {
- status.fail(this, new IOException(
- instance.getCompInstanceName() + ": IP is not available yet"));
- return status;
- }
-
- String ip = instance.getContainerStatus().getIPs().get(0);
- InetSocketAddress sockAddr = new InetSocketAddress(ip, port);
- Socket socket = new Socket();
- try {
- if (log.isDebugEnabled()) {
- log.debug(instance.getCompInstanceName() + ": Connecting " + sockAddr
- .toString() + ", timeout=" + MonitorUtils
- .millisToHumanTime(timeout));
- }
- socket.connect(sockAddr, timeout);
- status.succeed(this);
- } catch (Throwable e) {
- String error =
- instance.getCompInstanceName() + ": Probe " + sockAddr + " failed";
- log.debug(error, e);
- status.fail(this, new IOException(error, e));
- } finally {
- IOUtils.closeSocket(socket);
- }
- return status;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
deleted file mode 100644
index b851fb7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor.probe;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * Base class of all probes.
- */
-public abstract class Probe implements MonitorKeys {
-
- protected final Configuration conf;
- private String name;
-
- /**
- * Create a probe of a specific name
- *
- * @param name probe name
- * @param conf configuration being stored.
- */
- public Probe(String name, Configuration conf) {
- this.name = name;
- this.conf = conf;
- }
-
-
- protected void setName(String name) {
- this.name = name;
- }
-
- public String getName() {
- return name;
- }
-
-
- @Override
- public String toString() {
- return getName();
- }
-
- public static String getProperty(Map<String, String> props, String name,
- String defaultValue) throws IOException {
- String value = props.get(name);
- if (StringUtils.isEmpty(value)) {
- if (defaultValue == null) {
- throw new IOException(name + " not specified");
- }
- return defaultValue;
- }
- return value;
- }
-
- public static int getPropertyInt(Map<String, String> props, String name,
- Integer defaultValue) throws IOException {
- String value = props.get(name);
- if (StringUtils.isEmpty(value)) {
- if (defaultValue == null) {
- throw new IOException(name + " not specified");
- }
- return defaultValue;
- }
- return Integer.parseInt(value);
- }
-
- /**
- * perform any prelaunch initialization
- */
- public void init() throws IOException {
-
- }
-
- /**
- * Ping the endpoint. All exceptions must be caught and included in the
- * (failure) status.
- *
- * @param instance instance to ping
- * @return the status
- */
- public abstract ProbeStatus ping(ComponentInstance instance);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
deleted file mode 100644
index 7cd761c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor.probe;
-
-import java.io.Serializable;
-import java.util.Date;
-
-/**
- * Status message of a probe. This is designed to be sent over the wire, though the exception
- * Had better be unserializable at the far end if that is to work.
- */
-public final class ProbeStatus implements Serializable {
- private static final long serialVersionUID = 165468L;
-
- private long timestamp;
- private String timestampText;
- private boolean success;
- private boolean realOutcome;
- private String message;
- private Throwable thrown;
- private transient Probe originator;
-
- public ProbeStatus() {
- }
-
- public ProbeStatus(long timestamp, String message, Throwable thrown) {
- this.success = false;
- this.message = message;
- this.thrown = thrown;
- setTimestamp(timestamp);
- }
-
- public ProbeStatus(long timestamp, String message) {
- this.success = true;
- setTimestamp(timestamp);
- this.message = message;
- this.thrown = null;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public void setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- timestampText = new Date(timestamp).toString();
- }
-
- public boolean isSuccess() {
- return success;
- }
-
- /**
- * Set both the success and the real outcome bits to the same value
- * @param success the new value
- */
- public void setSuccess(boolean success) {
- this.success = success;
- realOutcome = success;
- }
-
- public String getTimestampText() {
- return timestampText;
- }
-
- public boolean getRealOutcome() {
- return realOutcome;
- }
-
- public String getMessage() {
- return message;
- }
-
- public void setMessage(String message) {
- this.message = message;
- }
-
- public Throwable getThrown() {
- return thrown;
- }
-
- public void setThrown(Throwable thrown) {
- this.thrown = thrown;
- }
-
- /**
- * Get the probe that generated this result. May be null
- * @return a possibly null reference to a probe
- */
- public Probe getOriginator() {
- return originator;
- }
-
- /**
- * The probe has succeeded -capture the current timestamp, set
- * success to true, and record any other data needed.
- * @param probe probe
- */
- public void succeed(Probe probe) {
- finish(probe, true, probe.getName(), null);
- }
-
- /**
- * A probe has failed either because the test returned false, or an exception
- * was thrown. The {@link #success} field is set to false, any exception
- * thrown is recorded.
- * @param probe probe that failed
- * @param thrown an exception that was thrown.
- */
- public void fail(Probe probe, Throwable thrown) {
- finish(probe, false, "Failure in " + probe, thrown);
- }
-
- public void finish(Probe probe, boolean succeeded, String text, Throwable thrown) {
- setTimestamp(System.currentTimeMillis());
- setSuccess(succeeded);
- originator = probe;
- message = text;
- this.thrown = thrown;
- }
-
- @Override
- public String toString() {
- LogEntryBuilder builder = new LogEntryBuilder("Probe Status");
- builder.elt("time", timestampText)
- .elt("outcome", (success ? "success" : "failure"));
-
- if (success != realOutcome) {
- builder.elt("originaloutcome", (realOutcome ? "success" : "failure"));
- }
- builder.elt("message", message);
- if (thrown != null) {
- builder.elt("exception", thrown);
- }
-
- return builder.toString();
- }
-
- /**
- * Flip the success bit on while the real outcome bit is kept false
- */
- public void markAsSuccessful() {
- success = true;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java
deleted file mode 100644
index cf4e836..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.timelineservice;
-
-import org.apache.commons.configuration2.SubsetConfiguration;
-import org.apache.hadoop.metrics2.MetricsRecord;
-import org.apache.hadoop.metrics2.MetricsSink;
-import org.apache.hadoop.metrics2.MetricsTag;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Write the metrics to a ATSv2. Generally, this class is instantiated via
- * hadoop-metrics2 property files. Specifically, you would create this class by
- * adding the following to by This would actually be set as: <code>
- * [prefix].sink.[some instance name].class
- * =org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink
- * </code>, where <tt>prefix</tt> is "atsv2": and <tt>some instance name</tt> is
- * just any unique name, so properties can be differentiated if there are
- * multiple sinks of the same type created
- */
-public class ServiceMetricsSink implements MetricsSink {
-
- private static final Logger log =
- LoggerFactory.getLogger(ServiceMetricsSink.class);
-
- private ServiceTimelinePublisher serviceTimelinePublisher;
-
- public ServiceMetricsSink() {
-
- }
-
- public ServiceMetricsSink(ServiceTimelinePublisher publisher) {
- serviceTimelinePublisher = publisher;
- }
-
- /**
- * Publishes service and component metrics to ATS.
- */
- @Override
- public void putMetrics(MetricsRecord record) {
- if (serviceTimelinePublisher.isStopped()) {
- log.warn("ServiceTimelinePublisher has stopped. "
- + "Not publishing any more metrics to ATS.");
- return;
- }
-
- boolean isServiceMetrics = false;
- boolean isComponentMetrics = false;
- String appId = null;
- for (MetricsTag tag : record.tags()) {
- if (tag.name().equals("type") && tag.value().equals("service")) {
- isServiceMetrics = true;
- } else if (tag.name().equals("type") && tag.value().equals("component")) {
- isComponentMetrics = true;
- break; // if component metrics, no more information required from tag so
- // break the loop
- } else if (tag.name().equals("appId")) {
- appId = tag.value();
- }
- }
-
- if (isServiceMetrics && appId != null) {
- if (log.isDebugEnabled()) {
- log.debug("Publishing service metrics. " + record);
- }
- serviceTimelinePublisher.publishMetrics(record.metrics(), appId,
- ServiceTimelineEntityType.SERVICE_ATTEMPT.toString(),
- record.timestamp());
- } else if (isComponentMetrics) {
- if (log.isDebugEnabled()) {
- log.debug("Publishing Component metrics. " + record);
- }
- serviceTimelinePublisher.publishMetrics(record.metrics(), record.name(),
- ServiceTimelineEntityType.COMPONENT.toString(), record.timestamp());
- }
- }
-
- @Override
- public void init(SubsetConfiguration conf) {
- }
-
- @Override
- public void flush() {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java
deleted file mode 100644
index d5c9539..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.timelineservice;
-
-/**
- * Slider entities that are published to ATS.
- */
-public enum ServiceTimelineEntityType {
- /**
- * Used for publishing service entity information.
- */
- SERVICE_ATTEMPT,
-
- /**
- * Used for publishing component entity information.
- */
- COMPONENT,
-
- /**
- * Used for publishing component instance entity information.
- */
- COMPONENT_INSTANCE
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
deleted file mode 100644
index 7f7f9a1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.timelineservice;
-
-/**
- * Events that are used to store in ATS.
- */
-public enum ServiceTimelineEvent {
- SERVICE_ATTEMPT_REGISTERED,
-
- SERVICE_ATTEMPT_UNREGISTERED,
-
- COMPONENT_INSTANCE_REGISTERED,
-
- COMPONENT_INSTANCE_UNREGISTERED,
-
- COMPONENT_INSTANCE_UPDATED
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
deleted file mode 100644
index 78a7171..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.timelineservice;
-
-/**
- * Constants which are stored as key in ATS
- */
-public final class ServiceTimelineMetricsConstants {
-
- public static final String URI = "URI";
-
- public static final String NAME = "NAME";
-
- public static final String STATE = "STATE";
-
- public static final String EXIT_STATUS_CODE = "EXIT_STATUS_CODE";
-
- public static final String EXIT_REASON = "EXIT_REASON";
-
- public static final String DIAGNOSTICS_INFO = "DIAGNOSTICS_INFO";
-
- public static final String LAUNCH_TIME = "LAUNCH_TIME";
-
- public static final String QUICK_LINKS = "QUICK_LINKS";
-
- public static final String LAUNCH_COMMAND = "LAUNCH_COMMAND";
-
- public static final String TOTAL_CONTAINERS = "NUMBER_OF_CONTAINERS";
-
- public static final String RUNNING_CONTAINERS =
- "NUMBER_OF_RUNNING_CONTAINERS";
-
- /**
- * Artifacts constants.
- */
- public static final String ARTIFACT_ID = "ARTIFACT_ID";
-
- public static final String ARTIFACT_TYPE = "ARTIFACT_TYPE";
-
- public static final String ARTIFACT_URI = "ARTIFACT_URI";
-
- /**
- * Resource constants.
- */
- public static final String RESOURCE_CPU = "RESOURCE_CPU";
-
- public static final String RESOURCE_MEMORY = "RESOURCE_MEMORY";
-
- public static final String RESOURCE_PROFILE = "RESOURCE_PROFILE";
-
- /**
- * component instance constants.
- */
- public static final String IP = "IP";
-
- public static final String HOSTNAME = "HOSTNAME";
-
- public static final String BARE_HOST = "BARE_HOST";
-
- public static final String COMPONENT_NAME = "COMPONENT_NAME";
-
- public static final String COMPONENT_INSTANCE_NAME = "COMPONENT_INSTANCE_NAME";
-
- /**
- * component constants.
- */
- public static final String DEPENDENCIES = "DEPENDENCIES";
-
- public static final String DESCRIPTION = "DESCRIPTION";
-
- public static final String RUN_PRIVILEGED_CONTAINER =
- "RUN_PRIVILEGED_CONTAINER";
-
- public static final String PLACEMENT_POLICY = "PLACEMENT_POLICY";
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
deleted file mode 100644
index 243baea..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.timelineservice;
-
-import org.apache.hadoop.metrics2.AbstractMetric;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.api.records.Configuration;
-import org.apache.hadoop.yarn.service.api.records.Container;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import static org.apache.hadoop.yarn.service.timelineservice.ServiceTimelineMetricsConstants.DIAGNOSTICS_INFO;
-
-/**
- * A single service that publishes all the Timeline Entities.
- */
-public class ServiceTimelinePublisher extends CompositeService {
-
- // Number of bytes of config which can be published in one shot to ATSv2.
- public static final int ATS_CONFIG_PUBLISH_SIZE_BYTES = 10 * 1024;
-
- private TimelineV2Client timelineClient;
-
- private volatile boolean stopped = false;
-
- private static final Logger log =
- LoggerFactory.getLogger(ServiceTimelinePublisher.class);
-
- @Override
- protected void serviceInit(org.apache.hadoop.conf.Configuration configuration)
- throws Exception {
- addService(timelineClient);
- }
-
-
- @Override
- protected void serviceStop() throws Exception {
- stopped = true;
- super.serviceStop();
- }
-
- public boolean isStopped() {
- return stopped;
- }
-
- public ServiceTimelinePublisher(TimelineV2Client client) {
- super(ServiceTimelinePublisher.class.getName());
- timelineClient = client;
- }
-
- public void serviceAttemptRegistered(Application application,
- org.apache.hadoop.conf.Configuration systemConf) {
- long currentTimeMillis = application.getLaunchTime() == null
- ? System.currentTimeMillis() : application.getLaunchTime().getTime();
-
- TimelineEntity entity = createServiceAttemptEntity(application.getId());
- entity.setCreatedTime(currentTimeMillis);
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.NAME, application.getName());
- entityInfos.put(ServiceTimelineMetricsConstants.STATE,
- application.getState().toString());
- entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME,
- currentTimeMillis);
- entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS,
- application.getQuicklinks());
- entity.addInfo(entityInfos);
-
- // add an event
- TimelineEvent startEvent = new TimelineEvent();
- startEvent.setId(ServiceTimelineEvent.SERVICE_ATTEMPT_REGISTERED.toString());
- startEvent.setTimestamp(currentTimeMillis);
- entity.addEvent(startEvent);
-
- // publish before configurations published
- putEntity(entity);
-
- // publish system config - YarnConfiguration
- populateTimelineEntity(systemConf.iterator(), application.getId(),
- ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
- // publish user conf
- publishUserConf(application.getConfiguration(), application.getId(),
- ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
-
- // publish component as separate entity.
- publishComponents(application.getComponents());
- }
-
- public void serviceAttemptUpdated(Application application) {
- TimelineEntity entity = createServiceAttemptEntity(application.getId());
- entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS,
- application.getQuicklinks());
- putEntity(entity);
- }
-
- public void serviceAttemptUnregistered(ServiceContext context,
- String diagnostics) {
- TimelineEntity entity = createServiceAttemptEntity(
- context.attemptId.getApplicationId().toString());
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.STATE,
- FinalApplicationStatus.ENDED);
- entityInfos.put(DIAGNOSTICS_INFO, diagnostics);
- entity.addInfo(entityInfos);
-
- // add an event
- TimelineEvent finishEvent = new TimelineEvent();
- finishEvent
- .setId(ServiceTimelineEvent.SERVICE_ATTEMPT_UNREGISTERED.toString());
- finishEvent.setTimestamp(System.currentTimeMillis());
- entity.addEvent(finishEvent);
-
- putEntity(entity);
- }
-
- public void componentInstanceStarted(Container container,
- ComponentInstance instance) {
-
- TimelineEntity entity = createComponentInstanceEntity(container.getId());
- entity.setCreatedTime(container.getLaunchTime().getTime());
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.BARE_HOST,
- container.getBareHost());
- entityInfos.put(ServiceTimelineMetricsConstants.STATE,
- container.getState().toString());
- entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME,
- container.getLaunchTime().getTime());
- entityInfos.put(ServiceTimelineMetricsConstants.COMPONENT_NAME,
- instance.getCompName());
- entityInfos.put(ServiceTimelineMetricsConstants.COMPONENT_INSTANCE_NAME,
- instance.getCompInstanceName());
- entity.addInfo(entityInfos);
-
- // add an event
- TimelineEvent startEvent = new TimelineEvent();
- startEvent
- .setId(ServiceTimelineEvent.COMPONENT_INSTANCE_REGISTERED.toString());
- startEvent.setTimestamp(container.getLaunchTime().getTime());
- entity.addEvent(startEvent);
-
- putEntity(entity);
- }
-
- public void componentInstanceFinished(ComponentInstance instance,
- int exitCode, ContainerState state, String diagnostics) {
- TimelineEntity entity = createComponentInstanceEntity(
- instance.getContainer().getId().toString());
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.EXIT_STATUS_CODE,
- exitCode);
- entityInfos.put(DIAGNOSTICS_INFO, diagnostics);
- entityInfos.put(ServiceTimelineMetricsConstants.STATE, state);
- entity.addInfo(entityInfos);
-
- // add an event
- TimelineEvent startEvent = new TimelineEvent();
- startEvent
- .setId(ServiceTimelineEvent.COMPONENT_INSTANCE_UNREGISTERED.toString());
- startEvent.setTimestamp(System.currentTimeMillis());
- entity.addEvent(startEvent);
-
- putEntity(entity);
- }
-
- public void componentInstanceUpdated(Container container) {
- TimelineEntity entity = createComponentInstanceEntity(container.getId());
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.IP, container.getIp());
- entityInfos.put(ServiceTimelineMetricsConstants.HOSTNAME,
- container.getHostname());
- entityInfos.put(ServiceTimelineMetricsConstants.STATE,
- container.getState().toString());
- entity.addInfo(entityInfos);
-
- TimelineEvent updateEvent = new TimelineEvent();
- updateEvent
- .setId(ServiceTimelineEvent.COMPONENT_INSTANCE_UPDATED.toString());
- updateEvent.setTimestamp(System.currentTimeMillis());
- entity.addEvent(updateEvent);
-
- putEntity(entity);
- }
-
- private void publishComponents(List<Component> components) {
- long currentTimeMillis = System.currentTimeMillis();
- for (Component component : components) {
- TimelineEntity entity = createComponentEntity(component.getName());
- entity.setCreatedTime(currentTimeMillis);
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_ID,
- component.getArtifact().getId());
- entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_TYPE,
- component.getArtifact().getType().toString());
- if (component.getResource().getProfile() != null) {
- entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_PROFILE,
- component.getResource().getProfile());
- }
- entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_CPU,
- component.getResource().getCpus());
- entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_MEMORY,
- component.getResource().getMemory());
-
- if (component.getLaunchCommand() != null) {
- entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_COMMAND,
- component.getLaunchCommand());
- }
- entityInfos.put(ServiceTimelineMetricsConstants.RUN_PRIVILEGED_CONTAINER,
- component.getRunPrivilegedContainer().toString());
- if (component.getPlacementPolicy() != null) {
- entityInfos.put(ServiceTimelineMetricsConstants.PLACEMENT_POLICY,
- component.getPlacementPolicy().getLabel());
- }
- entity.addInfo(entityInfos);
-
- putEntity(entity);
-
- // publish component specific configurations
- publishUserConf(component.getConfiguration(), component.getName(),
- ServiceTimelineEntityType.COMPONENT.toString());
- }
- }
-
- private void publishUserConf(Configuration configuration,
- String entityId, String entityType) {
- populateTimelineEntity(configuration.getProperties().entrySet().iterator(),
- entityId, entityType);
-
- populateTimelineEntity(configuration.getEnv().entrySet().iterator(),
- entityId, entityType);
-
- for (ConfigFile configFile : configuration.getFiles()) {
- populateTimelineEntity(configFile.getProps().entrySet().iterator(),
- entityId, entityType);
- }
- }
-
- private void populateTimelineEntity(Iterator<Entry<String, String>> iterator,
- String entityId, String entityType) {
- int configSize = 0;
- TimelineEntity entity = createTimelineEntity(entityId, entityType);
- while (iterator.hasNext()) {
- Entry<String, String> entry = iterator.next();
- int size = entry.getKey().length() + entry.getValue().length();
- configSize += size;
- // Configs are split into multiple entities if they exceed 100kb in size.
- if (configSize > ATS_CONFIG_PUBLISH_SIZE_BYTES) {
- if (entity.getConfigs().size() > 0) {
- putEntity(entity);
- entity = createTimelineEntity(entityId, entityType);
- }
- configSize = size;
- }
- entity.addConfig(entry.getKey(), entry.getValue());
- }
- if (configSize > 0) {
- putEntity(entity);
- }
- }
-
- /**
- * Called from ServiceMetricsSink at regular interval of time.
- * @param metrics of service or components
- * @param entityId Id of entity
- * @param entityType Type of entity
- * @param timestamp
- */
- public void publishMetrics(Iterable<AbstractMetric> metrics, String entityId,
- String entityType, long timestamp) {
- TimelineEntity entity = createTimelineEntity(entityId, entityType);
- Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>();
- for (AbstractMetric metric : metrics) {
- TimelineMetric timelineMetric = new TimelineMetric();
- timelineMetric.setId(metric.name());
- timelineMetric.addValue(timestamp, metric.value());
- entityMetrics.add(timelineMetric);
- }
- entity.setMetrics(entityMetrics);
- putEntity(entity);
- }
-
- private TimelineEntity createServiceAttemptEntity(String serviceId) {
- TimelineEntity entity = createTimelineEntity(serviceId,
- ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
- return entity;
- }
-
- private TimelineEntity createComponentInstanceEntity(String instanceId) {
- TimelineEntity entity = createTimelineEntity(instanceId,
- ServiceTimelineEntityType.COMPONENT_INSTANCE.toString());
- return entity;
- }
-
- private TimelineEntity createComponentEntity(String componentId) {
- TimelineEntity entity = createTimelineEntity(componentId,
- ServiceTimelineEntityType.COMPONENT.toString());
- return entity;
- }
-
- private TimelineEntity createTimelineEntity(String entityId,
- String entityType) {
- TimelineEntity entity = new TimelineEntity();
- entity.setId(entityId);
- entity.setType(entityType);
- return entity;
- }
-
- private void putEntity(TimelineEntity entity) {
- try {
- if (log.isDebugEnabled()) {
- log.debug("Publishing the entity " + entity + ", JSON-style content: "
- + TimelineUtils.dumpTimelineRecordtoJSON(entity));
- }
- if (timelineClient != null) {
- timelineClient.putEntitiesAsync(entity);
- } else {
- log.error("Seems like client has been removed before the entity "
- + "could be published for " + entity);
- }
- } catch (Exception e) {
- log.error("Error when publishing entity " + entity, e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java
deleted file mode 100644
index 72f7842..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * ATS implementation
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.yarn.service.timelineservice;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java
deleted file mode 100644
index 2607c08..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.codehaus.jackson.JsonGenerationException;
-import org.codehaus.jackson.JsonParseException;
-import org.codehaus.jackson.map.JsonMappingException;
-
-import java.io.IOException;
-
-/**
- * Persistence of {@link SerializedApplicationReport}
- *
- */
-public class ApplicationReportSerDeser
- extends JsonSerDeser<SerializedApplicationReport> {
- public ApplicationReportSerDeser() {
- super(SerializedApplicationReport.class);
- }
-
-
- private static final ApplicationReportSerDeser
- staticinstance = new ApplicationReportSerDeser();
-
- /**
- * Convert an instance to a JSON string -sync access to a shared ser/deser
- * object instance
- * @param instance object to convert
- * @return a JSON string description
- * @throws JsonParseException parse problems
- * @throws JsonMappingException O/J mapping problems
- */
- public static String toString(SerializedApplicationReport instance)
- throws IOException, JsonGenerationException, JsonMappingException {
- synchronized (staticinstance) {
- return staticinstance.toJson(instance);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
deleted file mode 100644
index 86896b2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
-import org.apache.hadoop.registry.client.impl.zk.RegistryInternalConstants;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.encodeForRegistry;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.convertUsername;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.getCurrentUsernameUnencoded;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
-
-/**
- * Generic code to get the URLs for clients via the registry
- */
-public class ClientRegistryBinder {
- private static final Logger log =
- LoggerFactory.getLogger(ClientRegistryBinder.class);
-
- private final RegistryOperations operations;
-
- public ClientRegistryBinder(RegistryOperations operations) {
- this.operations = operations;
- }
-
- /**
- * Buld the user path -switches to the system path if the user is "".
- * It also cross-converts the username to ascii via punycode
- * @param username username or ""
- * @return the path to the user
- */
- public static String homePathForUser(String username) {
- Preconditions.checkArgument(username != null, "null user");
-
- // catch recursion
- if (username.startsWith(RegistryConstants.PATH_USERS)) {
- return username;
- }
-
- if (username.isEmpty()) {
- return RegistryConstants.PATH_SYSTEM_SERVICES;
- }
-
- // convert username to registry name
- String convertedName = convertUsername(username);
-
- return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
- encodeForRegistry(convertedName));
- }
-
- /**
- * Get the current username, before any encoding has been applied.
- * @return the current user from the kerberos identity, falling back
- * to the user and/or env variables.
- */
- public static String currentUsernameUnencoded() {
- String env_hadoop_username = System.getenv(
- RegistryInternalConstants.HADOOP_USER_NAME);
- return getCurrentUsernameUnencoded(env_hadoop_username);
- }
-
- /**
- * Qualify a user.
- * <ol>
- * <li> <code>"~"</code> maps to user home path home</li>
- * <li> <code>"~user"</code> maps to <code>/users/$user</code></li>
- * <li> <code>"/"</code> maps to <code>/services/</code></li>
- * </ol>
- * @param user the username
- * @return the base path
- */
- public static String qualifyUser(String user) {
- // qualify the user
- String t = user.trim();
- if (t.startsWith("/")) {
- // already resolved
- return t;
- } else if (t.equals("~")) {
- // self
- return currentUsernameUnencoded();
- } else if (t.startsWith("~")) {
- // another user
- // convert username to registry name
- String convertedName = convertUsername(t.substring(1));
-
- return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
- encodeForRegistry(convertedName));
- } else {
- return "/" + t;
- }
- }
-
- /**
- * Look up an external REST API
- * @param user user which will be qualified as per {@link #qualifyUser(String)}
- * @param serviceClass service class
- * @param instance instance name
- * @param api API
- * @return the API, or an exception is raised.
- * @throws IOException
- */
- public String lookupExternalRestAPI(String user,
- String serviceClass,
- String instance,
- String api)
- throws IOException {
- String qualified = qualifyUser(user);
- String path = servicePath(qualified, serviceClass, instance);
- String restAPI = resolveExternalRestAPI(api, path);
- if (restAPI == null) {
- throw new PathNotFoundException(path + " API " + api);
- }
- return restAPI;
- }
-
- /**
- * Resolve a service record then return an external REST API exported it.
- *
- * @param api API to resolve
- * @param path path of the service record
- * @return null if the record exists but the API is absent or it has no
- * REST endpoints.
- * @throws IOException resolution problems, as covered in
- * {@link RegistryOperations#resolve(String)}
- */
- protected String resolveExternalRestAPI(String api, String path) throws
- IOException {
- ServiceRecord record = operations.resolve(path);
- return lookupRestAPI(record, api, true);
- }
-
- /**
- * Look up an external REST API endpoint
- * @param record service record
- * @param api URI of api
- * @param external flag to indicate this is an external record
- * @return the first endpoint of the implementation, or null if there
- * is no entry for the API, implementation or it's the wrong type.
- */
- public static String lookupRestAPI(ServiceRecord record,
- String api, boolean external) throws InvalidRecordException {
- try {
- String url = null;
- Endpoint endpoint = getEndpoint(record, api, external);
- List<String> addresses =
- RegistryTypeUtils.retrieveAddressesUriType(endpoint);
- if (addresses != null && !addresses.isEmpty()) {
- url = addresses.get(0);
- }
- return url;
- } catch (InvalidRecordException e) {
- log.debug("looking for API {}", api, e);
- return null;
- }
- }
-
- /**
- * Get an endpont by API
- * @param record service record
- * @param api API
- * @param external flag to indicate this is an external record
- * @return the endpoint or null
- */
- public static Endpoint getEndpoint(ServiceRecord record,
- String api,
- boolean external) {
- return external ? record.getExternalEndpoint(api)
- : record.getInternalEndpoint(api);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java
deleted file mode 100644
index 9f0e5d4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import java.io.Serializable;
-import java.util.Comparator;
-
-/**
- * Some general comparators
- */
-public class Comparators {
-
- public static class LongComparator implements Comparator<Long>, Serializable {
- @Override
- public int compare(Long o1, Long o2) {
- return o1.compareTo(o2);
- }
- }
-
- public static class InvertedLongComparator
- implements Comparator<Long>, Serializable {
- @Override
- public int compare(Long o1, Long o2) {
- return o2.compareTo(o1);
- }
- }
-
- /**
- * Little template class to reverse any comparitor
- * @param <CompareType> the type that is being compared
- */
- public static class ComparatorReverser<CompareType> implements Comparator<CompareType>,
- Serializable {
-
- final Comparator<CompareType> instance;
-
- public ComparatorReverser(Comparator<CompareType> instance) {
- this.instance = instance;
- }
-
- @Override
- public int compare(CompareType first, CompareType second) {
- return instance.compare(second, first);
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[67/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
new file mode 100644
index 0000000..c3a2752
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -0,0 +1,892 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+import org.apache.hadoop.yarn.service.ServiceMaster;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
+import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
+import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import org.apache.hadoop.yarn.service.client.params.ClientArgs;
+import org.apache.hadoop.yarn.service.client.params.CommonArgs;
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.containerlaunch.JavaCommandLineBuilder;
+import org.apache.hadoop.yarn.service.utils.ZookeeperUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.text.MessageFormat;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_CREATE;
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_FLEX;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_QUEUE;
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
+import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class ServiceClient extends CompositeService
+ implements SliderExitCodes, YarnServiceConstants {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceClient.class);
+ private SliderFileSystem fs;
+ //TODO disable retry so that client / rest API doesn't block?
+ protected YarnClient yarnClient;
+ // Avoid looking up applicationId from fs all the time.
+ private Map<String, ApplicationId> cachedAppIds = new ConcurrentHashMap<>();
+
+ private RegistryOperations registryClient;
+ private CuratorFramework curatorClient;
+ private YarnRPC rpc;
+
+ private static EnumSet<YarnApplicationState> terminatedStates =
+ EnumSet.of(FINISHED, FAILED, KILLED);
+ private static EnumSet<YarnApplicationState> liveStates =
+ EnumSet.of(NEW, NEW_SAVING, SUBMITTED, ACCEPTED, RUNNING);
+ private static EnumSet<YarnApplicationState> preRunningStates =
+ EnumSet.of(NEW, NEW_SAVING, SUBMITTED, ACCEPTED);
+
+ public ServiceClient() {
+ super(ServiceClient.class.getName());
+ }
+
+ @Override protected void serviceInit(Configuration configuration)
+ throws Exception {
+ fs = new SliderFileSystem(configuration);
+ yarnClient = YarnClient.createYarnClient();
+ rpc = YarnRPC.create(configuration);
+ addService(yarnClient);
+ super.serviceInit(configuration);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (registryClient != null) {
+ registryClient.stop();
+ }
+ super.serviceStop();
+ }
+
+ private Service loadAppJsonFromLocalFS(
+ AbstractClusterBuildingActionArgs args) throws IOException {
+ File file = args.getAppDef();
+ Path filePath = new Path(file.getAbsolutePath());
+ LOG.info("Loading app json from: " + filePath);
+ Service service = jsonSerDeser
+ .load(FileSystem.getLocal(getConfig()), filePath);
+ if (args.lifetime > 0) {
+ service.setLifetime(args.lifetime);
+ }
+ service.setName(args.getClusterName());
+ return service;
+ }
+
+ public int actionBuild(AbstractClusterBuildingActionArgs args)
+ throws IOException, YarnException {
+ return actionBuild(loadAppJsonFromLocalFS(args));
+ }
+
+ public int actionBuild(Service service)
+ throws YarnException, IOException {
+ Path appDir = checkAppNotExistOnHdfs(service);
+ ServiceApiUtil.validateAndResolveService(service, fs, getConfig());
+ createDirAndPersistApp(appDir, service);
+ return EXIT_SUCCESS;
+ }
+
+ public int actionCreate(AbstractClusterBuildingActionArgs args)
+ throws IOException, YarnException {
+ actionCreate(loadAppJsonFromLocalFS(args));
+ return EXIT_SUCCESS;
+ }
+
+ public ApplicationId actionCreate(Service service)
+ throws IOException, YarnException {
+ String serviceName = service.getName();
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ ServiceApiUtil.validateAndResolveService(service, fs, getConfig());
+ verifyNoLiveAppInRM(serviceName, "create");
+ Path appDir = checkAppNotExistOnHdfs(service);
+
+ // Write the definition first and then submit - AM will read the definition
+ createDirAndPersistApp(appDir, service);
+ ApplicationId appId = submitApp(service);
+ cachedAppIds.put(serviceName, appId);
+ service.setId(appId.toString());
+ // update app definition with appId
+ persistAppDef(appDir, service);
+ return appId;
+ }
+
+ // Called by ServiceCLI
+ protected int actionFlexByCLI(ClientArgs args)
+ throws YarnException, IOException {
+ ActionFlexArgs flexArgs = args.getActionFlexArgs();
+ Map<String, Long> componentCounts =
+ new HashMap<>(flexArgs.getComponentMap().size());
+ Service persistedService =
+ ServiceApiUtil.loadService(fs, flexArgs.getClusterName());
+ if (!StringUtils.isEmpty(persistedService.getId())) {
+ cachedAppIds.put(persistedService.getName(),
+ ApplicationId.fromString(persistedService.getId()));
+ } else {
+ throw new YarnException(persistedService.getName()
+ + " appId is null, may be not submitted to YARN yet");
+ }
+
+ for (Map.Entry<String, String> entry : flexArgs.getComponentMap()
+ .entrySet()) {
+ String compName = entry.getKey();
+ ServiceApiUtil.validateNameFormat(compName, getConfig());
+ Component component = persistedService.getComponent(compName);
+ if (component == null) {
+ throw new IllegalArgumentException(entry.getKey() + " does not exist !");
+ }
+ long numberOfContainers =
+ parseNumberOfContainers(component, entry.getValue());
+ componentCounts.put(compName, numberOfContainers);
+ }
+ // throw usage exception if no changes proposed
+ if (componentCounts.size() == 0) {
+ actionHelp(ACTION_FLEX, args);
+ }
+ flexComponents(args.getClusterName(), componentCounts, persistedService);
+ return EXIT_SUCCESS;
+ }
+
+ // Parse the number of containers requested by user, e.g.
+ // +5 means add 5 additional containers
+ // -5 means reduce 5 containers, if it goes to negative, sets it to 0
+ // 5 means sets it to 5 containers.
+ private long parseNumberOfContainers(Component component, String newNumber) {
+
+ long orig = component.getNumberOfContainers();
+ if (newNumber.startsWith("+")) {
+ return orig + Long.parseLong(newNumber.substring(1));
+ } else if (newNumber.startsWith("-")) {
+ long ret = orig - Long.parseLong(newNumber.substring(1));
+ if (ret < 0) {
+ LOG.warn(MessageFormat.format(
+ "[COMPONENT {}]: component count goes to negative ({}{} = {}), reset it to 0.",
+ component.getName(), orig, newNumber, ret));
+ ret = 0;
+ }
+ return ret;
+ } else {
+ return Long.parseLong(newNumber);
+ }
+ }
+
+ // Called by Rest Service
+ public Map<String, Long> flexByRestService(String serviceName,
+ Map<String, Long> componentCounts) throws YarnException, IOException {
+ // load app definition
+ Service persistedService = ServiceApiUtil.loadService(fs, serviceName);
+ if (StringUtils.isEmpty(persistedService.getId())) {
+ throw new YarnException(
+ serviceName + " appId is null, may be not submitted to YARN yet");
+ }
+ cachedAppIds.put(persistedService.getName(),
+ ApplicationId.fromString(persistedService.getId()));
+ return flexComponents(serviceName, componentCounts, persistedService);
+ }
+
+ private Map<String, Long> flexComponents(String serviceName,
+ Map<String, Long> componentCounts, Service persistedService)
+ throws YarnException, IOException {
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+
+ Map<String, Long> original = new HashMap<>(componentCounts.size());
+
+ ComponentCountProto.Builder countBuilder = ComponentCountProto.newBuilder();
+ FlexComponentsRequestProto.Builder requestBuilder =
+ FlexComponentsRequestProto.newBuilder();
+
+ for (Component persistedComp : persistedService.getComponents()) {
+ String name = persistedComp.getName();
+ if (componentCounts.containsKey(persistedComp.getName())) {
+ original.put(name, persistedComp.getNumberOfContainers());
+ persistedComp.setNumberOfContainers(componentCounts.get(name));
+
+ // build the request
+ countBuilder.setName(persistedComp.getName())
+ .setNumberOfContainers(persistedComp.getNumberOfContainers());
+ requestBuilder.addComponents(countBuilder.build());
+ }
+ }
+ if (original.size() < componentCounts.size()) {
+ componentCounts.keySet().removeAll(original.keySet());
+ throw new YarnException("Components " + componentCounts.keySet()
+ + " do not exist in app definition.");
+ }
+ jsonSerDeser
+ .save(fs.getFileSystem(), ServiceApiUtil.getServiceJsonPath(fs, serviceName),
+ persistedService, true);
+
+ ApplicationReport appReport =
+ yarnClient.getApplicationReport(getAppId(serviceName));
+ if (appReport.getYarnApplicationState() != RUNNING) {
+ String message =
+ serviceName + " is at " + appReport.getYarnApplicationState()
+ + " state, flex can only be invoked when service is running";
+ LOG.error(message);
+ throw new YarnException(message);
+ }
+ if (StringUtils.isEmpty(appReport.getHost())) {
+ throw new YarnException(serviceName + " AM hostname is empty");
+ }
+ ClientAMProtocol proxy =
+ createAMProxy(appReport.getHost(), appReport.getRpcPort());
+ proxy.flexComponents(requestBuilder.build());
+ for (Map.Entry<String, Long> entry : original.entrySet()) {
+ LOG.info("[COMPONENT {}]: number of containers changed from {} to {}",
+ entry.getKey(), entry.getValue(),
+ componentCounts.get(entry.getKey()));
+ }
+ return original;
+ }
+
+ public int actionStop(String serviceName, boolean waitForAppStopped)
+ throws YarnException, IOException {
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ ApplicationId currentAppId = getAppId(serviceName);
+ ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
+ if (terminatedStates.contains(report.getYarnApplicationState())) {
+ LOG.info("Service {} is already in a terminated state {}", serviceName,
+ report.getYarnApplicationState());
+ return EXIT_SUCCESS;
+ }
+ if (preRunningStates.contains(report.getYarnApplicationState())) {
+ String msg = serviceName + " is at " + report.getYarnApplicationState()
+ + ", forcefully killed by user!";
+ yarnClient.killApplication(currentAppId, msg);
+ LOG.info(msg);
+ return EXIT_SUCCESS;
+ }
+ if (StringUtils.isEmpty(report.getHost())) {
+ throw new YarnException(serviceName + " AM hostname is empty");
+ }
+ LOG.info("Stopping service {}, with appId = {}", serviceName, currentAppId);
+ try {
+ ClientAMProtocol proxy =
+ createAMProxy(report.getHost(), report.getRpcPort());
+ cachedAppIds.remove(serviceName);
+ if (proxy != null) {
+ // try to stop the app gracefully.
+ StopRequestProto request = StopRequestProto.newBuilder().build();
+ proxy.stop(request);
+ LOG.info("Service " + serviceName + " is being gracefully stopped...");
+ } else {
+ yarnClient.killApplication(currentAppId,
+ serviceName + " is forcefully killed by user!");
+ LOG.info("Forcefully kill the service: " + serviceName);
+ return EXIT_SUCCESS;
+ }
+
+ if (!waitForAppStopped) {
+ return EXIT_SUCCESS;
+ }
+ // Wait until the app is killed.
+ long startTime = System.currentTimeMillis();
+ int pollCount = 0;
+ while (true) {
+ Thread.sleep(2000);
+ report = yarnClient.getApplicationReport(currentAppId);
+ if (terminatedStates.contains(report.getYarnApplicationState())) {
+ LOG.info("Service " + serviceName + " is stopped.");
+ break;
+ }
+ // Forcefully kill after 10 seconds.
+ if ((System.currentTimeMillis() - startTime) > 10000) {
+ LOG.info("Stop operation timeout stopping, forcefully kill the app "
+ + serviceName);
+ yarnClient.killApplication(currentAppId,
+ "Forcefully kill the app by user");
+ break;
+ }
+ if (++pollCount % 10 == 0) {
+ LOG.info("Waiting for service " + serviceName + " to be stopped.");
+ }
+ }
+ } catch (IOException | YarnException | InterruptedException e) {
+ LOG.info("Failed to stop " + serviceName
+ + " gracefully, forcefully kill the app.");
+ yarnClient.killApplication(currentAppId, "Forcefully kill the app");
+ }
+ return EXIT_SUCCESS;
+ }
+
+ public int actionDestroy(String serviceName) throws Exception {
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ verifyNoLiveAppInRM(serviceName, "Destroy");
+ Path appDir = fs.buildClusterDirPath(serviceName);
+ FileSystem fileSystem = fs.getFileSystem();
+ // remove from the appId cache
+ cachedAppIds.remove(serviceName);
+ if (fileSystem.exists(appDir)) {
+ if (fileSystem.delete(appDir, true)) {
+ LOG.info("Successfully deleted service dir for " + serviceName + ": "
+ + appDir);
+ } else {
+ String message =
+ "Failed to delete service + " + serviceName + " at: " + appDir;
+ LOG.info(message);
+ throw new YarnException(message);
+ }
+ }
+ deleteZKNode(serviceName);
+ String registryPath = ServiceRegistryUtils.registryPathForInstance(serviceName);
+ try {
+ getRegistryClient().delete(registryPath, true);
+ } catch (IOException e) {
+ LOG.warn("Error deleting registry entry {}", registryPath, e);
+ }
+ LOG.info("Destroyed cluster {}", serviceName);
+ return EXIT_SUCCESS;
+ }
+
+ private synchronized RegistryOperations getRegistryClient()
+ throws SliderException, IOException {
+
+ if (registryClient == null) {
+ registryClient =
+ RegistryOperationsFactory.createInstance("ServiceClient", getConfig());
+ registryClient.init(getConfig());
+ registryClient.start();
+ }
+ return registryClient;
+ }
+
+ private void deleteZKNode(String clusterName) throws Exception {
+ CuratorFramework curatorFramework = getCuratorClient();
+ String user = RegistryUtils.currentUser();
+ String zkPath = ServiceRegistryUtils.mkClusterPath(user, clusterName);
+ if (curatorFramework.checkExists().forPath(zkPath) != null) {
+ curatorFramework.delete().deletingChildrenIfNeeded().forPath(zkPath);
+ LOG.info("Deleted zookeeper path: " + zkPath);
+ }
+ }
+
+ private synchronized CuratorFramework getCuratorClient()
+ throws BadConfigException {
+ String registryQuorum =
+ getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+
+ // though if neither is set: trouble
+ if (SliderUtils.isUnset(registryQuorum)) {
+ throw new BadConfigException(
+ "No Zookeeper quorum provided in the" + " configuration property "
+ + RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+ }
+ ZookeeperUtils.splitToHostsAndPortsStrictly(registryQuorum);
+
+ if (curatorClient == null) {
+ curatorClient =
+ CuratorFrameworkFactory.builder().connectString(registryQuorum)
+ .sessionTimeoutMs(10000).retryPolicy(new RetryNTimes(5, 2000))
+ .build();
+ curatorClient.start();
+ }
+ return curatorClient;
+ }
+
+ private int actionHelp(String actionName, CommonArgs args)
+ throws YarnException, IOException {
+ throw new UsageException(CommonArgs.usage(args, actionName));
+ }
+
+ private void verifyNoLiveAppInRM(String serviceName, String action)
+ throws IOException, YarnException {
+ Set<String> types = new HashSet<>(1);
+ types.add(YarnServiceConstants.APP_TYPE);
+ Set<String> tags = null;
+ if (serviceName != null) {
+ tags = Collections.singleton(SliderUtils.createNameTag(serviceName));
+ }
+ GetApplicationsRequest request = GetApplicationsRequest.newInstance();
+ request.setApplicationTypes(types);
+ request.setApplicationTags(tags);
+ request.setApplicationStates(liveStates);
+ List<ApplicationReport> reports = yarnClient.getApplications(request);
+ if (!reports.isEmpty()) {
+ throw new YarnException(
+ "Failed to " + action + " service, as " + serviceName
+ + " already exists.");
+ }
+ }
+
+ private ApplicationId submitApp(Service app)
+ throws IOException, YarnException {
+ String serviceName = app.getName();
+ Configuration conf = getConfig();
+ Path appRootDir = fs.buildClusterDirPath(app.getName());
+
+ YarnClientApplication yarnApp = yarnClient.createApplication();
+ ApplicationSubmissionContext submissionContext =
+ yarnApp.getApplicationSubmissionContext();
+ ServiceApiUtil.validateCompResourceSize(
+ yarnApp.getNewApplicationResponse().getMaximumResourceCapability(),
+ app);
+
+ submissionContext.setKeepContainersAcrossApplicationAttempts(true);
+ if (app.getLifetime() > 0) {
+ Map<ApplicationTimeoutType, Long> appTimeout = new HashMap<>();
+ appTimeout.put(ApplicationTimeoutType.LIFETIME, app.getLifetime());
+ submissionContext.setApplicationTimeouts(appTimeout);
+ }
+ submissionContext.setMaxAppAttempts(conf.getInt(
+ YarnServiceConf.AM_RESTART_MAX, 2));
+
+ Map<String, LocalResource> localResources = new HashMap<>();
+
+ // copy local slideram-log4j.properties to hdfs and add to localResources
+ boolean hasAMLog4j =
+ addAMLog4jResource(serviceName, conf, localResources);
+ // copy jars to hdfs and add to localResources
+ addJarResource(serviceName, localResources);
+ // add keytab if in secure env
+ addKeytabResourceIfSecure(fs, localResources, conf, serviceName);
+ if (LOG.isDebugEnabled()) {
+ printLocalResources(localResources);
+ }
+ Map<String, String> env = addAMEnv(conf);
+
+ // create AM CLI
+ String cmdStr =
+ buildCommandLine(serviceName, conf, appRootDir, hasAMLog4j);
+ submissionContext.setResource(Resource.newInstance(YarnServiceConf
+ .getLong(YarnServiceConf.AM_RESOURCE_MEM, YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM,
+ app.getConfiguration(), conf), 1));
+ String queue = app.getQueue();
+ if (StringUtils.isEmpty(queue)) {
+ queue = conf.get(YARN_QUEUE, "default");
+ }
+ submissionContext.setQueue(queue);
+ submissionContext.setApplicationName(serviceName);
+ submissionContext.setApplicationType(YarnServiceConstants.APP_TYPE);
+ Set<String> appTags =
+ AbstractClientProvider.createApplicationTags(serviceName, null, null);
+ if (!appTags.isEmpty()) {
+ submissionContext.setApplicationTags(appTags);
+ }
+ ContainerLaunchContext amLaunchContext =
+ Records.newRecord(ContainerLaunchContext.class);
+ amLaunchContext.setCommands(Collections.singletonList(cmdStr));
+ amLaunchContext.setEnvironment(env);
+ amLaunchContext.setLocalResources(localResources);
+ submissionContext.setAMContainerSpec(amLaunchContext);
+ yarnClient.submitApplication(submissionContext);
+ return submissionContext.getApplicationId();
+ }
+
+ private void printLocalResources(Map<String, LocalResource> map) {
+ LOG.debug("Added LocalResource for localization: ");
+ StringBuilder builder = new StringBuilder();
+ for (Map.Entry<String, LocalResource> entry : map.entrySet()) {
+ builder.append(entry.getKey()).append(" -> ")
+ .append(entry.getValue().getResource().getFile())
+ .append(System.lineSeparator());
+ }
+ LOG.debug(builder.toString());
+ }
+
+ private String buildCommandLine(String serviceName, Configuration conf,
+ Path appRootDir, boolean hasSliderAMLog4j) throws BadConfigException {
+ JavaCommandLineBuilder CLI = new JavaCommandLineBuilder();
+ CLI.forceIPv4().headless();
+ //TODO CLI.setJVMHeap
+ //TODO CLI.addJVMOPTS
+ if (hasSliderAMLog4j) {
+ CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, YARN_SERVICE_LOG4J_FILENAME);
+ CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ }
+ CLI.add(ServiceMaster.class.getCanonicalName());
+ CLI.add(ACTION_CREATE, serviceName);
+ //TODO debugAM CLI.add(Arguments.ARG_DEBUG)
+ CLI.add(Arguments.ARG_CLUSTER_URI, new Path(appRootDir, serviceName + ".json"));
+ // pass the registry binding
+ CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
+ RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
+ CLI.addMandatoryConfOption(conf, RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+
+ // write out the path output
+ CLI.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
+ String cmdStr = CLI.build();
+ LOG.info("AM launch command: {}", cmdStr);
+ return cmdStr;
+ }
+
+ private Map<String, String> addAMEnv(Configuration conf) throws IOException {
+ Map<String, String> env = new HashMap<>();
+ ClasspathConstructor classpath =
+ buildClasspath(YarnServiceConstants.SUBMITTED_CONF_DIR, "lib", fs, getConfig()
+ .getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false));
+ env.put("CLASSPATH", classpath.buildClasspath());
+ env.put("LANG", "en_US.UTF-8");
+ env.put("LC_ALL", "en_US.UTF-8");
+ env.put("LANGUAGE", "en_US.UTF-8");
+ String jaas = System.getenv("HADOOP_JAAS_DEBUG");
+ if (jaas != null) {
+ env.put("HADOOP_JAAS_DEBUG", jaas);
+ }
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ String userName = UserGroupInformation.getCurrentUser().getUserName();
+ LOG.info("Run as user " + userName);
+ // HADOOP_USER_NAME env is used by UserGroupInformation when log in
+ // This env makes AM run as this user
+ env.put("HADOOP_USER_NAME", userName);
+ }
+ LOG.info("AM env: \n{}", stringifyMap(env));
+ return env;
+ }
+
+ protected Path addJarResource(String serviceName,
+ Map<String, LocalResource> localResources)
+ throws IOException, SliderException {
+ Path libPath = fs.buildClusterDirPath(serviceName);
+ ProviderUtils
+ .addProviderJar(localResources, ServiceMaster.class, SERVICE_CORE_JAR, fs,
+ libPath, "lib", false);
+ Path dependencyLibTarGzip = fs.getDependencyTarGzip();
+ if (fs.isFile(dependencyLibTarGzip)) {
+ LOG.info("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
+ + dependencyLibTarGzip);
+ SliderUtils.putAmTarGzipAndUpdate(localResources, fs);
+ } else {
+ String[] libs = SliderUtils.getLibDirs();
+ for (String libDirProp : libs) {
+ ProviderUtils.addAllDependencyJars(localResources, fs, libPath, "lib",
+ libDirProp);
+ }
+ }
+ return libPath;
+ }
+
+ private boolean addAMLog4jResource(String serviceName, Configuration conf,
+ Map<String, LocalResource> localResources)
+ throws IOException, BadClusterStateException {
+ boolean hasAMLog4j = false;
+ String hadoopConfDir =
+ System.getenv(ApplicationConstants.Environment.HADOOP_CONF_DIR.name());
+ if (hadoopConfDir != null) {
+ File localFile =
+ new File(hadoopConfDir, YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
+ if (localFile.exists()) {
+ Path localFilePath = createLocalPath(localFile);
+ Path appDirPath = fs.buildClusterDirPath(serviceName);
+ Path remoteConfPath =
+ new Path(appDirPath, YarnServiceConstants.SUBMITTED_CONF_DIR);
+ Path remoteFilePath =
+ new Path(remoteConfPath, YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
+ copy(conf, localFilePath, remoteFilePath);
+ LocalResource localResource =
+ fs.createAmResource(remoteConfPath, LocalResourceType.FILE);
+ localResources.put(localFilePath.getName(), localResource);
+ hasAMLog4j = true;
+ } else {
+ LOG.warn("AM log4j property file doesn't exist: " + localFile);
+ }
+ }
+ return hasAMLog4j;
+ }
+
+ public int actionStart(String serviceName) throws YarnException, IOException {
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ Path appDir = checkAppExistOnHdfs(serviceName);
+ Service service = ServiceApiUtil.loadService(fs, serviceName);
+ ServiceApiUtil.validateAndResolveService(service, fs, getConfig());
+ // see if it is actually running and bail out;
+ verifyNoLiveAppInRM(serviceName, "thaw");
+ ApplicationId appId = submitApp(service);
+ service.setId(appId.toString());
+ // write app definition on to hdfs
+ createDirAndPersistApp(appDir, service);
+ return 0;
+ }
+
+ private Path checkAppNotExistOnHdfs(Service service)
+ throws IOException, SliderException {
+ Path appDir = fs.buildClusterDirPath(service.getName());
+ fs.verifyDirectoryNonexistent(
+ new Path(appDir, service.getName() + ".json"));
+ return appDir;
+ }
+
+ private Path checkAppExistOnHdfs(String serviceName)
+ throws IOException, SliderException {
+ Path appDir = fs.buildClusterDirPath(serviceName);
+ fs.verifyPathExists(new Path(appDir, serviceName + ".json"));
+ return appDir;
+ }
+
+ private void createDirAndPersistApp(Path appDir, Service service)
+ throws IOException, SliderException {
+ FsPermission appDirPermission = new FsPermission("750");
+ fs.createWithPermissions(appDir, appDirPermission);
+ persistAppDef(appDir, service);
+ }
+
+ private void persistAppDef(Path appDir, Service service)
+ throws IOException {
+ Path appJson = new Path(appDir, service.getName() + ".json");
+ jsonSerDeser
+ .save(fs.getFileSystem(), appJson, service, true);
+ LOG.info(
+ "Persisted service " + service.getName() + " at " + appJson);
+ }
+
+ private void addKeytabResourceIfSecure(SliderFileSystem fileSystem,
+ Map<String, LocalResource> localResource, Configuration conf,
+ String serviceName) throws IOException, BadConfigException {
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ return;
+ }
+ String keytabPreInstalledOnHost =
+ conf.get(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
+ if (StringUtils.isEmpty(keytabPreInstalledOnHost)) {
+ String amKeytabName =
+ conf.get(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
+ String keytabDir = conf.get(YarnServiceConf.KEY_HDFS_KEYTAB_DIR);
+ Path keytabPath =
+ fileSystem.buildKeytabPath(keytabDir, amKeytabName, serviceName);
+ if (fileSystem.getFileSystem().exists(keytabPath)) {
+ LocalResource keytabRes =
+ fileSystem.createAmResource(keytabPath, LocalResourceType.FILE);
+ localResource
+ .put(YarnServiceConstants.KEYTAB_DIR + "/" + amKeytabName, keytabRes);
+ LOG.info("Adding AM keytab on hdfs: " + keytabPath);
+ } else {
+ LOG.warn("No keytab file was found at {}.", keytabPath);
+ if (conf.getBoolean(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) {
+ throw new BadConfigException("No keytab file was found at %s.",
+ keytabPath);
+ } else {
+ LOG.warn("The AM will be "
+ + "started without a kerberos authenticated identity. "
+ + "The service is therefore not guaranteed to remain "
+ + "operational beyond 24 hours.");
+ }
+ }
+ }
+ }
+
+ public String updateLifetime(String serviceName, long lifetime)
+ throws YarnException, IOException {
+ ApplicationId currentAppId = getAppId(serviceName);
+ ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
+ if (report == null) {
+ throw new YarnException("Service not found for " + serviceName);
+ }
+ ApplicationId appId = report.getApplicationId();
+ LOG.info("Updating lifetime of an service: serviceName = " + serviceName
+ + ", appId = " + appId + ", lifetime = " + lifetime);
+ Map<ApplicationTimeoutType, String> map = new HashMap<>();
+ String newTimeout =
+ Times.formatISO8601(System.currentTimeMillis() + lifetime * 1000);
+ map.put(ApplicationTimeoutType.LIFETIME, newTimeout);
+ UpdateApplicationTimeoutsRequest request =
+ UpdateApplicationTimeoutsRequest.newInstance(appId, map);
+ yarnClient.updateApplicationTimeouts(request);
+ LOG.info(
+ "Successfully updated lifetime for an service: serviceName = " + serviceName
+ + ", appId = " + appId + ". New expiry time in ISO8601 format is "
+ + newTimeout);
+ return newTimeout;
+ }
+
+ public ServiceState convertState(FinalApplicationStatus status) {
+ switch (status) {
+ case UNDEFINED:
+ return ServiceState.ACCEPTED;
+ case FAILED:
+ case KILLED:
+ return ServiceState.FAILED;
+ case ENDED:
+ case SUCCEEDED:
+ return ServiceState.STOPPED;
+ }
+ return ServiceState.ACCEPTED;
+ }
+
+ public Service getStatus(String serviceName)
+ throws IOException, YarnException {
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ ApplicationId currentAppId = getAppId(serviceName);
+ ApplicationReport appReport = yarnClient.getApplicationReport(currentAppId);
+ Service appSpec = new Service();
+ appSpec.setName(serviceName);
+ appSpec.setState(convertState(appReport.getFinalApplicationStatus()));
+ ApplicationTimeout lifetime =
+ appReport.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME);
+ if (lifetime != null) {
+ appSpec.setLifetime(lifetime.getRemainingTime());
+ }
+
+ if (appReport.getYarnApplicationState() != RUNNING) {
+ LOG.info("Service {} is at {} state", serviceName,
+ appReport.getYarnApplicationState());
+ return appSpec;
+ }
+ if (StringUtils.isEmpty(appReport.getHost())) {
+ LOG.warn(serviceName + " AM hostname is empty");
+ return appSpec;
+ }
+ ClientAMProtocol amProxy =
+ createAMProxy(appReport.getHost(), appReport.getRpcPort());
+ GetStatusResponseProto response =
+ amProxy.getStatus(GetStatusRequestProto.newBuilder().build());
+ appSpec = jsonSerDeser.fromJson(response.getStatus());
+
+ return appSpec;
+ }
+
+ public YarnClient getYarnClient() {
+ return this.yarnClient;
+ }
+
+ public int actionDependency(ActionDependencyArgs args)
+ throws IOException, YarnException {
+ String currentUser = RegistryUtils.currentUser();
+ LOG.info("Running command as user {}", currentUser);
+
+ Path dependencyLibTarGzip = fs.getDependencyTarGzip();
+
+ // Check if dependency has already been uploaded, in which case log
+ // appropriately and exit success (unless overwrite has been requested)
+ if (fs.isFile(dependencyLibTarGzip) && !args.overwrite) {
+ System.out.println(String.format(
+ "Dependency libs are already uploaded to %s. Use %s "
+ + "if you want to re-upload", dependencyLibTarGzip.toUri(),
+ Arguments.ARG_OVERWRITE));
+ return EXIT_SUCCESS;
+ }
+
+ String[] libDirs = SliderUtils.getLibDirs();
+ if (libDirs.length > 0) {
+ File tempLibTarGzipFile = File.createTempFile(
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+ // copy all jars
+ tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter());
+
+ LOG.info("Version Info: " + VersionInfo.getBuildVersion());
+ fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip,
+ new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS));
+ return EXIT_SUCCESS;
+ } else {
+ return EXIT_FALSE;
+ }
+ }
+
+ protected ClientAMProtocol createAMProxy(String host, int port)
+ throws IOException {
+ InetSocketAddress address =
+ NetUtils.createSocketAddrForHost(host, port);
+ return ClientAMProxy.createProxy(getConfig(), ClientAMProtocol.class,
+ UserGroupInformation.getCurrentUser(), rpc, address);
+ }
+
+ private synchronized ApplicationId getAppId(String serviceName)
+ throws IOException, YarnException {
+ if (cachedAppIds.containsKey(serviceName)) {
+ return cachedAppIds.get(serviceName);
+ }
+ Service persistedService = ServiceApiUtil.loadService(fs, serviceName);
+ if (persistedService == null) {
+ throw new YarnException("Service " + serviceName
+ + " doesn't exist on hdfs. Please check if the app exists in RM");
+ }
+ ApplicationId currentAppId = ApplicationId.fromString(persistedService.getId());
+ cachedAppIds.put(serviceName, currentAppId);
+ return currentAppId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
new file mode 100644
index 0000000..ea3bb0a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Base args for all actions
+ */
+public abstract class AbstractActionArgs extends ArgOps implements Arguments {
+ protected static final Logger log =
+ LoggerFactory.getLogger(AbstractActionArgs.class);
+
+
+ protected AbstractActionArgs() {
+ }
+
+ /**
+ * URI/binding to the filesystem
+ */
+ @Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG},
+ description = "Filesystem Binding")
+ public String filesystemBinding;
+
+ @Parameter(names = {ARG_BASE_PATH},
+ description = "Service base path on the filesystem",
+ converter = PathArgumentConverter.class)
+ public Path basePath;
+
+ /**
+ * This is the default parameter
+ */
+ @Parameter
+ public final List<String> parameters = new ArrayList<>();
+
+ /**
+ * get the name: relies on arg 1 being the cluster name in all operations
+ * @return the name argument, null if there is none
+ */
+ public String getClusterName() {
+ return (parameters.isEmpty()) ? null : parameters.get(0);
+ }
+
+ /**
+ -D name=value
+
+ Define an configuration option which overrides any options in
+ the configuration XML files of the image or in the image configuration
+ directory. The values will be persisted.
+ Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
+
+ */
+
+ @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions")
+ public final List<String> definitions = new ArrayList<>();
+
+ /**
+ * System properties
+ */
+ @Parameter(names = {ARG_SYSPROP}, arity = 1,
+ description = "system properties in the form name value" +
+ " These are set after the JVM is started.")
+ public final List<String> sysprops = new ArrayList<>(0);
+
+
+ @Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER},
+ description = "Binding (usually hostname:port) of the YARN resource manager")
+ public String manager;
+
+
+ @Parameter(names = ARG_DEBUG, description = "Debug mode")
+ public boolean debug = false;
+
+ @Parameter(names = {ARG_HELP}, description = "Help", help = true)
+ public boolean help = false;
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 1;
+ }
+
+ /**
+ * Get the name of the action
+ * @return the action name
+ */
+ public abstract String getActionName() ;
+
+ /**
+ * Get the max #of params expected
+ * @return the number of params in the {@link #parameters} field;
+ */
+ public int getMaxParams() {
+ return getMinParams();
+ }
+
+ public void validate() throws BadCommandArgumentsException, UsageException {
+
+ int minArgs = getMinParams();
+ int actionArgSize = parameters.size();
+ if (minArgs > actionArgSize) {
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName() +
+ ", Expected minimum " + minArgs + " but got " + actionArgSize);
+ }
+ int maxArgs = getMaxParams();
+ if (maxArgs == -1) {
+ maxArgs = minArgs;
+ }
+ if (actionArgSize > maxArgs) {
+ String message = String.format("%s for action %s: limit is %d but saw %d: ",
+ ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
+ getActionName(), maxArgs,
+ actionArgSize);
+
+ log.error(message);
+ int index = 1;
+ StringBuilder buf = new StringBuilder(message);
+ for (String actionArg : parameters) {
+ log.error("[{}] \"{}\"", index++, actionArg);
+ buf.append(" \"").append(actionArg).append("\" ");
+ }
+ throw new BadCommandArgumentsException(buf.toString());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + ": " + getActionName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
new file mode 100644
index 0000000..457e357
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import org.apache.hadoop.yarn.service.client.params.ArgOps;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+
+/**
+ * Base class for all the delegates
+ */
+public class AbstractArgsDelegate extends ArgOps implements Arguments {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
new file mode 100644
index 0000000..3a3a19a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParametersDelegate;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Abstract Action to build things; shares args across build and
+ * list
+ */
+public abstract class AbstractClusterBuildingActionArgs
+ extends AbstractActionArgs {
+ @Parameter(names = {ARG_APPDEF},
+ description = "Template service definition file in JSON format.")
+ public File appDef;
+
+ public File getAppDef() {
+ return appDef;
+ }
+
+ @Parameter(names = {
+ ARG_QUEUE }, description = "Queue to submit the service")
+ public String queue;
+
+ @Parameter(names = {
+ ARG_LIFETIME }, description = "Lifetime of the service from the time of request")
+ public long lifetime;
+
+ @ParametersDelegate
+ public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
+
+ @ParametersDelegate
+ public OptionArgsDelegate optionsDelegate =
+ new OptionArgsDelegate();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
new file mode 100644
index 0000000..c2ff545
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_BUILD},
+ commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
+
+public class ActionBuildArgs extends AbstractClusterBuildingActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_BUILD;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
new file mode 100644
index 0000000..c43d61a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+import java.io.File;
+
+@Parameters(commandNames = { SliderActions.ACTION_CLIENT},
+ commandDescription = SliderActions.DESCRIBE_ACTION_CLIENT)
+
+public class ActionClientArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_CLIENT;
+ }
+
+ @Parameter(names = {ARG_INSTALL},
+ description = "Install client")
+ public boolean install;
+
+ @Parameter(names = {ARG_NAME},
+ description = "The name of the service")
+ public String name;
+
+ @Parameter(names = {ARG_PACKAGE},
+ description = "Path to app package")
+ public String packageURI;
+
+ @Parameter(names = {ARG_DEST},
+ description = "The location where to install the client")
+ public File installLocation;
+
+ @Parameter(names = {ARG_CONFIG},
+ description = "Client configuration")
+ public File clientConfig;
+
+ /**
+ * Get the min #of params expected
+ *
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
new file mode 100644
index 0000000..eecffb6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_CREATE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
+
+public class ActionCreateArgs extends AbstractClusterBuildingActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_CREATE;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
new file mode 100644
index 0000000..51e07c9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.client.params;
+
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_DEPENDENCY },
+ commandDescription = SliderActions.DESCRIBE_ACTION_DEPENDENCY)
+public class ActionDependencyArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_DEPENDENCY;
+ }
+
+ @Parameter(names = { ARG_UPLOAD },
+ description = "Upload AM and agent libraries to HDFS for this client")
+ public boolean upload;
+
+ @Parameter(names = { ARG_OVERWRITE },
+ description = "Overwrite current uploaded dependency libs")
+ public boolean overwrite = false;
+
+ /**
+ * Get the min #of params expected
+ *
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 1;
+ }
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+
+ if (!upload) {
+ throw new UsageException("Option " + ARG_UPLOAD + " is mandatory");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
new file mode 100644
index 0000000..8c41c04
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_DESTROY},
+ commandDescription = SliderActions.DESCRIBE_ACTION_DESTROY)
+
+public class ActionDestroyArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_DESTROY;
+ }
+
+ @Parameter(names = {ARG_FORCE},
+ description = "force the operation")
+ public boolean force;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
new file mode 100644
index 0000000..088ad47
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+import java.io.File;
+
+@Parameters(commandNames = { SliderActions.ACTION_EXISTS},
+ commandDescription = SliderActions.DESCRIBE_ACTION_EXISTS)
+
+public class ActionExistsArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_EXISTS;
+ }
+
+ @Parameter(names = {ARG_LIVE},
+ description = "verify that the service is running")
+ public boolean live;
+
+ @Parameter(names = {ARG_STATE},
+ description = "verify that the service is in the specific YARN state")
+ public String state = "";
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for any service report")
+ public File out;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
new file mode 100644
index 0000000..b7acf58
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+
+import java.util.List;
+import java.util.Map;
+
+@Parameters(commandNames = { SliderActions.ACTION_FLEX},
+ commandDescription = SliderActions.DESCRIBE_ACTION_FLEX)
+
+public class ActionFlexArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_FLEX;
+ }
+
+ @ParametersDelegate
+ public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
+
+ /**
+ * Get the component mapping (may be empty, but never null)
+ * @return mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, String> getComponentMap() throws
+ BadCommandArgumentsException {
+ return componentDelegate.getComponentMap();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
new file mode 100644
index 0000000..aecf0eb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+@Parameters(commandNames = { SliderActions.ACTION_STOP },
+ commandDescription = SliderActions.DESCRIBE_ACTION_FREEZE)
+
+public class ActionFreezeArgs extends AbstractActionArgs implements
+ WaitTimeAccessor {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_STOP;
+ }
+
+ public static final String FREEZE_COMMAND_ISSUED = "stop command issued";
+ @ParametersDelegate
+ public WaitArgsDelegate waitDelegate = new WaitArgsDelegate();
+
+ @Override
+ public int getWaittime() {
+ return waitDelegate.getWaittime();
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ waitDelegate.setWaittime(waittime);
+ }
+
+ @Parameter(names={ARG_MESSAGE},
+ description = "reason for the operation")
+ public String message = FREEZE_COMMAND_ISSUED;
+
+ @Parameter(names = {ARG_FORCE},
+ description = "force the operation")
+ public boolean force;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
new file mode 100644
index 0000000..51aa88a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+/**
+ * The Help command
+ */
+@Parameters(commandNames = { SliderActions.ACTION_HELP},
+ commandDescription = SliderActions.DESCRIBE_ACTION_HELP)
+public class ActionHelpArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_HELP;
+ }
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
new file mode 100644
index 0000000..061121e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+@Parameters(commandNames = { SliderActions.ACTION_KDIAG},
+ commandDescription = SliderActions.DESCRIBE_ACTION_KDIAG)
+
+public class ActionKDiagArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_KDIAG;
+ }
+
+ @Parameter(names = {ARG_SERVICES}, variableArity = true,
+ description =" list of services to check")
+ public List<String> services = new ArrayList<>();
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for report")
+ public File out;
+
+ @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
+ public File keytab;
+
+ @Parameter(names = {ARG_KEYLEN}, description = "minimum key length")
+ public int keylen = 256;
+
+ @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
+ public String principal;
+
+ @Parameter(names = {ARG_SECURE}, description = "Is security required")
+ public boolean secure = false;
+
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+ if (keytab != null && SliderUtils.isUnset(principal)) {
+ throw new UsageException("Missing argument " + ARG_PRINCIPAL);
+ }
+ if (keytab == null && SliderUtils.isSet(principal)) {
+ throw new UsageException("Missing argument " + ARG_KEYTAB);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
new file mode 100644
index 0000000..7e51457
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+@Parameters(commandNames = { SliderActions.ACTION_KEYTAB},
+ commandDescription = SliderActions.DESCRIBE_ACTION_KEYTAB)
+
+public class ActionKeytabArgs extends AbstractActionArgs {
+
+ public ActionKeytabArgs() {
+ super();
+ }
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_INSTALL_KEYTAB;
+ }
+
+ @Parameter(names = {ARG_KEYTABINSTALL},
+ description = "Install the keytab")
+ public boolean install;
+
+ @Parameter(names = {ARG_KEYTABDELETE},
+ description = "Delete the keytab")
+ public boolean delete;
+
+ @Parameter(names = {ARG_KEYTABLIST},
+ description = "List of installed keytabs")
+ public boolean list;
+
+ @Parameter(names = {ARG_KEYTAB},
+ description = "Path or name of the keytab")
+ public String keytab;
+
+ @Parameter(names = {ARG_FOLDER},
+ description = "The name of the folder in which to store the keytab")
+ public String folder;
+
+ @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing keytab")
+ public boolean overwrite = false;
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 3;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
new file mode 100644
index 0000000..c05e602
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+@Parameters(commandNames = { SliderActions.ACTION_LIST},
+ commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
+
+public class ActionListArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_LIST;
+ }
+
+ @Parameter(names = {ARG_LIVE},
+ description = "List only live service instances")
+ public boolean live;
+
+ @Parameter(names = {ARG_STATE},
+ description = "list only applications in the specific YARN state")
+ public String state = "";
+
+ @Parameter(names = {ARG_VERBOSE},
+ description = "print out information in details")
+ public boolean verbose = false;
+
+ @Parameter(names = {ARG_CONTAINERS},
+ description = "List containers of a service instance")
+ public boolean containers;
+
+ @Parameter(names = {ARG_VERSION},
+ description = "Filter containers by app version (used with " +
+ ARG_CONTAINERS + ")")
+ public String version;
+
+ @Parameter(names = {ARG_COMPONENTS}, variableArity = true,
+ description = "Filter containers by component names (used with " +
+ ARG_CONTAINERS + ")")
+ public Set<String> components = new HashSet<>(0);
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 1;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
deleted file mode 100644
index 405f690..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.service.utils.ApplicationReportSerDeser;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.io.IOException;
-
-/**
- * Serialized form of an application report which can be persisted
- * and then parsed. It can not be converted back into a
- * real YARN application report
- *
- * Useful for testing
- */
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-
-public class SerializedApplicationReport {
-
- public String applicationId;
- public String applicationAttemptId;
- public String name;
- public String applicationType;
- public String user;
- public String queue;
- public String host;
- public Integer rpcPort;
- public String state;
- public String diagnostics;
- public String url;
- /**
- * This value is non-null only when a report is generated from a submission context.
- * The YARN {@link ApplicationReport} structure does not propagate this value
- * from the RM.
- */
- public Long submitTime;
- public Long startTime;
- public Long finishTime;
- public String finalStatus;
- public String origTrackingUrl;
- public Float progress;
-
- public SerializedApplicationReport() {
- }
-
- public SerializedApplicationReport(ApplicationReport report) {
- this.applicationId = report.getApplicationId().toString();
- ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId();
- this.applicationAttemptId = attemptId != null ? attemptId.toString() : "N/A";
- this.name = report.getName();
- this.applicationType = report.getApplicationType();
- this.user = report.getUser();
- this.queue = report.getQueue();
- this.host = report.getHost();
- this.rpcPort = report.getRpcPort();
- this.state = report.getYarnApplicationState().toString();
- this.diagnostics = report.getDiagnostics();
- this.startTime = report.getStartTime();
- this.finishTime = report.getFinishTime();
- FinalApplicationStatus appStatus = report.getFinalApplicationStatus();
- this.finalStatus = appStatus == null ? "" : appStatus.toString();
- this.progress = report.getProgress();
- this.url = report.getTrackingUrl();
- this.origTrackingUrl= report.getOriginalTrackingUrl();
- }
-
- @Override
- public String toString() {
- try {
- return ApplicationReportSerDeser.toString(this);
- } catch (IOException e) {
- return super.toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
deleted file mode 100644
index 21cb049..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ /dev/null
@@ -1,443 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Configuration;
-import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-import org.apache.hadoop.yarn.service.servicemonitor.probe.MonitorUtils;
-import org.apache.hadoop.yarn.service.conf.RestApiConstants;
-import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
-import org.codehaus.jackson.map.PropertyNamingStrategy;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class ServiceApiUtil {
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceApiUtil.class);
- public static JsonSerDeser<Application> jsonSerDeser =
- new JsonSerDeser<>(Application.class,
- PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
- private static final PatternValidator compNamePattern
- = new PatternValidator("[a-z][a-z0-9-]*");
-
- @VisibleForTesting
- public static void setJsonSerDeser(JsonSerDeser jsd) {
- jsonSerDeser = jsd;
- }
-
- @VisibleForTesting
- public static void validateAndResolveApplication(Application application,
- SliderFileSystem fs, org.apache.hadoop.conf.Configuration conf) throws
- IOException {
- boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED,
- RegistryConstants.DEFAULT_DNS_ENABLED);
- if (dnsEnabled && RegistryUtils.currentUser().length() > RegistryConstants
- .MAX_FQDN_LABEL_LENGTH) {
- throw new IllegalArgumentException(RestApiErrorMessages
- .ERROR_USER_NAME_INVALID);
- }
- if (StringUtils.isEmpty(application.getName())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID);
- }
- if (!SliderUtils.isClusternameValid(application.getName()) || (dnsEnabled
- && application.getName().length() > RegistryConstants
- .MAX_FQDN_LABEL_LENGTH)) {
- throw new IllegalArgumentException(String.format(
- RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT,
- application.getName()));
- }
-
- // If the application has no components do top-level checks
- if (!hasComponent(application)) {
- // If artifact is of type APPLICATION, read other application components
- if (application.getArtifact() != null && application.getArtifact()
- .getType() == Artifact.TypeEnum.APPLICATION) {
- if (StringUtils.isEmpty(application.getArtifact().getId())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
- }
- Application otherApplication = loadApplication(fs,
- application.getArtifact().getId());
- application.setComponents(otherApplication.getComponents());
- application.setArtifact(null);
- SliderUtils.mergeMapsIgnoreDuplicateKeys(application.getQuicklinks(),
- otherApplication.getQuicklinks());
- } else {
- // Since it is a simple app with no components, create a default
- // component
- Component comp = createDefaultComponent(application);
- validateComponent(comp, fs.getFileSystem());
- application.getComponents().add(comp);
- if (application.getLifetime() == null) {
- application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME);
- }
- return;
- }
- }
-
- // Validate there are no component name collisions (collisions are not
- // currently supported) and add any components from external applications
- // TODO allow name collisions? see AppState#roles
- // TODO or add prefix to external component names?
- Configuration globalConf = application.getConfiguration();
- Set<String> componentNames = new HashSet<>();
- List<Component> componentsToRemove = new ArrayList<>();
- List<Component> componentsToAdd = new ArrayList<>();
- for (Component comp : application.getComponents()) {
- int maxCompLength = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
- maxCompLength = maxCompLength - Long.toString(Long.MAX_VALUE).length();
- if (dnsEnabled && comp.getName().length() > maxCompLength) {
- throw new IllegalArgumentException(String.format(RestApiErrorMessages
- .ERROR_COMPONENT_NAME_INVALID, maxCompLength, comp.getName()));
- }
- if (componentNames.contains(comp.getName())) {
- throw new IllegalArgumentException("Component name collision: " +
- comp.getName());
- }
- // If artifact is of type APPLICATION (which cannot be filled from
- // global), read external application and add its components to this
- // application
- if (comp.getArtifact() != null && comp.getArtifact().getType() ==
- Artifact.TypeEnum.APPLICATION) {
- if (StringUtils.isEmpty(comp.getArtifact().getId())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
- }
- LOG.info("Marking {} for removal", comp.getName());
- componentsToRemove.add(comp);
- List<Component> externalComponents = getApplicationComponents(fs,
- comp.getArtifact().getId());
- for (Component c : externalComponents) {
- Component override = application.getComponent(c.getName());
- if (override != null && override.getArtifact() == null) {
- // allow properties from external components to be overridden /
- // augmented by properties in this component, except for artifact
- // which must be read from external component
- override.mergeFrom(c);
- LOG.info("Merging external component {} from external {}", c
- .getName(), comp.getName());
- } else {
- if (componentNames.contains(c.getName())) {
- throw new IllegalArgumentException("Component name collision: " +
- c.getName());
- }
- componentNames.add(c.getName());
- componentsToAdd.add(c);
- LOG.info("Adding component {} from external {}", c.getName(),
- comp.getName());
- }
- }
- } else {
- // otherwise handle as a normal component
- componentNames.add(comp.getName());
- // configuration
- comp.getConfiguration().mergeFrom(globalConf);
- }
- }
- application.getComponents().removeAll(componentsToRemove);
- application.getComponents().addAll(componentsToAdd);
-
- // Validate components and let global values take effect if component level
- // values are not provided
- Artifact globalArtifact = application.getArtifact();
- Resource globalResource = application.getResource();
- Long globalNumberOfContainers = application.getNumberOfContainers();
- String globalLaunchCommand = application.getLaunchCommand();
- for (Component comp : application.getComponents()) {
- // fill in global artifact unless it is type APPLICATION
- if (comp.getArtifact() == null && application.getArtifact() != null
- && application.getArtifact().getType() != Artifact.TypeEnum
- .APPLICATION) {
- comp.setArtifact(globalArtifact);
- }
- // fill in global resource
- if (comp.getResource() == null) {
- comp.setResource(globalResource);
- }
- // fill in global container count
- if (comp.getNumberOfContainers() == null) {
- comp.setNumberOfContainers(globalNumberOfContainers);
- }
- // fill in global launch command
- if (comp.getLaunchCommand() == null) {
- comp.setLaunchCommand(globalLaunchCommand);
- }
- // validate dependency existence
- if (comp.getDependencies() != null) {
- for (String dependency : comp.getDependencies()) {
- if (!componentNames.contains(dependency)) {
- throw new IllegalArgumentException(String.format(
- RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, dependency,
- comp.getName()));
- }
- }
- }
- validateComponent(comp, fs.getFileSystem());
- }
-
- // validate dependency tree
- sortByDependencies(application.getComponents());
-
- // Application lifetime if not specified, is set to unlimited lifetime
- if (application.getLifetime() == null) {
- application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME);
- }
- }
-
- public static void validateComponent(Component comp, FileSystem fs)
- throws IOException {
- validateCompName(comp.getName());
-
- AbstractClientProvider compClientProvider = ProviderFactory
- .getClientProvider(comp.getArtifact());
- compClientProvider.validateArtifact(comp.getArtifact(), fs);
-
- if (comp.getLaunchCommand() == null && (comp.getArtifact() == null || comp
- .getArtifact().getType() != Artifact.TypeEnum.DOCKER)) {
- throw new IllegalArgumentException(RestApiErrorMessages
- .ERROR_ABSENT_LAUNCH_COMMAND);
- }
-
- validateApplicationResource(comp.getResource(), comp);
-
- if (comp.getNumberOfContainers() == null
- || comp.getNumberOfContainers() < 0) {
- throw new IllegalArgumentException(String.format(
- RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID
- + ": " + comp.getNumberOfContainers(), comp.getName()));
- }
- compClientProvider.validateConfigFiles(comp.getConfiguration()
- .getFiles(), fs);
-
- MonitorUtils.getProbe(comp.getReadinessCheck());
- }
-
- // Check component name format and transform to lower case.
- public static void validateCompName(String compName) {
- if (StringUtils.isEmpty(compName)) {
- throw new IllegalArgumentException("Component name can not be empty");
- }
- // validate component name
- if (compName.contains("_")) {
- throw new IllegalArgumentException(
- "Invalid format for component name: " + compName
- + ", can not use '_' as DNS hostname does not allow underscore. Use '-' Instead. ");
- }
- compNamePattern.validate(compName);
- }
-
- @VisibleForTesting
- public static List<Component> getApplicationComponents(SliderFileSystem
- fs, String appName) throws IOException {
- return loadApplication(fs, appName).getComponents();
- }
-
- public static Application loadApplication(SliderFileSystem fs, String
- appName) throws IOException {
- Path appJson = getAppJsonPath(fs, appName);
- LOG.info("Loading application definition from " + appJson);
- return jsonSerDeser.load(fs.getFileSystem(), appJson);
- }
-
- public static Application loadApplicationFrom(SliderFileSystem fs,
- Path appDefPath) throws IOException {
- LOG.info("Loading application definition from " + appDefPath);
- return jsonSerDeser.load(fs.getFileSystem(), appDefPath);
- }
-
- public static Path getAppJsonPath(SliderFileSystem fs, String appName) {
- Path appDir = fs.buildClusterDirPath(appName);
- Path appJson = new Path(appDir, appName + ".json");
- return appJson;
- }
-
- private static void validateApplicationResource(Resource resource,
- Component comp) {
- // Only apps/components of type APPLICATION can skip resource requirement
- if (resource == null) {
- throw new IllegalArgumentException(
- comp == null ? RestApiErrorMessages.ERROR_RESOURCE_INVALID : String
- .format(RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID,
- comp.getName()));
- }
- // One and only one of profile OR cpus & memory can be specified. Specifying
- // both raises validation error.
- if (StringUtils.isNotEmpty(resource.getProfile()) && (
- resource.getCpus() != null || StringUtils
- .isNotEmpty(resource.getMemory()))) {
- throw new IllegalArgumentException(comp == null ?
- RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED :
- String.format(
- RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
- comp.getName()));
- }
- // Currently resource profile is not supported yet, so we will raise
- // validation error if only resource profile is specified
- if (StringUtils.isNotEmpty(resource.getProfile())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET);
- }
-
- String memory = resource.getMemory();
- Integer cpus = resource.getCpus();
- if (StringUtils.isEmpty(memory)) {
- throw new IllegalArgumentException(
- comp == null ? RestApiErrorMessages.ERROR_RESOURCE_MEMORY_INVALID :
- String.format(
- RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID,
- comp.getName()));
- }
- if (cpus == null) {
- throw new IllegalArgumentException(
- comp == null ? RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID :
- String.format(
- RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID,
- comp.getName()));
- }
- if (cpus <= 0) {
- throw new IllegalArgumentException(comp == null ?
- RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID_RANGE : String
- .format(
- RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE,
- comp.getName()));
- }
- }
-
- // check if comp mem size exceeds cluster limit
- public static void validateCompResourceSize(
- org.apache.hadoop.yarn.api.records.Resource maxResource,
- Application application) throws YarnException {
- for (Component component : application.getComponents()) {
- // only handle mem now.
- long mem = Long.parseLong(component.getResource().getMemory());
- if (mem > maxResource.getMemorySize()) {
- throw new YarnException(
- "Component " + component.getName() + " memory size (" + mem
- + ") is larger than configured max container memory size ("
- + maxResource.getMemorySize() + ")");
- }
- }
- }
-
- public static boolean hasComponent(Application application) {
- if (application.getComponents() == null || application.getComponents()
- .isEmpty()) {
- return false;
- }
- return true;
- }
-
- public static Component createDefaultComponent(Application app) {
- Component comp = new Component();
- comp.setName(RestApiConstants.DEFAULT_COMPONENT_NAME);
- comp.setArtifact(app.getArtifact());
- comp.setResource(app.getResource());
- comp.setNumberOfContainers(app.getNumberOfContainers());
- comp.setLaunchCommand(app.getLaunchCommand());
- comp.setConfiguration(app.getConfiguration());
- return comp;
- }
-
- public static Collection<Component> sortByDependencies(List<Component>
- components) {
- Map<String, Component> sortedComponents =
- sortByDependencies(components, null);
- return sortedComponents.values();
- }
-
- /**
- * Each internal call of sortByDependencies will identify all of the
- * components with the same dependency depth (the lowest depth that has not
- * been processed yet) and add them to the sortedComponents list, preserving
- * their original ordering in the components list.
- *
- * So the first time it is called, all components with no dependencies
- * (depth 0) will be identified. The next time it is called, all components
- * that have dependencies only on the the depth 0 components will be
- * identified (depth 1). This will be repeated until all components have
- * been added to the sortedComponents list. If no new components are
- * identified but the sortedComponents list is not complete, an error is
- * thrown.
- */
- private static Map<String, Component> sortByDependencies(List<Component>
- components, Map<String, Component> sortedComponents) {
- if (sortedComponents == null) {
- sortedComponents = new LinkedHashMap<>();
- }
-
- Map<String, Component> componentsToAdd = new LinkedHashMap<>();
- List<Component> componentsSkipped = new ArrayList<>();
- for (Component component : components) {
- String name = component.getName();
- if (sortedComponents.containsKey(name)) {
- continue;
- }
- boolean dependenciesAlreadySorted = true;
- if (!SliderUtils.isEmpty(component.getDependencies())) {
- for (String dependency : component.getDependencies()) {
- if (!sortedComponents.containsKey(dependency)) {
- dependenciesAlreadySorted = false;
- break;
- }
- }
- }
- if (dependenciesAlreadySorted) {
- componentsToAdd.put(name, component);
- } else {
- componentsSkipped.add(component);
- }
- }
-
- if (componentsToAdd.size() == 0) {
- throw new IllegalArgumentException(String.format(RestApiErrorMessages
- .ERROR_DEPENDENCY_CYCLE, componentsSkipped));
- }
- sortedComponents.putAll(componentsToAdd);
- if (sortedComponents.size() == components.size()) {
- return sortedComponents;
- }
- return sortByDependencies(components, sortedComponents);
- }
-
- public static String $(String s) {
- return "${" + s +"}";
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java
deleted file mode 100644
index 7440b11..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-
-
-public class ServiceRegistryUtils {
-
- /**
- * Base path for services
- */
- public static final String ZK_SERVICES = "services";
-
- /**
- * Base path for all Slider references
- */
- public static final String ZK_SLIDER = "slider";
- public static final String ZK_USERS = "users";
- public static final String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
- public static final String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
-
- /**
- * Get the registry path for an instance under the user's home node
- * @param instanceName application instance
- * @return a path to the registry location for this application instance.
- */
- public static String registryPathForInstance(String instanceName) {
- return RegistryUtils.servicePath(
- RegistryUtils.currentUser(), YarnServiceConstants.APP_TYPE, instanceName
- );
- }
-
- /**
- * Build the path to a cluster; exists once the cluster has come up.
- * Even before that, a ZK watcher could wait for it.
- * @param username user
- * @param clustername name of the cluster
- * @return a strin
- */
- public static String mkClusterPath(String username, String clustername) {
- return mkSliderUserPath(username) + "/" + clustername;
- }
-
- /**
- * Build the path to a cluster; exists once the cluster has come up.
- * Even before that, a ZK watcher could wait for it.
- * @param username user
- * @return a string
- */
- public static String mkSliderUserPath(String username) {
- return SVC_SLIDER_USERS + "/" + username;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
deleted file mode 100644
index d6d664e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import java.io.IOException;
-
-/**
- * Extends Core Filesystem with operations to manipulate ClusterDescription
- * persistent state
- */
-public class SliderFileSystem extends CoreFileSystem {
-
- Path appDir = null;
-
- public SliderFileSystem(FileSystem fileSystem,
- Configuration configuration) {
- super(fileSystem, configuration);
- }
-
- public SliderFileSystem(Configuration configuration) throws IOException {
- super(configuration);
- }
-
- public void setAppDir(Path appDir) {
- this.appDir = appDir;
- }
-
- public Path getAppDir() {
- return this.appDir;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
deleted file mode 100644
index 415392a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
+++ /dev/null
@@ -1,1699 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.GlobFilter;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.zookeeper.server.util.KerberosUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.Serializable;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.regex.Pattern;
-import java.util.zip.GZIPOutputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
-
-/**
- * These are slider-specific Util methods
- */
-public final class SliderUtils {
-
- private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
-
- /**
- * Atomic bool to track whether or not process security has already been
- * turned on (prevents re-entrancy)
- */
- private static final AtomicBoolean processSecurityAlreadyInitialized =
- new AtomicBoolean(false);
- public static final String JAVA_SECURITY_KRB5_REALM =
- "java.security.krb5.realm";
- public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
-
- /**
- * Winutils
- */
- public static final String WINUTILS = "WINUTILS.EXE";
- /**
- * name of openssl program
- */
- public static final String OPENSSL = "openssl";
-
- /**
- * name of python program
- */
- public static final String PYTHON = "python";
-
- /**
- * type of docker standalone application
- */
- public static final String DOCKER = "docker";
- /**
- * type of docker on yarn application
- */
- public static final String DOCKER_YARN = "yarn_docker";
-
- public static final int NODE_LIST_LIMIT = 10;
-
- private SliderUtils() {
- }
-
- /**
- * Implementation of set-ness, groovy definition of true/false for a string
- * @param s string
- * @return true iff the string is neither null nor empty
- */
- public static boolean isUnset(String s) {
- return s == null || s.isEmpty();
- }
-
- public static boolean isSet(String s) {
- return !isUnset(s);
- }
-
- public static boolean isEmpty(Collection l) {
- return l == null || l.isEmpty();
- }
-
- /**
- * Probe for a collection existing and not being empty
- * @param l collection
- * @return true if the reference is valid and it contains entries
- */
-
- public static boolean isNotEmpty(Collection l) {
- return l != null && !l.isEmpty();
- }
-
- /**
- * Probe for a map existing and not being empty
- * @param m map
- * @return true if the reference is valid and it contains map entries
- */
- public static boolean isNotEmpty(Map m) {
- return m != null && !m.isEmpty();
- }
-
- /*
- * Validates whether num is an integer
- * @param num
- * @param msg the message to be shown in exception
- */
- @SuppressWarnings("ResultOfMethodCallIgnored")
- private static void validateNumber(String num, String msg) throws
- BadConfigException {
- try {
- Integer.parseInt(num);
- } catch (NumberFormatException nfe) {
- throw new BadConfigException(msg + num);
- }
- }
-
- /*
- * Translates the trailing JVM heapsize unit: g, G, m, M
- * This assumes designated unit of 'm'
- * @param heapsize
- * @return heapsize in MB
- */
- public static String translateTrailingHeapUnit(String heapsize) throws
- BadConfigException {
- String errMsg = "Bad heapsize: ";
- if (heapsize.endsWith("m") || heapsize.endsWith("M")) {
- String num = heapsize.substring(0, heapsize.length() - 1);
- validateNumber(num, errMsg);
- return num;
- }
- if (heapsize.endsWith("g") || heapsize.endsWith("G")) {
- String num = heapsize.substring(0, heapsize.length() - 1) + "000";
- validateNumber(num, errMsg);
- return num;
- }
- // check if specified heap size is a number
- validateNumber(heapsize, errMsg);
- return heapsize;
- }
-
- /**
- * recursive directory delete
- * @param dir dir to delete
- * @throws IOException on any problem
- */
- public static void deleteDirectoryTree(File dir) throws IOException {
- if (dir.exists()) {
- if (dir.isDirectory()) {
- log.info("Cleaning up {}", dir);
- //delete the children
- File[] files = dir.listFiles();
- if (files == null) {
- throw new IOException("listfiles() failed for " + dir);
- }
- for (File file : files) {
- log.info("deleting {}", file);
- if (!file.delete()) {
- log.warn("Unable to delete " + file);
- }
- }
- if (!dir.delete()) {
- log.warn("Unable to delete " + dir);
- }
- } else {
- throw new IOException("Not a directory " + dir);
- }
- } else {
- //not found, do nothing
- log.debug("No output dir yet");
- }
- }
-
- /**
- * Find a containing JAR
- * @param clazz class to find
- * @return the file
- * @throws IOException any IO problem, including the class not having a
- * classloader
- * @throws FileNotFoundException if the class did not resolve to a file
- */
- public static File findContainingJarOrFail(Class clazz) throws IOException {
- File localFile = SliderUtils.findContainingJar(clazz);
- if (null == localFile) {
- throw new FileNotFoundException("Could not find JAR containing " + clazz);
- }
- return localFile;
- }
-
-
- /**
- * Find a containing JAR
- * @param my_class class to find
- * @return the file or null if it is not found
- * @throws IOException any IO problem, including the class not having a
- * classloader
- */
- public static File findContainingJar(Class my_class) throws IOException {
- ClassLoader loader = my_class.getClassLoader();
- if (loader == null) {
- throw new IOException(
- "Class " + my_class + " does not have a classloader!");
- }
- String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
- Enumeration<URL> urlEnumeration = loader.getResources(class_file);
- for (; urlEnumeration.hasMoreElements(); ) {
- URL url = urlEnumeration.nextElement();
- if ("jar".equals(url.getProtocol())) {
- String toReturn = url.getPath();
- if (toReturn.startsWith("file:")) {
- toReturn = toReturn.substring("file:".length());
- }
- // URLDecoder is a misnamed class, since it actually decodes
- // x-www-form-urlencoded MIME type rather than actual
- // URL encoding (which the file path has). Therefore it would
- // decode +s to ' 's which is incorrect (spaces are actually
- // either unencoded or encoded as "%20"). Replace +s first, so
- // that they are kept sacred during the decoding process.
- toReturn = toReturn.replaceAll("\\+", "%2B");
- toReturn = URLDecoder.decode(toReturn, "UTF-8");
- String jarFilePath = toReturn.replaceAll("!.*$", "");
- return new File(jarFilePath);
- } else {
- log.info("could not locate JAR containing {} URL={}", my_class, url);
- }
- }
- return null;
- }
-
- public static void checkPort(String hostname, int port, int connectTimeout)
- throws IOException {
- InetSocketAddress addr = new InetSocketAddress(hostname, port);
- checkPort(hostname, addr, connectTimeout);
- }
-
- @SuppressWarnings("SocketOpenedButNotSafelyClosed")
- public static void checkPort(String name,
- InetSocketAddress address,
- int connectTimeout)
- throws IOException {
- try(Socket socket = new Socket()) {
- socket.connect(address, connectTimeout);
- } catch (Exception e) {
- throw new IOException("Failed to connect to " + name
- + " at " + address
- + " after " + connectTimeout + "milliseconds"
- + ": " + e,
- e);
- }
- }
-
- public static void checkURL(String name, String url, int timeout) throws
- IOException {
- InetSocketAddress address = NetUtils.createSocketAddr(url);
- checkPort(name, address, timeout);
- }
-
- /**
- * A required file
- * @param role role of the file (for errors)
- * @param filename the filename
- * @throws ExitUtil.ExitException if the file is missing
- * @return the file
- */
- public static File requiredFile(String filename, String role) throws
- IOException {
- if (filename.isEmpty()) {
- throw new ExitUtil.ExitException(-1, role + " file not defined");
- }
- File file = new File(filename);
- if (!file.exists()) {
- throw new ExitUtil.ExitException(-1,
- role + " file not found: " +
- file.getCanonicalPath());
- }
- return file;
- }
-
- private static final PatternValidator clusternamePattern
- = new PatternValidator("[a-z][a-z0-9_-]*");
-
- private static final PatternValidator compNamePattern
- = new PatternValidator("[a-z][a-z0-9-]*");
-
- public static void validateCompName(String compName) {
- compNamePattern.validate(compName);
- }
-
- /**
- * Normalize a cluster name then verify that it is valid
- * @param name proposed cluster name
- * @return true iff it is valid
- */
- public static boolean isClusternameValid(String name) {
- return name != null && clusternamePattern.matches(name);
- }
-
- public static boolean oldIsClusternameValid(String name) {
- if (name == null || name.isEmpty()) {
- return false;
- }
- int first = name.charAt(0);
- if (0 == (Character.getType(first) & Character.LOWERCASE_LETTER)) {
- return false;
- }
-
- for (int i = 0; i < name.length(); i++) {
- int elt = (int) name.charAt(i);
- int t = Character.getType(elt);
- if (0 == (t & Character.LOWERCASE_LETTER)
- && 0 == (t & Character.DECIMAL_DIGIT_NUMBER)
- && elt != '-'
- && elt != '_') {
- return false;
- }
- if (!Character.isLetterOrDigit(elt) && elt != '-' && elt != '_') {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Copy a directory to a new FS -both paths must be qualified. If
- * a directory needs to be created, supplied permissions can override
- * the default values. Existing directories are not touched
- * @param conf conf file
- * @param srcDirPath src dir
- * @param destDirPath dest dir
- * @param permission permission for the dest directory; null means "default"
- * @return # of files copies
- */
- @SuppressWarnings("deprecation")
- public static int copyDirectory(Configuration conf,
- Path srcDirPath,
- Path destDirPath,
- FsPermission permission) throws
- IOException,
- BadClusterStateException {
- FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf);
- FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf);
- //list all paths in the src.
- if (!srcFS.exists(srcDirPath)) {
- throw new FileNotFoundException("Source dir not found " + srcDirPath);
- }
- if (!srcFS.isDirectory(srcDirPath)) {
- throw new FileNotFoundException(
- "Source dir not a directory " + srcDirPath);
- }
- GlobFilter dotFilter = new GlobFilter("[!.]*");
- FileStatus[] entries = srcFS.listStatus(srcDirPath, dotFilter);
- int srcFileCount = entries.length;
- if (srcFileCount == 0) {
- return 0;
- }
- if (permission == null) {
- permission = FsPermission.getDirDefault();
- }
- if (!destFS.exists(destDirPath)) {
- new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath,
- permission);
- }
- Path[] sourcePaths = new Path[srcFileCount];
- for (int i = 0; i < srcFileCount; i++) {
- FileStatus e = entries[i];
- Path srcFile = e.getPath();
- if (srcFS.isDirectory(srcFile)) {
- String msg = "Configuration dir " + srcDirPath
- + " contains a directory " + srcFile;
- log.warn(msg);
- throw new IOException(msg);
- }
- log.debug("copying src conf file {}", srcFile);
- sourcePaths[i] = srcFile;
- }
- log.debug("Copying {} files from {} to dest {}", srcFileCount,
- srcDirPath,
- destDirPath);
- FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf);
- return srcFileCount;
- }
-
- /**
- * Copy a file to a new FS -both paths must be qualified.
- * @param conf conf file
- * @param srcFile src file
- * @param destFile dest file
- */
- @SuppressWarnings("deprecation")
- public static void copy(Configuration conf,
- Path srcFile,
- Path destFile) throws
- IOException,
- BadClusterStateException {
- FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
- //list all paths in the src.
- if (!srcFS.exists(srcFile)) {
- throw new FileNotFoundException("Source file not found " + srcFile);
- }
- if (!srcFS.isFile(srcFile)) {
- throw new FileNotFoundException(
- "Source file not a file " + srcFile);
- }
- FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
- FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
- }
-
- /**
- * Take a collection, return a list containing the string value of every
- * element in the collection.
- * @param c collection
- * @return a stringified list
- */
- public static List<String> collectionToStringList(Collection c) {
- List<String> l = new ArrayList<>(c.size());
- for (Object o : c) {
- l.add(o.toString());
- }
- return l;
- }
-
- /**
- * Join an collection of objects with a separator that appears after every
- * instance in the list -including at the end
- * @param collection collection to call toString() on each element
- * @param separator separator string
- * @return the joined entries
- */
- public static String join(Collection collection, String separator) {
- return join(collection, separator, true);
- }
-
- /**
- * Join an collection of objects with a separator that appears after every
- * instance in the list -optionally at the end
- * @param collection collection to call toString() on each element
- * @param separator separator string
- * @param trailing add a trailing entry or not
- * @return the joined entries
- */
- public static String join(Collection collection,
- String separator,
- boolean trailing) {
- StringBuilder b = new StringBuilder();
- // fast return on empty collection
- if (collection.isEmpty()) {
- return trailing ? separator : "";
- }
- for (Object o : collection) {
- b.append(o);
- b.append(separator);
- }
- int length = separator.length();
- String s = b.toString();
- return (trailing || s.isEmpty()) ?
- s : (b.substring(0, b.length() - length));
- }
-
- /**
- * Join an array of strings with a separator that appears after every
- * instance in the list -including at the end
- * @param collection strings
- * @param separator separator string
- * @return the joined entries
- */
- public static String join(String[] collection, String separator) {
- return join(collection, separator, true);
-
-
- }
-
- /**
- * Join an array of strings with a separator that appears after every
- * instance in the list -optionally at the end
- * @param collection strings
- * @param separator separator string
- * @param trailing add a trailing entry or not
- * @return the joined entries
- */
- public static String join(String[] collection, String separator,
- boolean trailing) {
- return join(Arrays.asList(collection), separator, trailing);
- }
-
- /**
- * Join an array of strings with a separator that appears after every
- * instance in the list -except at the end
- * @param collection strings
- * @param separator separator string
- * @return the list
- */
- public static String joinWithInnerSeparator(String separator,
- Object... collection) {
- StringBuilder b = new StringBuilder();
- boolean first = true;
-
- for (Object o : collection) {
- if (first) {
- first = false;
- } else {
- b.append(separator);
- }
- b.append(o.toString());
- b.append(separator);
- }
- return b.toString();
- }
-
- /**
- * Resolve a mandatory environment variable
- * @param key env var
- * @return the resolved value
- * @throws BadClusterStateException
- */
- public static String mandatoryEnvVariable(String key) throws
- BadClusterStateException {
- String v = System.getenv(key);
- if (v == null) {
- throw new BadClusterStateException("Missing Environment variable " + key);
- }
- return v;
- }
-
- public static String appReportToString(ApplicationReport r,
- String separator) {
- StringBuilder builder = new StringBuilder(512);
- builder.append("application ")
- .append(
- r.getName())
- .append("/")
- .append(r.getApplicationType())
- .append(separator);
- Set<String> tags = r.getApplicationTags();
- if (!tags.isEmpty()) {
- for (String tag : tags) {
- builder.append(tag).append(separator);
- }
- }
- DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
- dateFormat.setTimeZone(TimeZone.getDefault());
- builder.append("state: ").append(r.getYarnApplicationState());
- String trackingUrl = r.getTrackingUrl();
- if (isSet(trackingUrl)) {
- builder.append(separator).append("URL: ").append(trackingUrl);
- }
- builder.append(separator)
- .append("Started: ")
- .append(dateFormat.format(new Date(r.getStartTime())));
- long finishTime = r.getFinishTime();
- if (finishTime > 0) {
- builder.append(separator)
- .append("Finished: ")
- .append(dateFormat.format(new Date(finishTime)));
- }
- String rpcHost = r.getHost();
- if (!isSet(rpcHost)) {
- builder.append(separator)
- .append("RPC :")
- .append(rpcHost)
- .append(':')
- .append(r.getRpcPort());
- }
- String diagnostics = r.getDiagnostics();
- if (!isSet(diagnostics)) {
- builder.append(separator).append("Diagnostics :").append(diagnostics);
- }
- return builder.toString();
- }
-
- /**
- * Filter a string value given a single filter
- *
- * @param value
- * the string value to check
- * @param filter
- * a single string filter
- * @return return true if value should be trapped, false if it should be let
- * through
- */
- public static boolean filter(String value, String filter) {
- return !(StringUtils.isEmpty(filter) || filter.equals(value));
- }
-
- /**
- * Filter a string value given a set of filters
- *
- * @param value
- * the string value to check
- * @param filters
- * a set of string filters
- * @return return true if value should be trapped, false if it should be let
- * through
- */
- public static boolean filter(String value, Set<String> filters) {
- return !(filters.isEmpty() || filters.contains(value));
- }
-
- /**
- * Sorts the given list of application reports, most recently started
- * or finished instance first.
- *
- * @param instances list of instances
- */
- public static void sortApplicationsByMostRecent(List<ApplicationReport> instances) {
- Collections.sort(instances, new MostRecentlyStartedOrFinishedFirst());
- }
-
- /**
- * Sorts the given list of application reports
- * Finished instances are ordered by finished time and running/accepted instances are
- * ordered by start time
- * Finally Instance are order by finished instances coming after running instances
- *
- * @param instances list of instances
- */
- public static void sortApplicationReport(List<ApplicationReport> instances) {
- if (instances.size() <= 1) {
- return;
- }
- List<ApplicationReport> nonLiveInstance =
- new ArrayList<>(instances.size());
- List<ApplicationReport> liveInstance =
- new ArrayList<>(instances.size());
-
- for (ApplicationReport report : instances) {
- if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
- ||
- report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
- liveInstance.add(report);
- } else {
- nonLiveInstance.add(report);
- }
- }
-
- if (liveInstance.size() > 1) {
- Collections.sort(liveInstance, new MostRecentlyStartedAppFirst());
- }
- if (nonLiveInstance.size() > 1) {
- Collections.sort(nonLiveInstance, new MostRecentAppFinishFirst());
- }
- instances.clear();
- instances.addAll(liveInstance);
- instances.addAll(nonLiveInstance);
- }
-
- /**
- * Merge in one map to another -all entries in the second map are
- * merged into the first -overwriting any duplicate keys.
- * @param first first map -the updated one.
- * @param second the map that is merged in
- * @return the first map
- */
- public static Map<String, String> mergeMap(Map<String, String> first,
- Map<String, String> second) {
- first.putAll(second);
- return first;
- }
-
- /**
- * Merge a set of entries into a map. This will take the entryset of
- * a map, or a Hadoop collection itself
- * @param dest destination
- * @param entries entries
- * @return dest -with the entries merged in
- */
- public static Map<String, String> mergeEntries(Map<String, String> dest,
- Iterable<Map.Entry<String, String>> entries) {
- for (Map.Entry<String, String> entry : entries) {
- dest.put(entry.getKey(), entry.getValue());
- }
- return dest;
- }
-
- /**
- * Generic map merge logic
- * @param first first map
- * @param second second map
- * @param <T1> key type
- * @param <T2> value type
- * @return 'first' merged with the second
- */
- public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
- Map<T1, T2> second) {
- first.putAll(second);
- return first;
- }
-
- /**
- * Generic map merge logic
- * @param first first map
- * @param second second map
- * @param <T1> key type
- * @param <T2> value type
- * @return 'first' merged with the second
- */
- public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
- Map<T1, T2> second) {
- Preconditions.checkArgument(first != null, "Null 'first' value");
- Preconditions.checkArgument(second != null, "Null 'second' value");
- for (Map.Entry<T1, T2> entry : second.entrySet()) {
- T1 key = entry.getKey();
- if (!first.containsKey(key)) {
- first.put(key, entry.getValue());
- }
- }
- return first;
- }
-
- /**
- * Convert a map to a multi-line string for printing
- * @param map map to stringify
- * @return a string representation of the map
- */
- public static String stringifyMap(Map<String, String> map) {
- StringBuilder builder = new StringBuilder();
- for (Map.Entry<String, String> entry : map.entrySet()) {
- builder.append(entry.getKey())
- .append("=\"")
- .append(entry.getValue())
- .append("\"\n");
-
- }
- return builder.toString();
- }
-
- /**
- * Parse an int value, replacing it with defval if undefined;
- * @param errorKey key to use in exceptions
- * @param defVal default value to use if the key is not in the map
- * @param min min value or -1 for do not check
- * @param max max value or -1 for do not check
- * @return the int value the integer value
- * @throws BadConfigException if the value could not be parsed
- */
- public static int parseAndValidate(String errorKey,
- String valS,
- int defVal,
- int min, int max) throws
- BadConfigException {
- if (valS == null) {
- valS = Integer.toString(defVal);
- }
- String trim = valS.trim();
- int val;
- try {
- val = Integer.decode(trim);
- } catch (NumberFormatException e) {
- throw new BadConfigException("Failed to parse value of "
- + errorKey + ": \"" + trim + "\"");
- }
- if (min >= 0 && val < min) {
- throw new BadConfigException("Value of "
- + errorKey + ": " + val + ""
- + "is less than the minimum of " + min);
- }
- if (max >= 0 && val > max) {
- throw new BadConfigException("Value of "
- + errorKey + ": " + val + ""
- + "is more than the maximum of " + max);
- }
- return val;
- }
-
- public static InetSocketAddress getRmAddress(Configuration conf) {
- return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_PORT);
- }
-
- public static InetSocketAddress getRmSchedulerAddress(Configuration conf) {
- return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
- }
-
- /**
- * probe to see if the RM scheduler is defined
- * @param conf config
- * @return true if the RM scheduler address is set to
- * something other than 0.0.0.0
- */
- public static boolean isRmSchedulerAddressDefined(Configuration conf) {
- InetSocketAddress address = getRmSchedulerAddress(conf);
- return isAddressDefined(address);
- }
-
- /**
- * probe to see if the address
- * @param address network address
- * @return true if the scheduler address is set to
- * something other than 0.0.0.0
- */
- public static boolean isAddressDefined(InetSocketAddress address) {
- if (address == null || address.getHostString() == null) {
- return false;
- }
- return !(address.getHostString().equals("0.0.0.0"));
- }
-
- public static void setRmAddress(Configuration conf, String rmAddr) {
- conf.set(YarnConfiguration.RM_ADDRESS, rmAddr);
- }
-
- public static void setRmSchedulerAddress(Configuration conf, String rmAddr) {
- conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);
- }
-
- public static boolean hasAppFinished(ApplicationReport report) {
- return report == null ||
- report.getYarnApplicationState().ordinal() >=
- YarnApplicationState.FINISHED.ordinal();
- }
-
- public static String containerToString(Container container) {
- if (container == null) {
- return "null container";
- }
- return String.format(Locale.ENGLISH,
- "ContainerID=%s nodeID=%s http=%s priority=%s resource=%s",
- container.getId(),
- container.getNodeId(),
- container.getNodeHttpAddress(),
- container.getPriority(),
- container.getResource());
- }
-
- /**
- * convert an AM report to a string for diagnostics
- * @param report the report
- * @return the string value
- */
- public static String reportToString(ApplicationReport report) {
- if (report == null) {
- return "Null application report";
- }
-
- return "App " + report.getName() + "/" + report.getApplicationType() +
- "# " +
- report.getApplicationId() + " user " + report.getUser() +
- " is in state " + report.getYarnApplicationState() +
- " RPC: " + report.getHost() + ":" + report.getRpcPort() +
- " URL: " + report.getOriginalTrackingUrl();
- }
-
- /**
- * Convert a YARN URL into a string value of a normal URL
- * @param url URL
- * @return string representatin
- */
- public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
- StringBuilder builder = new StringBuilder();
- builder.append(url.getScheme()).append("://");
- if (url.getHost() != null) {
- builder.append(url.getHost()).append(":").append(url.getPort());
- }
- builder.append(url.getFile());
- return builder.toString();
- }
-
- /**
- * Get a random open port
- * @return true if the port was available for listening on
- */
- public static int getOpenPort() throws IOException {
- ServerSocket socket = null;
- try {
- socket = new ServerSocket(0);
- return socket.getLocalPort();
- } finally {
- if (socket != null) {
- socket.close();
- }
- }
- }
-
- /**
- * See if a port is available for listening on by trying to listen
- * on it and seeing if that works or fails.
- * @param port port to listen to
- * @return true if the port was available for listening on
- */
- public static boolean isPortAvailable(int port) {
- try {
- ServerSocket socket = new ServerSocket(port);
- socket.close();
- return true;
- } catch (IOException e) {
- return false;
- }
- }
-
- // Build env map: key -> value;
- // value will be replaced by the corresponding value in tokenMap, if any.
- public static Map<String, String> buildEnvMap(
- org.apache.hadoop.yarn.service.api.records.Configuration conf,
- Map<String,String> tokenMap) {
- if (tokenMap == null) {
- return conf.getEnv();
- }
- Map<String, String> env = new HashMap<>();
- for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) {
- String key = entry.getKey();
- String val = entry.getValue();
- for (Map.Entry<String,String> token : tokenMap.entrySet()) {
- val = val.replaceAll(Pattern.quote(token.getKey()),
- token.getValue());
- }
- env.put(key,val);
- }
- return env;
- }
-
- /**
- * Apply a set of command line options to a cluster role map
- * @param clusterRoleMap cluster role map to merge onto
- * @param commandOptions command opts
- */
- public static void applyCommandLineRoleOptsToRoleMap(
- Map<String, Map<String, String>> clusterRoleMap,
- Map<String, Map<String, String>> commandOptions) {
- for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
- String key = entry.getKey();
- Map<String, String> optionMap = entry.getValue();
- Map<String, String> existingMap = clusterRoleMap.get(key);
- if (existingMap == null) {
- existingMap = new HashMap<String, String>();
- }
- log.debug("Overwriting role options with command line values {}",
- stringifyMap(optionMap));
- mergeMap(existingMap, optionMap);
- //set or overwrite the role
- clusterRoleMap.put(key, existingMap);
- }
- }
-
- /**
- * verify that the supplied cluster name is valid
- * @param clustername cluster name
- * @throws BadCommandArgumentsException if it is invalid
- */
- public static void validateClusterName(String clustername)
- throws BadCommandArgumentsException {
- if (!isClusternameValid(clustername)) {
- throw new BadCommandArgumentsException(
- "Illegal cluster name: " + clustername);
- }
- }
-
- /**
- * Verify that a Kerberos principal has been set -if not fail
- * with an error message that actually tells you what is missing
- * @param conf configuration to look at
- * @param principal key of principal
- * @throws BadConfigException if the key is not set
- */
- public static void verifyPrincipalSet(Configuration conf,
- String principal) throws
- BadConfigException {
- String principalName = conf.get(principal);
- if (principalName == null) {
- throw new BadConfigException("Unset Kerberos principal : %s",
- principal);
- }
- log.debug("Kerberos princial {}={}", principal, principalName);
- }
-
- /**
- * Flag to indicate whether the cluster is in secure mode
- * @param conf configuration to look at
- * @return true if the slider client/service should be in secure mode
- */
- public static boolean isHadoopClusterSecure(Configuration conf) {
- return SecurityUtil.getAuthenticationMethod(conf) !=
- UserGroupInformation.AuthenticationMethod.SIMPLE;
- }
-
- /**
- * Init security if the cluster configuration declares the cluster is secure
- * @param conf configuration to look at
- * @return true if the cluster is secure
- * @throws IOException cluster is secure
- * @throws SliderException the configuration/process is invalid
- */
- public static boolean maybeInitSecurity(Configuration conf) throws
- IOException,
- SliderException {
- boolean clusterSecure = isHadoopClusterSecure(conf);
- if (clusterSecure) {
- log.debug("Enabling security");
- initProcessSecurity(conf);
- }
- return clusterSecure;
- }
-
- /**
- * Turn on security. This is setup to only run once.
- * @param conf configuration to build up security
- * @return true if security was initialized in this call
- * @throws IOException IO/Net problems
- * @throws BadConfigException the configuration and system state are inconsistent
- */
- public static boolean initProcessSecurity(Configuration conf) throws
- IOException,
- SliderException {
-
- if (processSecurityAlreadyInitialized.compareAndSet(true, true)) {
- //security is already inited
- return false;
- }
-
- log.info("JVM initialized into secure mode with kerberos realm {}",
- SliderUtils.getKerberosRealm());
- //this gets UGI to reset its previous world view (i.e simple auth)
- //security
- log.debug("java.security.krb5.realm={}",
- System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
- log.debug("java.security.krb5.kdc={}",
- System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
- log.debug("hadoop.security.authentication={}",
- conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION));
- log.debug("hadoop.security.authorization={}",
- conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION));
- UserGroupInformation.setConfiguration(conf);
- UserGroupInformation authUser = UserGroupInformation.getCurrentUser();
- log.debug("Authenticating as {}", authUser);
- log.debug("Login user is {}", UserGroupInformation.getLoginUser());
- if (!UserGroupInformation.isSecurityEnabled()) {
- throw new SliderException(LauncherExitCodes.EXIT_UNAUTHORIZED,
- "Although secure mode is enabled," +
- "the application has already set up its user as an insecure entity %s",
- authUser);
- }
- if (authUser.getAuthenticationMethod() ==
- UserGroupInformation.AuthenticationMethod.SIMPLE) {
- throw new BadConfigException("Auth User is not Kerberized %s" +
- " -security has already been set up with the wrong authentication method. "
- + "This can occur if a file system has already been created prior to the loading of "
- + "the security configuration.",
- authUser);
-
- }
-
- SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
- SliderUtils.verifyPrincipalSet(conf, "dfs.namenode.kerberos.principal");
- return true;
- }
-
- /**
- * Force an early login: This catches any auth problems early rather than
- * in RPC operations
- * @throws IOException if the login fails
- */
- public static void forceLogin() throws IOException {
- if (UserGroupInformation.isSecurityEnabled()) {
- if (UserGroupInformation.isLoginKeytabBased()) {
- UserGroupInformation.getLoginUser().reloginFromKeytab();
- } else {
- UserGroupInformation.getLoginUser().reloginFromTicketCache();
- }
- }
- }
-
- public static String getLibDir() {
- String[] libDirs = getLibDirs();
- if (libDirs == null || libDirs.length == 0) {
- return null;
- }
- return libDirs[0];
- }
-
- public static String[] getLibDirs() {
- String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
- if (isUnset(libDirStr)) {
- return ArrayUtils.EMPTY_STRING_ARRAY;
- }
- return StringUtils.split(libDirStr, ',');
- }
-
- /**
- * Submit a JAR containing a specific class and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- * @param clazz class to look for
- * @param libdir lib directory
- * @param jarName <i>At the destination</i>
- * @return the local resource ref
- * @throws IOException trouble copying to HDFS
- */
- public static LocalResource putJar(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Class clazz,
- Path tempPath,
- String libdir,
- String jarName
- )
- throws IOException, SliderException {
- LocalResource res = sliderFileSystem.submitJarWithClass(
- clazz,
- tempPath,
- libdir,
- jarName);
- providerResources.put(libdir + "/" + jarName, res);
- return res;
- }
-
- /**
- * Submit a JAR containing and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- * @param libDir lib directory
- * @param srcPath copy jars from
- */
- public static void putAllJars(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Path tempPath,
- String libDir,
- String srcPath) throws IOException, SliderException {
- log.info("Loading all dependencies from {}", srcPath);
- if (SliderUtils.isSet(srcPath)) {
- File srcFolder = new File(srcPath);
- FilenameFilter jarFilter = createJarFilter();
- File[] listOfJars = srcFolder.listFiles(jarFilter);
- if (listOfJars == null || listOfJars.length == 0) {
- return;
- }
- for (File jarFile : listOfJars) {
- LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
- providerResources.put(libDir + "/" + jarFile.getName(), res);
- }
- }
- }
-
- /**
- * Accept all filenames ending with {@code .jar}
- * @return a filename filter
- */
- public static FilenameFilter createJarFilter() {
- return new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
- }
- };
- }
-
- /**
- * Submit the AM tar.gz containing all dependencies and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- */
- public static void putAmTarGzipAndUpdate(
- Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem
- ) throws IOException, SliderException {
- log.info("Loading all dependencies from {}{}",
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME,
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
- sliderFileSystem.submitTarGzipAndUpdate(providerResources);
- }
-
- public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
- Map<String, Map<String, String>> dest = new HashMap<>();
- for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) {
- dest.put(entry.getKey(), stringMapClone(entry.getValue()));
- }
- return dest;
- }
-
- public static Map<String, String> stringMapClone(Map<String, String> src) {
- Map<String, String> dest = new HashMap<>();
- return mergeEntries(dest, src.entrySet());
- }
-
- /**
- * List a directory in the local filesystem
- * @param dir directory
- * @return a listing, one to a line
- */
- public static String listDir(File dir) {
- if (dir == null) {
- return "";
- }
- String[] confDirEntries = dir.list();
- if (confDirEntries == null) {
- return "";
- }
- StringBuilder builder = new StringBuilder();
- for (String entry : confDirEntries) {
- builder.append(entry).append("\n");
- }
- return builder.toString();
- }
-
- /**
- * Create a file:// path from a local file
- * @param file file to point the path
- * @return a new Path
- */
- public static Path createLocalPath(File file) {
- return new Path(file.toURI());
- }
-
- public static String getKerberosRealm() {
- try {
- return KerberosUtil.getDefaultRealm();
- } catch (Exception e) {
- log.debug("introspection into JVM internals failed", e);
- return "(unknown)";
-
- }
- }
-
- /**
- * Build up the classpath for execution
- * -behaves very differently on a mini test cluster vs a production
- * production one.
- *
- * @param sliderConfDir relative path to the dir containing slider config
- * options to put on the classpath -or null
- * @param libdir directory containing the JAR files
- * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
- * (and hence the current classpath should be used, not anything built up)
- * @return a classpath
- */
- public static ClasspathConstructor buildClasspath(String sliderConfDir,
- String libdir,
- SliderFileSystem sliderFileSystem,
- boolean usingMiniMRCluster) {
-
- ClasspathConstructor classpath = new ClasspathConstructor();
- classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
-
- // add the runtime classpath needed for tests to work
- if (usingMiniMRCluster) {
- // for mini cluster we pass down the java CP properties
- // and nothing else
- classpath.appendAll(classpath.localJVMClasspath());
- } else {
- if (sliderConfDir != null) {
- classpath.addClassDirectory(sliderConfDir);
- }
- classpath.addLibDir(libdir);
- if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
- classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
- } else {
- log.info(
- "For faster submission of apps, upload dependencies using cmd {} {}",
- SliderActions.ACTION_DEPENDENCY, Arguments.ARG_UPLOAD);
- }
- classpath.addRemoteClasspathEnvVar();
- classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
- }
- return classpath;
- }
-
- /**
- * Verify that a path refers to a directory. If not
- * logs the parent dir then throws an exception
- * @param dir the directory
- * @param errorlog log for output on an error
- * @throws FileNotFoundException if it is not a directory
- */
- public static void verifyIsDir(File dir, Logger errorlog) throws
- FileNotFoundException {
- if (!dir.exists()) {
- errorlog.warn("contents of {}: {}", dir,
- listDir(dir.getParentFile()));
- throw new FileNotFoundException(dir.toString());
- }
- if (!dir.isDirectory()) {
- errorlog.info("contents of {}: {}", dir,
- listDir(dir.getParentFile()));
- throw new FileNotFoundException(
- "Not a directory: " + dir);
- }
- }
-
- /**
- * Verify that a file exists
- * @param file file
- * @param errorlog log for output on an error
- * @throws FileNotFoundException
- */
- public static void verifyFileExists(File file, Logger errorlog) throws
- FileNotFoundException {
- if (!file.exists()) {
- errorlog.warn("contents of {}: {}", file,
- listDir(file.getParentFile()));
- throw new FileNotFoundException(file.toString());
- }
- if (!file.isFile()) {
- throw new FileNotFoundException("Not a file: " + file.toString());
- }
- }
-
- /**
- * verify that a config option is set
- * @param configuration config
- * @param key key
- * @return the value, in case it needs to be verified too
- * @throws BadConfigException if the key is missing
- */
- public static String verifyOptionSet(Configuration configuration, String key,
- boolean allowEmpty) throws BadConfigException {
- String val = configuration.get(key);
- if (val == null) {
- throw new BadConfigException(
- "Required configuration option \"%s\" not defined ", key);
- }
- if (!allowEmpty && val.isEmpty()) {
- throw new BadConfigException(
- "Configuration option \"%s\" must not be empty", key);
- }
- return val;
- }
-
- /**
- * Verify that a keytab property is defined and refers to a non-empty file
- *
- * @param siteConf configuration
- * @param prop property to look for
- * @return the file referenced
- * @throws BadConfigException on a failure
- */
- public static File verifyKeytabExists(Configuration siteConf,
- String prop) throws
- BadConfigException {
- String keytab = siteConf.get(prop);
- if (keytab == null) {
- throw new BadConfigException("Missing keytab property %s",
- prop);
-
- }
- File keytabFile = new File(keytab);
- if (!keytabFile.exists()) {
- throw new BadConfigException("Missing keytab file %s defined in %s",
- keytabFile,
- prop);
- }
- if (keytabFile.length() == 0 || !keytabFile.isFile()) {
- throw new BadConfigException("Invalid keytab file %s defined in %s",
- keytabFile,
- prop);
- }
- return keytabFile;
- }
-
- /**
- * Add a subpath to an existing URL. This extends
- * the path, inserting a / between all entries
- * if needed.
- * @param base base path/URL
- * @param path subpath
- * @return base+"/"+subpath
- */
- public static String appendToURL(String base, String path) {
- StringBuilder fullpath = new StringBuilder(base);
- if (!base.endsWith("/")) {
- fullpath.append("/");
- }
- if (path.startsWith("/")) {
- fullpath.append(path.substring(1));
- } else {
- fullpath.append(path);
- }
- return fullpath.toString();
- }
-
- /**
- * Truncate the given string to a maximum length provided
- * with a pad (...) added to the end if expected size if more than 10.
- * @param toTruncate string to truncate; may be null
- * @param maxSize maximum size
- * @return the truncated/padded string.
- */
- public static String truncate(String toTruncate, int maxSize) {
- if (toTruncate == null || maxSize < 1
- || toTruncate.length() <= maxSize) {
- return toTruncate;
- }
-
- String pad = "...";
- if (maxSize < 10) {
- pad = "";
- }
- return toTruncate.substring(0, maxSize - pad.length()).concat(pad);
- }
-
- /**
- * Given a source folder create zipped file
- *
- * @param srcFolder
- * @param zipFile
- *
- * @throws IOException
- */
- public static void zipFolder(File srcFolder, File zipFile) throws IOException {
- log.info("Zipping folder {} to {}", srcFolder.getAbsolutePath(), zipFile.getAbsolutePath());
- List<String> files = new ArrayList<>();
- generateFileList(files, srcFolder, srcFolder, true);
-
- byte[] buffer = new byte[1024];
-
- try (FileOutputStream fos = new FileOutputStream(zipFile)) {
- try (ZipOutputStream zos = new ZipOutputStream(fos)) {
-
- for (String file : files) {
- ZipEntry ze = new ZipEntry(file);
- zos.putNextEntry(ze);
- try (FileInputStream in = new FileInputStream(srcFolder + File.separator + file)) {
- int len;
- while ((len = in.read(buffer)) > 0) {
- zos.write(buffer, 0, len);
- }
- }
- }
- }
- }
- }
-
- /**
- * Given a source folder create a tar.gz file
- *
- * @param libDirs
- * @param tarGzipFile
- *
- * @throws IOException
- */
- public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
- FilenameFilter filter) throws IOException {
- log.info("Tar-gzipping folders {} to {}", libDirs,
- tarGzipFile.getAbsolutePath());
-
- try(TarArchiveOutputStream taos =
- new TarArchiveOutputStream(new GZIPOutputStream(
- new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
- for (String libDir : libDirs) {
- File srcFolder = new File(libDir);
- List<String> files = new ArrayList<>();
- generateFileList(files, srcFolder, srcFolder, true, filter);
- for (String file : files) {
- File srcFile = new File(srcFolder, file);
- TarArchiveEntry tarEntry = new TarArchiveEntry(
- srcFile, file);
- taos.putArchiveEntry(tarEntry);
- try(FileInputStream in = new FileInputStream(srcFile)) {
- org.apache.commons.io.IOUtils.copy(in, taos);
- }
- taos.flush();
- taos.closeArchiveEntry();
- }
- }
- }
- }
-
- private static void generateFileList(List<String> fileList, File node,
- File rootFolder, Boolean relative) {
- generateFileList(fileList, node, rootFolder, relative, null);
- }
-
- private static void generateFileList(List<String> fileList, File node,
- File rootFolder, Boolean relative, FilenameFilter filter) {
- if (node.isFile()) {
- String fileFullPath = node.toString();
- if (relative) {
- fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1,
- fileFullPath.length()));
- } else {
- fileList.add(fileFullPath);
- }
- }
-
- if (node.isDirectory()) {
- String[] subNode = node.list(filter);
- if (subNode == null || subNode.length == 0) {
- return;
- }
- for (String filename : subNode) {
- generateFileList(fileList, new File(node, filename), rootFolder,
- relative, filter);
- }
- }
- }
-
- /**
- * Check for any needed libraries being present. On Unix none are needed;
- * on windows they must be present
- * @return true if all is well
- */
- public static String checkForRequiredNativeLibraries() {
-
- if (!Shell.WINDOWS) {
- return "";
- }
- StringBuilder errorText = new StringBuilder("");
- if (!NativeIO.isAvailable()) {
- errorText.append("No native IO library. ");
- }
- try {
- String path = Shell.getQualifiedBinPath(WINUTILS);
- log.debug("winutils is at {}", path);
- } catch (IOException e) {
- errorText.append("No " + WINUTILS);
- log.warn("No winutils: {}", e, e);
- }
- try {
- File target = new File("target");
- FileUtil.canRead(target);
- } catch (UnsatisfiedLinkError e) {
- log.warn("Failing to link to native IO methods: {}", e, e);
- errorText.append("No native IO methods");
- }
- return errorText.toString();
- }
-
- /**
- * Strictly verify that windows utils is present.
- * Checks go as far as opening the file and looking for
- * the headers.
- * @throws IOException on any problem reading the file
- * @throws FileNotFoundException if the file is not considered valid
- */
- public static void maybeVerifyWinUtilsValid() throws
- IOException,
- SliderException {
- String errorText = SliderUtils.checkForRequiredNativeLibraries();
- if (!errorText.isEmpty()) {
- throw new BadClusterStateException(errorText);
- }
- }
-
- /**
- * Write bytes to a file
- * @param outfile output file
- * @param data data to write
- * @throws IOException on any IO problem
- */
- public static void write(File outfile, byte[] data)
- throws IOException {
- File parentDir = outfile.getCanonicalFile().getParentFile();
- if (parentDir == null) {
- throw new IOException(outfile.getPath() + " has no parent dir");
- }
- if (!parentDir.exists()) {
- if(!parentDir.mkdirs()) {
- throw new IOException("Failed to create parent directory " + parentDir);
- }
- }
- SliderUtils.verifyIsDir(parentDir, log);
- try(FileOutputStream out = new FileOutputStream(outfile)) {
- out.write(data);
- }
- }
-
- /**
- * Compare the times of two applications: most recent app comes first
- * Specifically: the one whose start time value is greater.
- */
- private static class MostRecentlyStartedAppFirst
- implements Comparator<ApplicationReport>, Serializable {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- long x = r1.getStartTime();
- long y = r2.getStartTime();
- return compareTwoLongsReverse(x, y);
- }
- }
-
- /**
- * Compare the times of two applications: most recent app comes first.
- * "Recent"== the app whose start time <i>or finish time</i> is the greatest.
- */
- private static class MostRecentlyStartedOrFinishedFirst
- implements Comparator<ApplicationReport>, Serializable {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- long started1 = r1.getStartTime();
- long started2 = r2.getStartTime();
- long finished1 = r1.getFinishTime();
- long finished2 = r2.getFinishTime();
- long lastEvent1 = Math.max(started1, finished1);
- long lastEvent2 = Math.max(started2, finished2);
- return compareTwoLongsReverse(lastEvent1, lastEvent2);
- }
- }
-
- /**
- * Compare the times of two applications: most recently finished app comes first
- * Specifically: the one whose finish time value is greater.
- */
- private static class MostRecentAppFinishFirst
- implements Comparator<ApplicationReport>, Serializable {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- long x = r1.getFinishTime();
- long y = r2.getFinishTime();
- return compareTwoLongsReverse(x, y);
- }
- }
-
- /**
- * Compare two long values for sorting. As the return value for
- * comparators must be int, the simple value of <code>x-y</code>
- * is inapplicable
- * @param x x value
- * @param y y value
- * @return +ve if x is less than y, -ve if y is greater than x; 0 for equality
- */
- public static int compareTwoLongsReverse(long x, long y) {
- return (x < y) ? 1 : ((x == y) ? 0 : -1);
- }
-
- public static String createNameTag(String name) {
- return "Name: " + name;
- }
-
- public static String createVersionTag(String version) {
- return "Version: " + version;
- }
-
- public static String createDescriptionTag(String description) {
- return "Description: " + description;
- }
-
- public static final String DAYS = ".days";
- public static final String HOURS = ".hours";
- public static final String MINUTES = ".minutes";
- public static final String SECONDS = ".seconds";
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
deleted file mode 100644
index 2f71004..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api;
-
-/**
- * These are the keys valid in resource options
- *
- /*
-
- Container failure window.
-
- The window is calculated in minutes as as (days * 24 *60 + hours* 24 + minutes)
-
- Every interval of this period after the AM is started/restarted becomes
- the time period in which the CONTAINER_FAILURE_THRESHOLD value is calculated.
-
- After the window limit is reached, the failure counts are reset. This
- is not a sliding window/moving average policy, simply a rule such as
- "every six hours the failure count is reset"
-
-
- <pre>
- ===========================================================================
- </pre>
-
- */
-public interface ResourceKeys {
-
-
- /**
- * #of instances of a component: {@value}
- *
- */
- String COMPONENT_INSTANCES = "yarn.component.instances";
-
- /**
- * Whether to use unique names for each instance of a component: {@value}
- */
- String UNIQUE_NAMES = "component.unique.names";
-
- /**
- * Amount of memory to ask YARN for in MB.
- * <i>Important:</i> this may be a hard limit on the
- * amount of RAM that the service can use
- * {@value}
- */
- String YARN_MEMORY = "yarn.memory";
-
- /** {@value} */
- int DEF_YARN_MEMORY = 256;
-
- /**
- * Number of cores/virtual cores to ask YARN for
- * {@value}
- */
- String YARN_CORES = "yarn.vcores";
-
- /**
- * If normalization is set to false, then if the resource (memory and/or
- * vcore) requested by a role is higher than YARN limits, then the resource
- * request is not normalized. If this causes failures at the YARN level then
- * applications are expecting that to happen. Default value is true.
- */
- String YARN_RESOURCE_NORMALIZATION_ENABLED =
- "yarn.resource.normalization.enabled";
-
- /**
- * Number of disks per instance to ask YARN for
- * {@value}
- */
- String YARN_DISKS = "yarn.disks.count-per-instance";
-
- /**
- * Disk size per disk to ask YARN for
- * {@value}
- */
- String YARN_DISK_SIZE = "yarn.disk.size";
-
- /** {@value} */
- int DEF_YARN_CORES = 1;
-
-
- /**
- * Label expression that this container must satisfy
- * {@value}
- */
- String YARN_LABEL_EXPRESSION = "yarn.label.expression";
-
- /** default label expression: */
- String DEF_YARN_LABEL_EXPRESSION = null;
-
-
- /**
- * Constant to indicate that the requirements of a YARN resource limit
- * (cores, memory, ...) should be set to the maximum allowed by
- * the queue into which the YARN container requests are placed.
- */
- String YARN_RESOURCE_MAX = "max";
-
- /**
- * Mandatory property for all roles
- * 1. this must be defined.
- * 2. this must be >= 1
- * 3. this must not match any other role priority in the cluster.
- */
- String COMPONENT_PRIORITY = "yarn.role.priority";
-
- /**
- * placement policy
- */
- String COMPONENT_PLACEMENT_POLICY = "yarn.component.placement.policy";
-
- /**
- * Maximum number of node failures that can be tolerated by a component on a specific node
- */
- String NODE_FAILURE_THRESHOLD =
- "yarn.node.failure.threshold";
-
- /**
- * maximum number of failed containers (in a single role)
- * before the cluster is deemed to have failed {@value}
- */
- String CONTAINER_FAILURE_THRESHOLD =
- "yarn.container.failure.threshold";
-
- /**
- * prefix for the time of the container failure reset window.
- * {@value}
- */
-
- String CONTAINER_FAILURE_WINDOW =
- "yarn.container.failure.window";
-
-
-
- long DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS = 0;
- long DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS = 6;
- long DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES = 0;
-
-
- /**
- * Default failure threshold: {@value}
- */
- int DEFAULT_CONTAINER_FAILURE_THRESHOLD = 5;
-
- /**
- * Default node failure threshold for a component instance: {@value}
- * Should to be lower than default component failure threshold to allow
- * the component to start elsewhere
- */
- int DEFAULT_NODE_FAILURE_THRESHOLD = 3;
-
- /**
- * Failure threshold is unlimited: {@value}
- */
- int NODE_FAILURE_THRESHOLD_UNLIMITED = -1;
-
- /**
- * Time in seconds to escalate placement delay
- */
- String PLACEMENT_ESCALATE_DELAY =
- "yarn.placement.escalate.seconds";
-
- /**
- * Time to have a strict placement policy outstanding before
- * downgrading to a lax placement (for those components which permit that).
- * <ol>
- * <li>For strictly placed components, there's no relaxation.</li>
- * <li>For components with no locality, there's no need to relax</li>
- * </ol>
- *
- */
- int DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS = 30;
-
- /**
- * Log aggregation include, exclude patterns
- */
- String YARN_LOG_INCLUDE_PATTERNS = "yarn.log.include.patterns";
- String YARN_LOG_EXCLUDE_PATTERNS = "yarn.log.exclude.patterns";
-
- String YARN_PROFILE_NAME = "yarn.resource-profile-name";
-
- /**
- * Window of time where application master's failure count
- * can be reset to 0.
- */
- String YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS =
- "yarn.resourcemanager.am.retry-count-window-ms";
-
- /**
- * The default window for Slider.
- */
- long DEFAULT_AM_RETRY_COUNT_WINDOW_MS = 300000;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/RoleKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
deleted file mode 100644
index ce413ff..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api;
-
-/**
- * Standard options for roles
- */
-public interface RoleKeys {
-
-
- /**
- * The name of a role: {@value}
- */
- String ROLE_NAME = "role.name";
-
- /**
- * The group of a role: {@value}
- */
- String ROLE_GROUP = "role.group";
-
- /**
- * The prefix of a role: {@value}
- */
- String ROLE_PREFIX = "role.prefix";
-
- /**
- * Status report: number actually granted : {@value}
- */
- String ROLE_ACTUAL_INSTANCES = "role.actual.instances";
-
- /**
- * Status report: number currently requested: {@value}
- */
- String ROLE_REQUESTED_INSTANCES = "role.requested.instances";
-
- /**
- * Status report: number currently being released: {@value}
- */
- String ROLE_RELEASING_INSTANCES = "role.releasing.instances";
-
- /**
- * Status report: total number that have failed: {@value}
- */
- String ROLE_FAILED_INSTANCES = "role.failed.instances";
-
- /**
- * Status report: number that have failed recently: {@value}
- */
- String ROLE_FAILED_RECENTLY_INSTANCES = "role.failed.recently.instances";
-
- /**
- * Status report: number that have failed for node-related issues: {@value}
- */
- String ROLE_NODE_FAILED_INSTANCES = "role.failed.node.instances";
-
- /**
- * Status report: number that been pre-empted: {@value}
- */
- String ROLE_PREEMPTED_INSTANCES = "role.failed.preempted.instances";
-
- /**
- * Number of pending anti-affine instances: {@value}
- */
- String ROLE_PENDING_AA_INSTANCES = "role.pending.aa.instances";
-
- /**
- * Status report: number currently being released: {@value}
- */
- String ROLE_FAILED_STARTING_INSTANCES = "role.failed.starting.instances";
-
- /**
- * Extra arguments (non-JVM) to use when starting this role
- */
- String ROLE_ADDITIONAL_ARGS = "role.additional.args";
-
- /**
- * JVM heap size for Java applications in MB. Only relevant for Java applications.
- * This MUST be less than or equal to the {@link ResourceKeys#YARN_MEMORY} option
- * {@value}
- */
- String JVM_HEAP = "jvm.heapsize";
-
- /*
- * GC options for Java applications.
- */
- String GC_OPTS = "gc.opts";
-
- /**
- * JVM options other than heap size. Only relevant for Java applications.
- * {@value}
- */
- String JVM_OPTS = "jvm.opts";
-
-
- /**
- * All keys w/ env. are converted into env variables and passed down
- */
- String ENV_PREFIX = "env.";
-
- /**
- * Container service record attribute prefix.
- */
- String SERVICE_RECORD_ATTRIBUTE_PREFIX = "service.record.attribute";
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java
deleted file mode 100644
index fa21211..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
-
-/**
- * This class defines constants that can be used in input spec for
- * variable substitutions
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public interface ServiceApiConstants {
-
- // Constants for service
- String SERVICE_NAME = $("SERVICE_NAME");
-
- String SERVICE_NAME_LC = $("SERVICE_NAME.lc");
-
- String USER = $("USER");
-
- String DOMAIN = $("DOMAIN");
-
- // Constants for component
- String COMPONENT_NAME = $("COMPONENT_NAME");
-
- String COMPONENT_NAME_LC = $("COMPONENT_NAME.lc");
-
- String COMPONENT_INSTANCE_NAME = $("COMPONENT_INSTANCE_NAME");
-
- // Constants for component instance
- String COMPONENT_ID = $("COMPONENT_ID");
-
- String CONTAINER_ID = $("CONTAINER_ID");
-
- // Constants for default cluster ZK
- String CLUSTER_ZK_QUORUM = $("CLUSTER_ZK_QUORUM");
-
- // URI for the default cluster fs
- String CLUSTER_FS_URI = $("CLUSTER_FS_URI");
-
- // the host component of the cluster fs UI
- String CLUSTER_FS_HOST = $("CLUSTER_FS_HOST");
-
- // Path in zookeeper for a specific service
- String SERVICE_ZK_PATH = $("SERVICE_ZK_PATH");
-
- // Constants for service specific hdfs dir
- String SERVICE_HDFS_DIR = $("SERVICE_HDFS_DIR");
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
deleted file mode 100644
index aaf2f88..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License") throws IOException, YarnException; you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api;
-
-import org.apache.hadoop.ipc.VersionedProtocol;
-import org.apache.hadoop.security.KerberosInfo;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.slider.api.proto.Messages;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-
-import java.io.IOException;
-
-/**
- * Cluster protocol. This can currently act as a versioned IPC
- * endpoint or be relayed via protobuf
- */
-@KerberosInfo(serverPrincipal = SliderXmlConfKeys.KEY_KERBEROS_PRINCIPAL)
-public interface SliderClusterProtocol extends VersionedProtocol {
- long versionID = 0x01;
-
- /**
- * Stop the cluster
- */
-
- Messages.StopClusterResponseProto stopCluster(Messages.StopClusterRequestProto request) throws
- IOException, YarnException;
- /**
- * Upgrade the application containers
- *
- * @param request upgrade containers request object
- * @return upgrade containers response object
- * @throws IOException
- * @throws YarnException
- */
- Messages.UpgradeContainersResponseProto upgradeContainers(
- Messages.UpgradeContainersRequestProto request) throws IOException,
- YarnException;
-
-
- Messages.FlexComponentsResponseProto flexComponents(
- Messages.FlexComponentsRequestProto request) throws IOException;
-
- /**
- * Get the current cluster status
- */
- Messages.GetJSONClusterStatusResponseProto getJSONClusterStatus(Messages.GetJSONClusterStatusRequestProto request)
- throws IOException, YarnException;
-
-
- /**
- * List all running nodes in a role
- */
- Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(Messages.ListNodeUUIDsByRoleRequestProto request)
- throws IOException, YarnException;
-
-
- /**
- * Get the details on a node
- */
- Messages.GetNodeResponseProto getNode(Messages.GetNodeRequestProto request)
- throws IOException, YarnException;
-
- /**
- * Get the
- * details on a list of nodes.
- * Unknown nodes are not returned
- * <i>Important: the order of the results are undefined</i>
- */
- Messages.GetClusterNodesResponseProto getClusterNodes(Messages.GetClusterNodesRequestProto request)
- throws IOException, YarnException;
-
- /**
- * Echo back the submitted text (after logging it).
- * Useful for adding information to the log, and for testing round trip
- * operations of the protocol
- * @param request request
- * @return response
- * @throws IOException
- * @throws YarnException
- */
- Messages.EchoResponseProto echo(Messages.EchoRequestProto request) throws IOException, YarnException;
-
- /**
- * Kill an identified container
- * @param request request containing the container to kill
- * @return the response
- * @throws IOException
- * @throws YarnException
- */
- Messages.KillContainerResponseProto killContainer(Messages.KillContainerRequestProto request)
- throws IOException, YarnException;
-
- /**
- * AM to commit suicide. If the Hadoop halt entry point has not been disabled,
- * this will fail rather than return with a response.
- * @param request request
- * @return response (this is not the expected outcome)
- * @throws IOException
- * @throws YarnException
- */
- Messages.AMSuicideResponseProto amSuicide(Messages.AMSuicideRequestProto request)
- throws IOException;
-
- /**
- * Get the application liveness
- * @return current liveness information
- * @throws IOException
- */
- Messages.ApplicationLivenessInformationProto getLivenessInformation(
- Messages.GetApplicationLivenessRequestProto request
- ) throws IOException;
-
- Messages.GetLiveContainersResponseProto getLiveContainers(
- Messages.GetLiveContainersRequestProto request
- ) throws IOException;
-
- Messages.ContainerInformationProto getLiveContainer(
- Messages.GetLiveContainerRequestProto request
- ) throws IOException;
-
- Messages.GetLiveComponentsResponseProto getLiveComponents(
- Messages.GetLiveComponentsRequestProto request
- ) throws IOException;
-
- Messages.ComponentInformationProto getLiveComponent(
- Messages.GetLiveComponentRequestProto request
- ) throws IOException;
-
- Messages.GetLiveNodesResponseProto getLiveNodes(
- Messages.GetLiveNodesRequestProto request
- ) throws IOException;
-
- Messages.NodeInformationProto getLiveNode(
- Messages.GetLiveNodeRequestProto request
- ) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/StateValues.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/StateValues.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/StateValues.java
deleted file mode 100644
index ad66a97..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/StateValues.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api;
-
-/**
- * Enumeration of state values.
- */
-public class StateValues {
-
- private StateValues() {}
-
- /**
- * Specification is incomplete & cannot
- * be used: {@value}.
- */
- public static final int STATE_INCOMPLETE = 0;
-
- /**
- * Spec has been submitted: {@value}
- */
- public static final int STATE_SUBMITTED = 1;
- /**
- * Cluster created: {@value}
- */
- public static final int STATE_CREATED = 2;
- /**
- * Live: {@value}
- */
- public static final int STATE_LIVE = 3;
- /**
- * Not ready.
- */
- public static final int STATE_NOT_READY = 4;
- /**
- * Ready.
- */
- public static final int STATE_READY = 5;
- /**
- * Stopped.
- */
- public static final int STATE_STOPPED = 99;
- /**
- * Destroyed.
- */
- public static final int STATE_DESTROYED = 100;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/StatusKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
deleted file mode 100644
index 8a2c4bb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api;
-import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES;
-/**
- * Contains status and statistics keys
- */
-public interface StatusKeys {
-
- String STATISTICS_CONTAINERS_ACTIVE_REQUESTS = "containers.active.requests";
- String STATISTICS_CONTAINERS_COMPLETED = "containers.completed";
- String STATISTICS_CONTAINERS_DESIRED = "containers.desired";
- String STATISTICS_CONTAINERS_FAILED = "containers.failed";
- String STATISTICS_CONTAINERS_FAILED_RECENTLY = "containers.failed.recently";
- String STATISTICS_CONTAINERS_FAILED_NODE = "containers.failed.node";
- String STATISTICS_CONTAINERS_PREEMPTED = "containers.failed.preempted";
- String STATISTICS_CONTAINERS_LIVE = "containers.live";
- String STATISTICS_CONTAINERS_REQUESTED = "containers.requested";
- String STATISTICS_CONTAINERS_ANTI_AFFINE_PENDING = "containers.anti-affine.pending";
- String STATISTICS_CONTAINERS_STARTED = "containers.start.started";
- String STATISTICS_CONTAINERS_START_FAILED =
- "containers.start.failed";
- String STATISTICS_CONTAINERS_SURPLUS =
- "containers.surplus";
- String STATISTICS_CONTAINERS_UNKNOWN_COMPLETED =
- "containers.unknown.completed";
- /**
- * No of containers provided on AM restart
- */
- String INFO_CONTAINERS_AM_RESTART = "containers.at.am-restart";
-
- String INFO_CREATE_TIME_MILLIS = "create.time.millis";
- String INFO_CREATE_TIME_HUMAN = "create.time";
- String INFO_LIVE_TIME_MILLIS = "live.time.millis";
- String INFO_LIVE_TIME_HUMAN = "live.time";
- String INFO_FLEX_TIME_MILLIS = "flex.time.millis";
- String INFO_FLEX_TIME_HUMAN = "flex.time";
-
- String INFO_MASTER_ADDRESS = "info.master.address";
-
- /**
- * System time in millis when the status report was generated
- */
- String INFO_STATUS_TIME_MILLIS = "status.time.millis";
-
- /**
- * System time in human form when the status report was generated
- */
- String INFO_STATUS_TIME_HUMAN = "status.time";
-
- String INFO_AM_APP_ID = "info.am.app.id";
- String INFO_AM_ATTEMPT_ID = "info.am.attempt.id";
- String INFO_AM_CONTAINER_ID = "info.am.container.id";
- String INFO_AM_HOSTNAME = "info.am.hostname";
- String INFO_AM_RPC_PORT = "info.am.rpc.port";
- String INFO_AM_WEB_PORT = "info.am.web.port";
- String INFO_AM_WEB_URL = "info.am.web.url";
- String INFO_AM_AGENT_STATUS_PORT = "info.am.agent.status.port";
- String INFO_AM_AGENT_OPS_PORT = "info.am.agent.ops.port";
- String INFO_AM_AGENT_OPS_URL = "info.am.agent.ops.url";
- String INFO_AM_AGENT_STATUS_URL = "info.am.agent.status.url";
-
- /**
- * info: #of instances of a component requested: {@value}
- *
- */
- String COMPONENT_INSTANCES_ACTUAL = COMPONENT_INSTANCES + ".actual";
-
- /**
- * info: #of instances of a component requested: {@value}
- *
- */
- String COMPONENT_INSTANCES_REQUESTING = COMPONENT_INSTANCES + ".requesting";
-
- /**
- * info: #of instances of a component being released: {@value}
- *
- */
- String COMPONENT_INSTANCES_RELEASING = COMPONENT_INSTANCES + ".releasing";
-
- /**
- * info: #of instances of a component failed: {@value}
- *
- */
- String COMPONENT_INSTANCES_FAILED = COMPONENT_INSTANCES + ".failed";
-
- /**
- * info: #of instances of a component started: {@value}
- *
- */
- String COMPONENT_INSTANCES_STARTED = COMPONENT_INSTANCES + ".started";
-
-
- /**
- * info: #of instances of a component completed: {@value}
- *
- */
- String COMPONENT_INSTANCES_COMPLETED = COMPONENT_INSTANCES + ".completed";
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java
deleted file mode 100644
index 626efb8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java
+++ /dev/null
@@ -1,463 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonPropertyOrder;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * An Application resource has the following attributes.
- **/
-
-@ApiModel(description = "An Application resource has the following attributes.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@XmlRootElement
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonPropertyOrder({ "name", "state", "resource", "number_of_containers",
- "lifetime", "containers" })
-public class Application extends BaseResource {
- private static final long serialVersionUID = -4491694636566094885L;
-
- private String name = null;
- private String id = null;
- private Artifact artifact = null;
- private Resource resource = null;
- private String launchCommand = null;
- private Date launchTime = null;
- private Long numberOfContainers = null;
- private Long numberOfRunningContainers = null;
- private Long lifetime = null;
- private PlacementPolicy placementPolicy = null;
- private List<Component> components = new ArrayList<>();
- private Configuration configuration = new Configuration();
- private List<Container> containers = new ArrayList<>();
- private ApplicationState state = null;
- private Map<String, String> quicklinks = new HashMap<>();
- private String queue = null;
-
- /**
- * A unique application name.
- **/
- public Application name(String name) {
- this.name = name;
- return this;
- }
-
- @ApiModelProperty(example = "null", required = true, value = "A unique application name.")
- @JsonProperty("name")
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- /**
- * A unique application id.
- **/
- public Application id(String id) {
- this.id = id;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A unique application id.")
- @JsonProperty("id")
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- /**
- * Artifact of single-component applications. Mandatory if components
- * attribute is not specified.
- **/
- public Application artifact(Artifact artifact) {
- this.artifact = artifact;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact of single-component applications. Mandatory if components attribute is not specified.")
- @JsonProperty("artifact")
- public Artifact getArtifact() {
- return artifact;
- }
-
- public void setArtifact(Artifact artifact) {
- this.artifact = artifact;
- }
-
- /**
- * Resource of single-component applications or the global default for
- * multi-component applications. Mandatory if it is a single-component
- * application and if cpus and memory are not specified at the Application
- * level.
- **/
- public Application resource(Resource resource) {
- this.resource = resource;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Resource of single-component applications or the global default for multi-component applications. Mandatory if it is a single-component application and if cpus and memory are not specified at the Application level.")
- @JsonProperty("resource")
- public Resource getResource() {
- return resource;
- }
-
- public void setResource(Resource resource) {
- this.resource = resource;
- }
-
- /**
- * The custom launch command of an application component (optional). If not
- * specified for applications with docker images say, it will default to the
- * default start command of the image. If there is a single component in this
- * application, you can specify this without the need to have a 'components'
- * section.
- **/
- public Application launchCommand(String launchCommand) {
- this.launchCommand = launchCommand;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The custom launch command of an application component (optional). If not specified for applications with docker images say, it will default to the default start command of the image. If there is a single component in this application, you can specify this without the need to have a 'components' section.")
- @JsonProperty("launch_command")
- public String getLaunchCommand() {
- return launchCommand;
- }
-
- @XmlElement(name = "launch_command")
- public void setLaunchCommand(String launchCommand) {
- this.launchCommand = launchCommand;
- }
-
- /**
- * The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.
- **/
- public Application launchTime(Date launchTime) {
- this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.")
- @JsonProperty("launch_time")
- public Date getLaunchTime() {
- return launchTime == null ? null : (Date) launchTime.clone();
- }
-
- @XmlElement(name = "launch_time")
- public void setLaunchTime(Date launchTime) {
- this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
- }
-
- /**
- * Number of containers for each app-component in the application. Each
- * app-component can further override this app-level global default.
- **/
- public Application numberOfContainers(Long numberOfContainers) {
- this.numberOfContainers = numberOfContainers;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Number of containers for each app-component in the application. Each app-component can further override this app-level global default.")
- @JsonProperty("number_of_containers")
- public Long getNumberOfContainers() {
- return numberOfContainers;
- }
-
- @XmlElement(name = "number_of_containers")
- public void setNumberOfContainers(Long numberOfContainers) {
- this.numberOfContainers = numberOfContainers;
- }
-
- /**
- * In get response this provides the total number of running containers for
- * this application (across all components) at the time of request. Note, a
- * subsequent request can return a different number as and when more
- * containers get allocated until it reaches the total number of containers or
- * if a flex request has been made between the two requests.
- **/
- public Application numberOfRunningContainers(Long numberOfRunningContainers) {
- this.numberOfRunningContainers = numberOfRunningContainers;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "In get response this provides the total number of running containers for this application (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.")
- @JsonProperty("number_of_running_containers")
- public Long getNumberOfRunningContainers() {
- return numberOfRunningContainers;
- }
-
- @XmlElement(name = "number_of_running_containers")
- public void setNumberOfRunningContainers(Long numberOfRunningContainers) {
- this.numberOfRunningContainers = numberOfRunningContainers;
- }
-
- /**
- * Life time (in seconds) of the application from the time it reaches the
- * RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For
- * unlimited lifetime do not set a lifetime value.
- **/
- public Application lifetime(Long lifetime) {
- this.lifetime = lifetime;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Life time (in seconds) of the application from the time it reaches the RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.")
- @JsonProperty("lifetime")
- public Long getLifetime() {
- return lifetime;
- }
-
- public void setLifetime(Long lifetime) {
- this.lifetime = lifetime;
- }
-
- /**
- * Advanced scheduling and placement policies (optional). If not specified, it
- * defaults to the default placement policy of the app owner. The design of
- * placement policies are in the works. It is not very clear at this point,
- * how policies in conjunction with labels be exposed to application owners.
- * This is a placeholder for now. The advanced structure of this attribute
- * will be determined by YARN-4902.
- **/
- public Application placementPolicy(PlacementPolicy placementPolicy) {
- this.placementPolicy = placementPolicy;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the app owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to application owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.")
- @JsonProperty("placement_policy")
- public PlacementPolicy getPlacementPolicy() {
- return placementPolicy;
- }
-
- @XmlElement(name = "placement_policy")
- public void setPlacementPolicy(PlacementPolicy placementPolicy) {
- this.placementPolicy = placementPolicy;
- }
-
- /**
- * Components of an application.
- **/
- public Application components(List<Component> components) {
- this.components = components;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Components of an application.")
- @JsonProperty("components")
- public List<Component> getComponents() {
- return components;
- }
-
- public void setComponents(List<Component> components) {
- this.components = components;
- }
-
- public void addComponent(Component component) {
- components.add(component);
- }
-
- public Component getComponent(String name) {
- for (Component component : components) {
- if (component.getName().equals(name)) {
- return component;
- }
- }
- return null;
- }
-
- /**
- * Config properties of an application. Configurations provided at the
- * application/global level are available to all the components. Specific
- * properties can be overridden at the component level.
- **/
- public Application configuration(Configuration configuration) {
- this.configuration = configuration;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Config properties of an application. Configurations provided at the application/global level are available to all the components. Specific properties can be overridden at the component level.")
- @JsonProperty("configuration")
- public Configuration getConfiguration() {
- return configuration;
- }
-
- public void setConfiguration(Configuration configuration) {
- this.configuration = configuration;
- }
-
- /**
- * Containers of a started application. Specifying a value for this attribute
- * for the POST payload raises a validation error. This blob is available only
- * in the GET response of a started application.
- **/
- public Application containers(List<Container> containers) {
- this.containers = containers;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Containers of a started application. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.")
- @JsonProperty("containers")
- public List<Container> getContainers() {
- return containers;
- }
-
- public void setContainers(List<Container> containers) {
- this.containers = containers;
- }
-
- public void addContainer(Container container) {
- this.containers.add(container);
- }
-
- /**
- * State of the application. Specifying a value for this attribute for the
- * POST payload raises a validation error. This attribute is available only in
- * the GET response of a started application.
- **/
- public Application state(ApplicationState state) {
- this.state = state;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "State of the application. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started application.")
- @JsonProperty("state")
- public ApplicationState getState() {
- return state;
- }
-
- public void setState(ApplicationState state) {
- this.state = state;
- }
-
- /**
- * A blob of key-value pairs of quicklinks to be exported for an application.
- **/
- public Application quicklinks(Map<String, String> quicklinks) {
- this.quicklinks = quicklinks;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A blob of key-value pairs of quicklinks to be exported for an application.")
- @JsonProperty("quicklinks")
- public Map<String, String> getQuicklinks() {
- return quicklinks;
- }
-
- public void setQuicklinks(Map<String, String> quicklinks) {
- this.quicklinks = quicklinks;
- }
-
- /**
- * The YARN queue that this application should be submitted to.
- **/
- public Application queue(String queue) {
- this.queue = queue;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The YARN queue that this application should be submitted to.")
- @JsonProperty("queue")
- public String getQueue() {
- return queue;
- }
-
- public void setQueue(String queue) {
- this.queue = queue;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Application application = (Application) o;
- return Objects.equals(this.name, application.name);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(name);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Application {\n");
-
- sb.append(" name: ").append(toIndentedString(name)).append("\n");
- sb.append(" id: ").append(toIndentedString(id)).append("\n");
- sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
- sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
- sb.append(" launchCommand: ").append(toIndentedString(launchCommand))
- .append("\n");
- sb.append(" launchTime: ").append(toIndentedString(launchTime))
- .append("\n");
- sb.append(" numberOfContainers: ")
- .append(toIndentedString(numberOfContainers)).append("\n");
- sb.append(" numberOfRunningContainers: ")
- .append(toIndentedString(numberOfRunningContainers)).append("\n");
- sb.append(" lifetime: ").append(toIndentedString(lifetime)).append("\n");
- sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy))
- .append("\n");
- sb.append(" components: ").append(toIndentedString(components))
- .append("\n");
- sb.append(" configuration: ").append(toIndentedString(configuration))
- .append("\n");
- sb.append(" containers: ").append(toIndentedString(containers))
- .append("\n");
- sb.append(" state: ").append(toIndentedString(state)).append("\n");
- sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
- .append("\n");
- sb.append(" queue: ").append(toIndentedString(queue)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ApplicationState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ApplicationState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ApplicationState.java
deleted file mode 100644
index 6827c16..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ApplicationState.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import io.swagger.annotations.ApiModel;
-
-/**
- * The current state of an application.
- **/
-
-@ApiModel(description = "The current state of an application.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public enum ApplicationState {
- ACCEPTED, STARTED, READY, STOPPED, FAILED;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ApplicationStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ApplicationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ApplicationStatus.java
deleted file mode 100644
index 06960a8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ApplicationStatus.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.util.Objects;
-
-import javax.xml.bind.annotation.XmlRootElement;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * The current status of a submitted application, returned as a response to the
- * GET API.
- **/
-
-@ApiModel(description = "The current status of a submitted application, returned as a response to the GET API.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@XmlRootElement
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class ApplicationStatus extends BaseResource {
- private static final long serialVersionUID = -3469885905347851034L;
-
- private String diagnostics = null;
- private ApplicationState state = null;
- private Integer code = null;
-
- /**
- * Diagnostic information (if any) for the reason of the current state of the
- * application. It typically has a non-null value, if the application is in a
- * non-running state.
- **/
- public ApplicationStatus diagnostics(String diagnostics) {
- this.diagnostics = diagnostics;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Diagnostic information (if any) for the reason of the current state of the application. It typically has a non-null value, if the application is in a non-running state.")
- @JsonProperty("diagnostics")
- public String getDiagnostics() {
- return diagnostics;
- }
-
- public void setDiagnostics(String diagnostics) {
- this.diagnostics = diagnostics;
- }
-
- /**
- * Application state.
- **/
- public ApplicationStatus state(ApplicationState state) {
- this.state = state;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Application state.")
- @JsonProperty("state")
- public ApplicationState getState() {
- return state;
- }
-
- public void setState(ApplicationState state) {
- this.state = state;
- }
-
- /**
- * An error code specific to a scenario which app owners should be able to use
- * to understand the failure in addition to the diagnostic information.
- **/
- public ApplicationStatus code(Integer code) {
- this.code = code;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "An error code specific to a scenario which app owners should be able to use to understand the failure in addition to the diagnostic information.")
- @JsonProperty("code")
- public Integer getCode() {
- return code;
- }
-
- public void setCode(Integer code) {
- this.code = code;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- ApplicationStatus applicationStatus = (ApplicationStatus) o;
- return Objects.equals(this.diagnostics, applicationStatus.diagnostics)
- && Objects.equals(this.state, applicationStatus.state)
- && Objects.equals(this.code, applicationStatus.code);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(diagnostics, state, code);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class ApplicationStatus {\n");
-
- sb.append(" diagnostics: ").append(toIndentedString(diagnostics))
- .append("\n");
- sb.append(" state: ").append(toIndentedString(state)).append("\n");
- sb.append(" code: ").append(toIndentedString(code)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Artifact.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Artifact.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Artifact.java
deleted file mode 100644
index f274d7d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Artifact.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonValue;
-
-/**
- * Artifact of an application component.
- **/
-
-@ApiModel(description = "Artifact of an application component")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Artifact implements Serializable {
- private static final long serialVersionUID = 3608929500111099035L;
-
- private String id = null;
-
- public enum TypeEnum {
- DOCKER("DOCKER"), TARBALL("TARBALL"), APPLICATION("APPLICATION");
-
- private String value;
-
- TypeEnum(String value) {
- this.value = value;
- }
-
- @Override
- @JsonValue
- public String toString() {
- return value;
- }
- }
-
- private TypeEnum type = TypeEnum.DOCKER;
- private String uri = null;
-
- /**
- * Artifact id. Examples are package location uri for tarball based apps,
- * image name for docker, etc.
- **/
- public Artifact id(String id) {
- this.id = id;
- return this;
- }
-
- @ApiModelProperty(example = "null", required = true, value = "Artifact id. Examples are package location uri for tarball based apps, image name for docker, etc.")
- @JsonProperty("id")
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- /**
- * Artifact type, like docker, tarball, etc. (optional).
- **/
- public Artifact type(TypeEnum type) {
- this.type = type;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact type, like docker, tarball, etc. (optional).")
- @JsonProperty("type")
- public TypeEnum getType() {
- return type;
- }
-
- public void setType(TypeEnum type) {
- this.type = type;
- }
-
- /**
- * Artifact location to support multiple artifact stores (optional).
- **/
- public Artifact uri(String uri) {
- this.uri = uri;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact location to support multiple artifact stores (optional).")
- @JsonProperty("uri")
- public String getUri() {
- return uri;
- }
-
- public void setUri(String uri) {
- this.uri = uri;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Artifact artifact = (Artifact) o;
- return Objects.equals(this.id, artifact.id)
- && Objects.equals(this.type, artifact.type)
- && Objects.equals(this.uri, artifact.uri);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(id, type, uri);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Artifact {\n");
-
- sb.append(" id: ").append(toIndentedString(id)).append("\n");
- sb.append(" type: ").append(toIndentedString(type)).append("\n");
- sb.append(" uri: ").append(toIndentedString(uri)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/BaseResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/BaseResource.java
deleted file mode 100644
index a23c1fb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/BaseResource.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import java.io.Serializable;
-
-public class BaseResource implements Serializable {
- private static final long serialVersionUID = 1492603053176889431L;
-
- private String uri;
-
- /**
- * Resource location, e.g. \
- * "/applications/helloworld/containers/container_e3751_1458061340047_0008_01_000002\
- * "
- **/
- public String getUri() {
- return uri;
- }
-
- public void setUri(String uri) {
- this.uri = uri;
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append("BaseResource [uri=");
- builder.append(uri);
- builder.append("]");
- return builder.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java
deleted file mode 100644
index c15f82c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java
+++ /dev/null
@@ -1,449 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * One or more components of the application. If the application is HBase say,
- * then the component can be a simple role like master or regionserver. If the
- * application is a complex business webapp then a component can be other
- * applications say Kafka or Storm. Thereby it opens up the support for complex
- * and nested applications.
- **/
-
-@ApiModel(description = "One or more components of the application. If the application is HBase say, then the component can be a simple role like master or regionserver. If the application is a complex business webapp then a component can be other applications say Kafka or Storm. Thereby it opens up the support for complex and nested applications.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@XmlRootElement
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Component implements Serializable {
- private static final long serialVersionUID = -8430058381509087805L;
-
- private String name = null;
- private List<String> dependencies = new ArrayList<String>();
- private ReadinessCheck readinessCheck = null;
- private Artifact artifact = null;
- private String launchCommand = null;
- private Resource resource = null;
- private Long numberOfContainers = null;
- private Boolean uniqueComponentSupport = false;
- private Boolean runPrivilegedContainer = false;
- private PlacementPolicy placementPolicy = null;
- private Configuration configuration = new Configuration();
- private List<String> quicklinks = new ArrayList<String>();
- private List<Container> containers =
- Collections.synchronizedList(new ArrayList<Container>());
-
- /**
- * Name of the application component (mandatory).
- **/
- public Component name(String name) {
- this.name = name;
- return this;
- }
-
- @ApiModelProperty(example = "null", required = true, value = "Name of the application component (mandatory).")
- @JsonProperty("name")
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- /**
- * An array of application components which should be in READY state (as
- * defined by readiness check), before this component can be started. The
- * dependencies across all components of an application should be represented
- * as a DAG.
- **/
- public Component dependencies(List<String> dependencies) {
- this.dependencies = dependencies;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "An array of application components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an application should be represented as a DAG.")
- @JsonProperty("dependencies")
- public List<String> getDependencies() {
- return dependencies;
- }
-
- public void setDependencies(List<String> dependencies) {
- this.dependencies = dependencies;
- }
-
- /**
- * Readiness check for this app-component.
- **/
- public Component readinessCheck(ReadinessCheck readinessCheck) {
- this.readinessCheck = readinessCheck;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Readiness check for this app-component.")
- @JsonProperty("readiness_check")
- public ReadinessCheck getReadinessCheck() {
- return readinessCheck;
- }
-
- @XmlElement(name = "readiness_check")
- public void setReadinessCheck(ReadinessCheck readinessCheck) {
- this.readinessCheck = readinessCheck;
- }
-
- /**
- * Artifact of the component (optional). If not specified, the application
- * level global artifact takes effect.
- **/
- public Component artifact(Artifact artifact) {
- this.artifact = artifact;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact of the component (optional). If not specified, the application level global artifact takes effect.")
- @JsonProperty("artifact")
- public Artifact getArtifact() {
- return artifact;
- }
-
- public void setArtifact(Artifact artifact) {
- this.artifact = artifact;
- }
-
- /**
- * The custom launch command of this component (optional). When specified at
- * the component level, it overrides the value specified at the global level
- * (if any).
- **/
- public Component launchCommand(String launchCommand) {
- this.launchCommand = launchCommand;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The custom launch command of this component (optional). When specified at the component level, it overrides the value specified at the global level (if any).")
- @JsonProperty("launch_command")
- public String getLaunchCommand() {
- return launchCommand;
- }
-
- @XmlElement(name = "launch_command")
- public void setLaunchCommand(String launchCommand) {
- this.launchCommand = launchCommand;
- }
-
- /**
- * Resource of this component (optional). If not specified, the application
- * level global resource takes effect.
- **/
- public Component resource(Resource resource) {
- this.resource = resource;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Resource of this component (optional). If not specified, the application level global resource takes effect.")
- @JsonProperty("resource")
- public Resource getResource() {
- return resource;
- }
-
- public void setResource(Resource resource) {
- this.resource = resource;
- }
-
- /**
- * Number of containers for this app-component (optional). If not specified,
- * the application level global number_of_containers takes effect.
- **/
- public Component numberOfContainers(Long numberOfContainers) {
- this.numberOfContainers = numberOfContainers;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Number of containers for this app-component (optional). If not specified, the application level global number_of_containers takes effect.")
- @JsonProperty("number_of_containers")
- public Long getNumberOfContainers() {
- return numberOfContainers;
- }
-
- @XmlElement(name = "number_of_containers")
- public void setNumberOfContainers(Long numberOfContainers) {
- this.numberOfContainers = numberOfContainers;
- }
-
- @ApiModelProperty(example = "null", value = "Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.")
- @JsonProperty("containers")
- public List<Container> getContainers() {
- return containers;
- }
-
- public void setContainers(List<Container> containers) {
- this.containers = containers;
- }
-
- public void addContainer(Container container) {
- this.containers.add(container);
- }
-
- public void removeContainer(Container container) {
- containers.remove(container);
- }
- public Container getContainer(String id) {
- for (Container container : containers) {
- if (container.getId().equals(id)) {
- return container;
- }
- }
- return null;
- }
-
- /**
- * Certain applications need to define multiple components using the same
- * artifact and resource profile, differing only in configurations. In such
- * cases, this field helps app owners to avoid creating multiple component
- * definitions with repeated information. The number_of_containers field
- * dictates the initial number of components created. Component names
- * typically differ with a trailing id, but assumptions should not be made on
- * that, as the algorithm can change at any time. Configurations section will
- * be able to use placeholders like ${USER}, ${CLUSTER_NAME} and
- * ${COMPONENT_NAME} to be replaced at runtime with user the app is submitted
- * as, application name and application component name respectively. Launch
- * command can use placeholders like ${APP_COMPONENT_NAME} and ${APP_NAME} to
- * get its component name and app name respectively at runtime. The best part
- * of this feature is that when the component is flexed up, entirely new
- * components (with new trailing ids) are created.
- **/
- public Component uniqueComponentSupport(Boolean uniqueComponentSupport) {
- this.uniqueComponentSupport = uniqueComponentSupport;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Certain applications need to define multiple components using the same artifact and resource profile, differing only in configurations. In such cases, this field helps app owners to avoid creating multiple component definitions with repeated information. The number_of_containers field dictates the initial number of components created. Component names typically differ with a trailing id, but assumptions should not be made on that, as the algorithm can change at any time. Configurations section will be able to use placeholders like ${USER}, ${CLUSTER_NAME} and ${COMPONENT_NAME} to be replaced at runtime with user the app is submitted as, application name and application component name respectively. Launch command can use placeholders like ${APP_COMPONENT_NAME} and ${APP_NAME} to get its component name and app name respectively at runtime. The best part of this feature is that when the component is flexed up, entirely new components (with
new trailing ids) are created.")
- @JsonProperty("unique_component_support")
- public Boolean getUniqueComponentSupport() {
- return uniqueComponentSupport;
- }
-
- @XmlElement(name = "unique_component_support")
- public void setUniqueComponentSupport(Boolean uniqueComponentSupport) {
- this.uniqueComponentSupport = uniqueComponentSupport;
- }
-
- /**
- * Run all containers of this component in privileged mode (YARN-4262).
- **/
- public Component runPrivilegedContainer(Boolean runPrivilegedContainer) {
- this.runPrivilegedContainer = runPrivilegedContainer;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Run all containers of this component in privileged mode (YARN-4262).")
- @JsonProperty("run_privileged_container")
- public Boolean getRunPrivilegedContainer() {
- return runPrivilegedContainer;
- }
-
- @XmlElement(name = "run_privileged_container")
- public void setRunPrivilegedContainer(Boolean runPrivilegedContainer) {
- this.runPrivilegedContainer = runPrivilegedContainer;
- }
-
- /**
- * Advanced scheduling and placement policies for all containers of this
- * component (optional). If not specified, the app level placement_policy
- * takes effect. Refer to the description at the global level for more
- * details.
- **/
- public Component placementPolicy(PlacementPolicy placementPolicy) {
- this.placementPolicy = placementPolicy;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the app level placement_policy takes effect. Refer to the description at the global level for more details.")
- @JsonProperty("placement_policy")
- public PlacementPolicy getPlacementPolicy() {
- return placementPolicy;
- }
-
- @XmlElement(name = "placement_policy")
- public void setPlacementPolicy(PlacementPolicy placementPolicy) {
- this.placementPolicy = placementPolicy;
- }
-
- /**
- * Config properties for this app-component.
- **/
- public Component configuration(Configuration configuration) {
- this.configuration = configuration;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Config properties for this app-component.")
- @JsonProperty("configuration")
- public Configuration getConfiguration() {
- return configuration;
- }
-
- public void setConfiguration(Configuration configuration) {
- this.configuration = configuration;
- }
-
- /**
- * A list of quicklink keys defined at the application level, and to be
- * resolved by this component.
- **/
- public Component quicklinks(List<String> quicklinks) {
- this.quicklinks = quicklinks;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A list of quicklink keys defined at the application level, and to be resolved by this component.")
- @JsonProperty("quicklinks")
- public List<String> getQuicklinks() {
- return quicklinks;
- }
-
- public void setQuicklinks(List<String> quicklinks) {
- this.quicklinks = quicklinks;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Component component = (Component) o;
- return Objects.equals(this.name, component.name)
- && Objects.equals(this.dependencies, component.dependencies)
- && Objects.equals(this.readinessCheck, component.readinessCheck)
- && Objects.equals(this.artifact, component.artifact)
- && Objects.equals(this.launchCommand, component.launchCommand)
- && Objects.equals(this.resource, component.resource)
- && Objects.equals(this.numberOfContainers, component.numberOfContainers)
- && Objects.equals(this.uniqueComponentSupport,
- component.uniqueComponentSupport)
- && Objects.equals(this.runPrivilegedContainer,
- component.runPrivilegedContainer)
- && Objects.equals(this.placementPolicy, component.placementPolicy)
- && Objects.equals(this.configuration, component.configuration)
- && Objects.equals(this.quicklinks, component.quicklinks);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(name, dependencies, readinessCheck, artifact,
- launchCommand, resource, numberOfContainers, uniqueComponentSupport,
- runPrivilegedContainer, placementPolicy, configuration, quicklinks);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Component {\n");
-
- sb.append(" name: ").append(toIndentedString(name)).append("\n");
- sb.append(" dependencies: ").append(toIndentedString(dependencies))
- .append("\n");
- sb.append(" readinessCheck: ").append(toIndentedString(readinessCheck))
- .append("\n");
- sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
- sb.append(" launchCommand: ").append(toIndentedString(launchCommand))
- .append("\n");
- sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
- sb.append(" numberOfContainers: ")
- .append(toIndentedString(numberOfContainers)).append("\n");
- sb.append(" containers: ").append(toIndentedString(containers))
- .append("\n");
- sb.append(" uniqueComponentSupport: ")
- .append(toIndentedString(uniqueComponentSupport)).append("\n");
- sb.append(" runPrivilegedContainer: ")
- .append(toIndentedString(runPrivilegedContainer)).append("\n");
- sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy))
- .append("\n");
- sb.append(" configuration: ").append(toIndentedString(configuration))
- .append("\n");
- sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
- .append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-
- /**
- * Merge from another component into this component without overwriting.
- */
- public void mergeFrom(Component that) {
- if (this.getArtifact() == null) {
- this.setArtifact(that.getArtifact());
- }
- if (this.getResource() == null) {
- this.setResource(that.getResource());
- }
- if (this.getNumberOfContainers() == null) {
- this.setNumberOfContainers(that.getNumberOfContainers());
- }
- if (this.getLaunchCommand() == null) {
- this.setLaunchCommand(that.getLaunchCommand());
- }
- this.getConfiguration().mergeFrom(that.getConfiguration());
- if (this.getQuicklinks() == null) {
- this.setQuicklinks(that.getQuicklinks());
- }
- if (this.getRunPrivilegedContainer() == null) {
- this.setRunPrivilegedContainer(that.getRunPrivilegedContainer());
- }
- if (this.getUniqueComponentSupport() == null) {
- this.setUniqueComponentSupport(that.getUniqueComponentSupport());
- }
- if (this.getDependencies() == null) {
- this.setDependencies(that.getDependencies());
- }
- if (this.getPlacementPolicy() == null) {
- this.setPlacementPolicy(that.getPlacementPolicy());
- }
- if (this.getReadinessCheck() == null) {
- this.setReadinessCheck(that.getReadinessCheck());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java
deleted file mode 100644
index b4040b6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonValue;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * A config file that needs to be created and made available as a volume in an
- * application component container.
- **/
-
-@ApiModel(description = "A config file that needs to be created and made available as a volume in an application component container.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@XmlRootElement
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class ConfigFile implements Serializable {
- private static final long serialVersionUID = -7009402089417704612L;
-
- public enum TypeEnum {
- XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
- "TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),;
-
- private String value;
-
- TypeEnum(String value) {
- this.value = value;
- }
-
- @Override
- @JsonValue
- public String toString() {
- return value;
- }
- }
-
- private TypeEnum type = null;
- private String destFile = null;
- private String srcFile = null;
- private Map<String, String> props = new HashMap<>();
-
- public ConfigFile copy() {
- ConfigFile copy = new ConfigFile();
- copy.setType(this.getType());
- copy.setSrcFile(this.getSrcFile());
- copy.setDestFile(this.getDestFile());
- if (this.getProps() != null && !this.getProps().isEmpty()) {
- copy.getProps().putAll(this.getProps());
- }
- return copy;
- }
-
- /**
- * Config file in the standard format like xml, properties, json, yaml,
- * template.
- **/
- public ConfigFile type(TypeEnum type) {
- this.type = type;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Config file in the standard format like xml, properties, json, yaml, template.")
- @JsonProperty("type")
- public TypeEnum getType() {
- return type;
- }
-
- public void setType(TypeEnum type) {
- this.type = type;
- }
-
- /**
- * The absolute path that this configuration file should be mounted as, in the
- * application container.
- **/
- public ConfigFile destFile(String destFile) {
- this.destFile = destFile;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The absolute path that this configuration file should be mounted as, in the application container.")
- @JsonProperty("dest_file")
- public String getDestFile() {
- return destFile;
- }
-
- @XmlElement(name = "dest_file")
- public void setDestFile(String destFile) {
- this.destFile = destFile;
- }
-
- /**
- * This provides the source location of the configuration file, the content
- * of which is dumped to dest_file post property substitutions, in the format
- * as specified in type. Typically the src_file would point to a source
- * controlled network accessible file maintained by tools like puppet, chef,
- * or hdfs etc. Currently, only hdfs is supported.
- **/
- public ConfigFile srcFile(String srcFile) {
- this.srcFile = srcFile;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "This provides the source location of the configuration file, "
- + "the content of which is dumped to dest_file post property substitutions, in the format as specified in type. "
- + "Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.")
- @JsonProperty("src_file")
- public String getSrcFile() {
- return srcFile;
- }
-
- @XmlElement(name = "src_file")
- public void setSrcFile(String srcFile) {
- this.srcFile = srcFile;
- }
-
- /**
- A blob of key value pairs that will be dumped in the dest_file in the format
- as specified in type. If src_file is specified, src_file content are dumped
- in the dest_file and these properties will overwrite, if any, existing
- properties in src_file or be added as new properties in src_file.
- **/
- public ConfigFile props(Map<String, String> props) {
- this.props = props;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be dumped in the dest_file in the format as specified in type."
- + " If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any,"
- + " existing properties in src_file or be added as new properties in src_file.")
- @JsonProperty("props")
- public Map<String, String> getProps() {
- return props;
- }
-
- public void setProps(Map<String, String> props) {
- this.props = props;
- }
-
- public long getLong(String name, long defaultValue) {
- if (name == null) {
- return defaultValue;
- }
- String value = props.get(name.trim());
- return Long.parseLong(value);
- }
-
- public boolean getBoolean(String name, boolean defaultValue) {
- if (name == null) {
- return defaultValue;
- }
- return Boolean.valueOf(props.get(name.trim()));
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- ConfigFile configFile = (ConfigFile) o;
- return Objects.equals(this.type, configFile.type)
- && Objects.equals(this.destFile, configFile.destFile)
- && Objects.equals(this.srcFile, configFile.srcFile);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(type, destFile, srcFile, props);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class ConfigFile {\n");
-
- sb.append(" type: ").append(toIndentedString(type)).append("\n");
- sb.append(" destFile: ").append(toIndentedString(destFile)).append("\n");
- sb.append(" srcFile: ").append(toIndentedString(srcFile)).append("\n");
- sb.append(" props: ").append(toIndentedString(props)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/727e6d78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/727e6d78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/727e6d78
Branch: refs/heads/yarn-native-services
Commit: 727e6d78a2c24e089398cfe6982a30badc930fd2
Parents: 4a9bc1f
Author: Billie Rinaldi <bi...@apache.org>
Authored: Wed Aug 23 12:31:47 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 25 16:37:21 2017 -0700
----------------------------------------------------------------------
.../assemblies/hadoop-yarn-services-dist.xml | 30 +
.../assemblies/hadoop-yarn-slider-dist.xml | 30 -
hadoop-yarn-project/hadoop-yarn/bin/yarn | 8 +-
.../service/webapp/ApplicationApiService.java | 275 ++
.../service/webapp/ApplicationApiWebApp.java | 123 +
.../yarn/services/api/ApplicationApi.java | 36 -
.../api/impl/ApplicationApiService.java | 288 --
.../services/webapp/ApplicationApiWebApp.java | 125 -
...RN-Simplified-V1-API-Layer-For-Services.yaml | 3 -
.../src/main/webapp/WEB-INF/web.xml | 2 +-
.../dev-support/findbugs-exclude.xml | 93 +-
.../conf/slideram-log4j.properties | 68 -
.../conf/yarnservice-log4j.properties | 62 +
.../hadoop-yarn-slider-core/pom.xml | 30 +-
.../slider/server/avro/RoleHistoryRecord.avsc | 114 -
.../apache/hadoop/security/KerberosDiags.java | 677 -----
.../yarn/service/ContainerFailureTracker.java | 4 +-
.../yarn/service/ContainerLaunchService.java | 101 -
.../hadoop/yarn/service/ServiceContext.java | 6 +-
.../hadoop/yarn/service/ServiceMaster.java | 9 +-
.../hadoop/yarn/service/ServiceMonitor.java | 149 -
.../hadoop/yarn/service/ServiceScheduler.java | 44 +-
.../api/constants/ServiceApiConstants.java | 69 +
.../yarn/service/api/records/Application.java | 466 +++
.../service/api/records/ApplicationState.java | 33 +
.../service/api/records/ApplicationStatus.java | 148 +
.../yarn/service/api/records/Artifact.java | 160 +
.../yarn/service/api/records/BaseResource.java | 52 +
.../yarn/service/api/records/Component.java | 412 +++
.../yarn/service/api/records/ConfigFile.java | 225 ++
.../yarn/service/api/records/ConfigFormat.java | 67 +
.../yarn/service/api/records/Configuration.java | 225 ++
.../yarn/service/api/records/Container.java | 297 ++
.../service/api/records/ContainerState.java | 30 +
.../hadoop/yarn/service/api/records/Error.java | 129 +
.../service/api/records/PlacementPolicy.java | 102 +
.../service/api/records/ReadinessCheck.java | 175 ++
.../yarn/service/api/records/Resource.java | 159 +
.../hadoop/yarn/service/client/ServiceCLI.java | 14 +-
.../yarn/service/client/ServiceClient.java | 249 +-
.../client/params/AbstractActionArgs.java | 34 +-
.../client/params/AbstractArgsDelegate.java | 28 +
.../AbstractClusterBuildingActionArgs.java | 58 +
.../service/client/params/ActionBuildArgs.java | 2 -
.../service/client/params/ActionClientArgs.java | 71 +
.../service/client/params/ActionCreateArgs.java | 2 -
.../client/params/ActionDependencyArgs.java | 4 +-
.../service/client/params/ActionExistsArgs.java | 49 +
.../service/client/params/ActionFlexArgs.java | 7 +-
.../service/client/params/ActionFreezeArgs.java | 56 +
.../service/client/params/ActionHelpArgs.java | 44 +
.../service/client/params/ActionKDiagArgs.java | 76 +
.../service/client/params/ActionKeytabArgs.java | 76 +
.../service/client/params/ActionListArgs.java | 76 +
.../client/params/ActionRegistryArgs.java | 218 ++
.../client/params/ActionResolveArgs.java | 153 +
.../client/params/ActionResourceArgs.java | 70 +
.../service/client/params/ActionStatusArgs.java | 51 +
.../service/client/params/ActionThawArgs.java | 67 +
.../service/client/params/ActionTokensArgs.java | 78 +
.../service/client/params/ActionUpdateArgs.java | 32 +
.../yarn/service/client/params/ArgOps.java | 4 +-
.../yarn/service/client/params/Arguments.java | 37 +-
.../yarn/service/client/params/ClientArgs.java | 147 +-
.../yarn/service/client/params/CommonArgs.java | 30 +-
.../client/params/ComponentArgsDelegate.java | 4 +-
.../client/params/DontSplitArguments.java | 34 +
.../client/params/LaunchArgsAccessor.java | 30 +
.../client/params/LaunchArgsDelegate.java | 51 +
.../client/params/OptionArgsDelegate.java | 66 +
.../client/params/PathArgumentConverter.java | 34 +
.../client/params/SliderAMCreateAction.java | 4 -
.../service/client/params/SliderActions.java | 48 +-
.../service/client/params/WaitArgsDelegate.java | 42 +
.../service/client/params/WaitTimeAccessor.java | 24 +
.../service/compinstance/ComponentInstance.java | 20 +-
.../yarn/service/component/Component.java | 15 +-
.../yarn/service/conf/RestApiConstants.java | 43 +
.../yarn/service/conf/SliderExitCodes.java | 2 +-
.../hadoop/yarn/service/conf/SliderKeys.java | 195 --
.../yarn/service/conf/SliderXmlConfKeys.java | 191 --
.../yarn/service/conf/YarnServiceConf.java | 50 +-
.../yarn/service/conf/YarnServiceConstants.java | 90 +
.../containerlaunch/AbstractLauncher.java | 271 ++
.../containerlaunch/ClasspathConstructor.java | 172 ++
.../containerlaunch/CommandLineBuilder.java | 86 +
.../containerlaunch/ContainerLaunchService.java | 101 +
.../containerlaunch/CredentialUtils.java | 319 ++
.../containerlaunch/JavaCommandLineBuilder.java | 181 ++
.../exceptions/BadClusterStateException.java | 36 +
.../BadCommandArgumentsException.java | 30 +
.../service/exceptions/BadConfigException.java | 39 +
.../yarn/service/exceptions/ErrorStrings.java | 57 +
.../service/exceptions/ExitCodeProvider.java | 32 +
.../service/exceptions/LauncherExitCodes.java | 196 ++
.../exceptions/RestApiErrorMessages.java | 92 +
.../exceptions/ServiceLaunchException.java | 73 +
.../service/exceptions/SliderException.java | 66 +
.../yarn/service/exceptions/UsageException.java | 34 +
.../provider/AbstractClientProvider.java | 14 +-
.../provider/AbstractProviderService.java | 18 +-
.../yarn/service/provider/ProviderFactory.java | 3 +-
.../yarn/service/provider/ProviderService.java | 8 +-
.../yarn/service/provider/ProviderUtils.java | 111 +-
.../defaultImpl/DefaultClientProvider.java | 4 +-
.../defaultImpl/DefaultProviderService.java | 6 +-
.../provider/docker/DockerClientProvider.java | 10 +-
.../provider/docker/DockerProviderService.java | 6 +-
.../provider/tarball/TarballClientProvider.java | 10 +-
.../tarball/TarballProviderService.java | 6 +-
.../registry/CustomRegistryConstants.java | 57 +
.../registry/YarnRegistryViewForProviders.java | 225 ++
.../yarn/service/rest/BaseRestClient.java | 149 +
.../yarn/service/rest/ExceptionConverter.java | 128 +
.../hadoop/yarn/service/rest/HttpVerb.java | 57 +
.../rest/SliderURLConnectionFactory.java | 176 ++
.../yarn/service/rest/UgiJerseyBinding.java | 153 +
.../service/rest/UrlConnectionOperations.java | 83 +
.../service/servicemonitor/ServiceMonitor.java | 148 +
.../service/servicemonitor/probe/HttpProbe.java | 110 +
.../servicemonitor/probe/LogEntryBuilder.java | 76 +
.../servicemonitor/probe/MonitorKeys.java | 66 +
.../servicemonitor/probe/MonitorUtils.java | 84 +
.../service/servicemonitor/probe/PortProbe.java | 98 +
.../service/servicemonitor/probe/Probe.java | 100 +
.../servicemonitor/probe/ProbeStatus.java | 160 +
.../ServiceTimelineMetricsConstants.java | 3 -
.../ServiceTimelinePublisher.java | 120 +-
.../utils/ApplicationReportSerDeser.java | 56 +
.../service/utils/ClientRegistryBinder.java | 201 ++
.../hadoop/yarn/service/utils/Comparators.java | 62 +
.../hadoop/yarn/service/utils/ConfigHelper.java | 157 +
.../hadoop/yarn/service/utils/ConfigUtils.java | 97 +
.../yarn/service/utils/CoreFileSystem.java | 521 ++++
.../hadoop/yarn/service/utils/Duration.java | 109 +
.../hadoop/yarn/service/utils/JsonSerDeser.java | 249 ++
.../yarn/service/utils/KerberosDiags.java | 680 +++++
.../yarn/service/utils/PatternValidator.java | 58 +
.../hadoop/yarn/service/utils/PortScanner.java | 113 +
.../service/utils/PublishedConfiguration.java | 196 ++
.../utils/PublishedConfigurationOutputter.java | 212 ++
.../utils/SerializedApplicationReport.java | 98 +
.../yarn/service/utils/ServiceApiUtil.java | 24 +-
.../service/utils/ServiceRegistryUtils.java | 71 +
.../yarn/service/utils/SliderFileSystem.java | 51 +
.../hadoop/yarn/service/utils/SliderUtils.java | 1699 +++++++++++
.../yarn/service/utils/ZookeeperUtils.java | 146 +
.../src/main/java/org/apache/slider/Slider.java | 52 -
.../java/org/apache/slider/api/ClusterNode.java | 220 --
.../org/apache/slider/api/InternalKeys.java | 210 --
.../java/org/apache/slider/api/OptionKeys.java | 85 -
.../org/apache/slider/api/ResourceKeys.java | 210 --
.../java/org/apache/slider/api/RoleKeys.java | 121 -
.../apache/slider/api/ServiceApiConstants.java | 69 -
.../slider/api/SliderClusterProtocol.java | 152 -
.../java/org/apache/slider/api/StateValues.java | 63 -
.../java/org/apache/slider/api/StatusKeys.java | 117 -
.../apache/slider/api/resource/Application.java | 463 ---
.../slider/api/resource/ApplicationState.java | 30 -
.../slider/api/resource/ApplicationStatus.java | 145 -
.../apache/slider/api/resource/Artifact.java | 157 -
.../slider/api/resource/BaseResource.java | 48 -
.../apache/slider/api/resource/Component.java | 449 ---
.../apache/slider/api/resource/ConfigFile.java | 222 --
.../slider/api/resource/Configuration.java | 222 --
.../apache/slider/api/resource/Container.java | 294 --
.../slider/api/resource/ContainerState.java | 25 -
.../org/apache/slider/api/resource/Error.java | 125 -
.../slider/api/resource/PlacementPolicy.java | 99 -
.../slider/api/resource/ReadinessCheck.java | 172 --
.../apache/slider/api/resource/Resource.java | 156 -
.../types/ApplicationLivenessInformation.java | 50 -
.../slider/api/types/ComponentInformation.java | 107 -
.../slider/api/types/ContainerInformation.java | 58 -
.../slider/api/types/NodeEntryInformation.java | 78 -
.../slider/api/types/NodeInformation.java | 59 -
.../slider/api/types/NodeInformationList.java | 41 -
.../slider/api/types/PingInformation.java | 47 -
.../slider/api/types/RestTypeMarshalling.java | 257 --
.../apache/slider/api/types/RoleStatistics.java | 60 -
.../api/types/SliderInstanceDescription.java | 54 -
.../slider/client/ClientRegistryBinder.java | 201 --
.../org/apache/slider/client/ClientUtils.java | 111 -
.../org/apache/slider/client/SliderClient.java | 2783 ------------------
.../apache/slider/client/SliderClientAPI.java | 258 --
.../slider/client/SliderYarnClientImpl.java | 294 --
.../apache/slider/client/TokensOperation.java | 108 -
.../client/ipc/SliderClusterOperations.java | 355 ---
.../slider/client/rest/BaseRestClient.java | 152 -
.../org/apache/slider/common/Constants.java | 35 -
.../common/SliderXMLConfKeysForTesting.java | 83 -
.../common/params/AbstractArgsDelegate.java | 28 -
.../AbstractClusterBuildingActionArgs.java | 91 -
.../common/params/ActionAMSuicideArgs.java | 46 -
.../slider/common/params/ActionClientArgs.java | 71 -
.../common/params/ActionDiagnosticArgs.java | 75 -
.../slider/common/params/ActionExistsArgs.java | 49 -
.../slider/common/params/ActionFreezeArgs.java | 58 -
.../slider/common/params/ActionHelpArgs.java | 53 -
.../slider/common/params/ActionKDiagArgs.java | 88 -
.../slider/common/params/ActionKeytabArgs.java | 76 -
.../common/params/ActionKillContainerArgs.java | 39 -
.../slider/common/params/ActionListArgs.java | 76 -
.../slider/common/params/ActionLookupArgs.java | 78 -
.../slider/common/params/ActionNodesArgs.java | 73 -
.../common/params/ActionRegistryArgs.java | 221 --
.../slider/common/params/ActionResolveArgs.java | 155 -
.../common/params/ActionResourceArgs.java | 70 -
.../slider/common/params/ActionStatusArgs.java | 51 -
.../slider/common/params/ActionThawArgs.java | 69 -
.../slider/common/params/ActionTokensArgs.java | 80 -
.../slider/common/params/ActionUpdateArgs.java | 33 -
.../slider/common/params/ActionUpgradeArgs.java | 45 -
.../slider/common/params/ActionVersionArgs.java | 48 -
.../slider/common/params/AddonArgsDelegate.java | 54 -
.../common/params/DontSplitArguments.java | 34 -
.../common/params/LaunchArgsAccessor.java | 30 -
.../common/params/LaunchArgsDelegate.java | 51 -
.../common/params/OptionArgsDelegate.java | 66 -
.../common/params/PathArgumentConverter.java | 34 -
.../common/params/URIArgumentConverter.java | 40 -
.../common/params/URLArgumentConverter.java | 40 -
.../slider/common/params/WaitArgsDelegate.java | 42 -
.../slider/common/params/WaitTimeAccessor.java | 24 -
.../apache/slider/common/tools/Comparators.java | 62 -
.../slider/common/tools/ConfigHelper.java | 611 ----
.../slider/common/tools/CoreFileSystem.java | 700 -----
.../apache/slider/common/tools/Duration.java | 109 -
.../apache/slider/common/tools/PortScanner.java | 113 -
.../slider/common/tools/SliderFileSystem.java | 51 -
.../apache/slider/common/tools/SliderUtils.java | 2548 ----------------
.../slider/common/tools/SliderVersionInfo.java | 108 -
.../slider/core/buildutils/BuildHelper.java | 48 -
.../apache/slider/core/conf/MapOperations.java | 344 ---
.../exceptions/BadClusterStateException.java | 35 -
.../BadCommandArgumentsException.java | 30 -
.../core/exceptions/BadConfigException.java | 39 -
.../slider/core/exceptions/ErrorStrings.java | 57 -
.../core/exceptions/ExceptionConverter.java | 128 -
.../core/exceptions/NoSuchNodeException.java | 32 -
.../core/exceptions/NotFoundException.java | 35 -
.../exceptions/ServiceNotReadyException.java | 43 -
.../slider/core/exceptions/SliderException.java | 67 -
.../SliderInternalStateException.java | 34 -
.../TriggerClusterTeardownException.java | 41 -
.../UnknownApplicationInstanceException.java | 51 -
.../slider/core/exceptions/UsageException.java | 34 -
.../core/exceptions/WaitTimeoutException.java | 34 -
.../slider/core/launch/AbstractLauncher.java | 271 --
.../core/launch/ClasspathConstructor.java | 172 --
.../slider/core/launch/CommandLineBuilder.java | 89 -
.../slider/core/launch/ContainerLauncher.java | 48 -
.../slider/core/launch/CredentialUtils.java | 379 ---
.../core/launch/JavaCommandLineBuilder.java | 182 --
.../launch/SerializedApplicationReport.java | 98 -
.../slider/core/main/ExitCodeProvider.java | 32 -
.../org/apache/slider/core/main/IrqHandler.java | 103 -
.../slider/core/main/LauncherExitCodes.java | 196 --
.../org/apache/slider/core/main/RunService.java | 62 -
.../core/main/ServiceLaunchException.java | 73 -
.../slider/core/main/ServiceLauncher.java | 642 ----
.../slider/core/main/ServiceShutdownHook.java | 80 -
.../core/persist/ApplicationReportSerDeser.java | 57 -
.../apache/slider/core/persist/Filenames.java | 28 -
.../slider/core/persist/JsonSerDeser.java | 249 --
.../slider/core/persist/LockHeldAction.java | 38 -
.../apache/slider/core/persist/PersistKeys.java | 25 -
.../core/registry/SliderRegistryUtils.java | 62 -
.../slider/core/registry/YarnAppListClient.java | 245 --
.../core/registry/docstore/ConfigFormat.java | 62 -
.../core/registry/docstore/ConfigUtils.java | 96 -
.../core/registry/docstore/ExportEntry.java | 140 -
.../registry/docstore/PublishedConfigSet.java | 100 -
.../docstore/PublishedConfiguration.java | 196 --
.../PublishedConfigurationOutputter.java | 212 --
.../registry/docstore/PublishedExports.java | 149 -
.../docstore/PublishedExportsOutputter.java | 104 -
.../registry/docstore/PublishedExportsSet.java | 98 -
.../slider/core/registry/docstore/UriMap.java | 38 -
.../registry/info/CustomRegistryConstants.java | 57 -
.../core/registry/retrieve/AMWebClient.java | 107 -
.../registry/retrieve/RegistryRetriever.java | 183 --
.../apache/slider/core/restclient/HttpVerb.java | 57 -
.../restclient/SliderURLConnectionFactory.java | 176 --
.../core/restclient/UgiJerseyBinding.java | 154 -
.../restclient/UrlConnectionOperations.java | 90 -
.../slider/core/zk/BlockingZKWatcher.java | 67 -
.../slider/core/zk/MiniZooKeeperCluster.java | 402 ---
.../org/apache/slider/core/zk/ZKCallback.java | 31 -
.../apache/slider/core/zk/ZKIntegration.java | 348 ---
.../apache/slider/core/zk/ZKPathBuilder.java | 82 -
.../apache/slider/core/zk/ZookeeperUtils.java | 147 -
.../providers/AbstractProviderService.java | 148 -
.../apache/slider/providers/MonitorDetail.java | 43 -
.../slider/providers/PlacementPolicy.java | 64 -
.../providers/PlacementPolicyOptions.java | 26 -
.../slider/providers/ProviderCompleted.java | 29 -
.../providers/ProviderCompletedCallable.java | 38 -
.../apache/slider/providers/ProviderCore.java | 31 -
.../apache/slider/providers/ProviderRole.java | 140 -
.../appmaster/AppMasterActionOperations.java | 29 -
.../server/appmaster/PrivilegedConnectToCM.java | 48 -
.../appmaster/ProtobufClusterServices.java | 36 -
.../server/appmaster/PublishedArtifacts.java | 31 -
.../server/appmaster/RoleLaunchService.java | 168 --
.../server/appmaster/SliderAppMaster.java | 2138 --------------
.../appmaster/actions/ActionFlexCluster.java | 41 -
.../server/appmaster/actions/ActionHalt.java | 59 -
.../appmaster/actions/ActionKillContainer.java | 86 -
.../actions/ActionRegisterServiceInstance.java | 52 -
.../appmaster/actions/ActionStartContainer.java | 62 -
.../appmaster/actions/ActionStopQueue.java | 56 -
.../appmaster/actions/ActionStopSlider.java | 162 -
.../actions/ActionUpgradeContainers.java | 106 -
.../server/appmaster/actions/AsyncAction.java | 138 -
.../actions/EscalateOutstandingRequests.java | 45 -
.../actions/MonitorComponentInstances.java | 37 -
.../actions/ProviderStartupCompleted.java | 36 -
.../server/appmaster/actions/QueueAccess.java | 72 -
.../server/appmaster/actions/QueueExecutor.java | 90 -
.../server/appmaster/actions/QueueService.java | 202 --
.../actions/RegisterComponentInstance.java | 57 -
.../appmaster/actions/RenewingAction.java | 141 -
.../appmaster/actions/ResetFailureWindow.java | 49 -
.../actions/ReviewAndFlexApplicationSize.java | 43 -
.../actions/UnregisterComponentInstance.java | 51 -
.../server/appmaster/management/BoolMetric.java | 72 -
.../management/BoolMetricPredicate.java | 44 -
.../server/appmaster/management/LongGauge.java | 98 -
.../management/LongMetricFunction.java | 44 -
.../appmaster/management/MeterAndCounter.java | 109 -
.../management/MetricsAndMonitoring.java | 170 --
.../management/MetricsBindingService.java | 151 -
.../appmaster/management/MetricsConstants.java | 58 -
.../appmaster/management/MetricsKeys.java | 92 -
.../management/PrefixedMetricsSet.java | 53 -
.../server/appmaster/management/Timestamp.java | 33 -
.../management/YarnServiceHealthCheck.java | 38 -
.../server/appmaster/monkey/ChaosEntry.java | 85 -
.../server/appmaster/monkey/ChaosKillAM.java | 48 -
.../appmaster/monkey/ChaosKillContainer.java | 84 -
.../appmaster/monkey/ChaosMonkeyService.java | 138 -
.../server/appmaster/monkey/ChaosTarget.java | 24 -
.../appmaster/monkey/MonkeyPlayAction.java | 48 -
.../operations/AbstractRMOperation.java | 30 -
.../operations/AsyncRMOperationHandler.java | 116 -
.../operations/CancelSingleRequest.java | 54 -
.../operations/ContainerReleaseOperation.java | 47 -
.../operations/ContainerRequestOperation.java | 62 -
.../operations/RMOperationHandler.java | 32 -
.../operations/RMOperationHandlerActions.java | 68 -
.../operations/UpdateBlacklistOperation.java | 45 -
.../slider/server/appmaster/rpc/RpcBinder.java | 307 --
.../appmaster/rpc/SliderAMPolicyProvider.java | 41 -
.../appmaster/rpc/SliderClusterProtocolPB.java | 27 -
.../rpc/SliderClusterProtocolPBImpl.java | 225 --
.../rpc/SliderClusterProtocolProxy.java | 270 --
.../server/appmaster/rpc/SliderIPCService.java | 406 ---
.../appmaster/rpc/SliderRPCSecurityInfo.java | 87 -
.../security/SecurityConfiguration.java | 161 -
.../state/AbstractClusterServices.java | 61 -
.../slider/server/appmaster/state/AppState.java | 2120 -------------
.../appmaster/state/AppStateBindingInfo.java | 60 -
.../state/ContainerAllocationOutcome.java | 44 -
.../state/ContainerAllocationResults.java | 50 -
.../appmaster/state/ContainerAssignment.java | 60 -
.../appmaster/state/ContainerOutcome.java | 61 -
.../appmaster/state/ContainerPriority.java | 109 -
.../state/ContainerReleaseSelector.java | 37 -
.../MostRecentContainerReleaseSelector.java | 51 -
.../server/appmaster/state/NodeEntry.java | 325 --
.../server/appmaster/state/NodeInstance.java | 419 ---
.../slider/server/appmaster/state/NodeMap.java | 174 --
.../appmaster/state/OutstandingRequest.java | 428 ---
.../state/OutstandingRequestTracker.java | 482 ---
.../appmaster/state/ProviderAppState.java | 277 --
.../server/appmaster/state/RoleHistory.java | 1123 -------
.../appmaster/state/RoleHistoryUtils.java | 50 -
.../appmaster/state/RoleHostnamePair.java | 75 -
.../server/appmaster/state/RoleInstance.java | 343 ---
.../server/appmaster/state/RoleStatus.java | 367 ---
.../appmaster/state/SimpleReleaseSelector.java | 33 -
.../state/StateAccessForProviders.java | 275 --
.../server/appmaster/web/HttpCacheHeaders.java | 35 -
.../appmaster/web/SliderAMController.java | 69 -
.../server/appmaster/web/SliderAMWebApp.java | 109 -
.../web/SliderDefaultWrapperServlet.java | 48 -
.../slider/server/appmaster/web/WebAppApi.java | 52 -
.../server/appmaster/web/WebAppApiImpl.java | 69 -
.../server/appmaster/web/layout/AppLayout.java | 32 -
.../web/layout/ClusterSpecificationView.java | 32 -
.../web/layout/ContainerStatsView.java | 33 -
.../appmaster/web/layout/WebUILayout.java | 43 -
.../appmaster/web/rest/AMWadlGenerator.java | 72 -
.../web/rest/AMWadlGeneratorConfig.java | 34 -
.../appmaster/web/rest/AMWebServices.java | 85 -
.../web/rest/AbstractSliderResource.java | 157 -
.../appmaster/web/rest/InsecureAmFilter.java | 104 -
.../web/rest/InsecureAmFilterInitializer.java | 102 -
.../server/appmaster/web/rest/RestPaths.java | 155 -
.../web/rest/SliderJacksonJaxbJsonProvider.java | 58 -
.../ApplicationResouceContentCacheFactory.java | 36 -
.../application/actions/RestActionPing.java | 50 -
.../application/actions/RestActionStop.java | 67 -
.../rest/application/actions/StopResponse.java | 29 -
.../web/rest/application/package-info.java | 24 -
.../application/resources/CachedContent.java | 121 -
.../application/resources/ContentCache.java | 67 -
.../resources/ResourceRefresher.java | 31 -
.../web/rest/publisher/PublisherResource.java | 271 --
.../web/rest/registry/PathEntryResource.java | 45 -
.../web/rest/registry/RegistryResource.java | 151 -
.../web/view/ClusterSpecificationBlock.java | 55 -
.../appmaster/web/view/ContainerStatsBlock.java | 275 --
.../server/appmaster/web/view/IndexBlock.java | 273 --
.../server/appmaster/web/view/NavBlock.java | 62 -
.../appmaster/web/view/SliderHamletBlock.java | 53 -
.../slider/server/avro/LoadedRoleHistory.java | 92 -
.../slider/server/avro/NewerFilesFirst.java | 43 -
.../slider/server/avro/OlderFilesFirst.java | 43 -
.../slider/server/avro/RoleHistoryWriter.java | 449 ---
.../slider/server/servicemonitor/HttpProbe.java | 110 -
.../server/servicemonitor/LogEntryBuilder.java | 76 -
.../server/servicemonitor/MonitorKeys.java | 66 -
.../server/servicemonitor/MonitorUtils.java | 84 -
.../slider/server/servicemonitor/PortProbe.java | 99 -
.../slider/server/servicemonitor/Probe.java | 101 -
.../server/servicemonitor/ProbeStatus.java | 160 -
.../utility/AbstractSliderLaunchedService.java | 118 -
.../services/utility/EndOfServiceWaiter.java | 87 -
.../LaunchedWorkflowCompositeService.java | 117 -
.../services/utility/PatternValidator.java | 61 -
.../server/services/utility/WebAppService.java | 69 -
.../services/workflow/ClosingService.java | 94 -
.../services/workflow/ForkedProcessService.java | 301 --
.../services/workflow/LongLivedProcess.java | 599 ----
.../LongLivedProcessLifecycleEvent.java | 41 -
.../server/services/workflow/ServiceParent.java | 44 -
.../workflow/ServiceTerminatingCallable.java | 92 -
.../workflow/ServiceTerminatingRunnable.java | 72 -
.../services/workflow/ServiceThreadFactory.java | 102 -
.../workflow/WorkflowCallbackService.java | 113 -
.../workflow/WorkflowCompositeService.java | 167 --
.../workflow/WorkflowExecutorService.java | 113 -
.../services/workflow/WorkflowRpcService.java | 76 -
.../WorkflowScheduledExecutorService.java | 38 -
.../workflow/WorkflowSequenceService.java | 306 --
.../server/services/workflow/package-info.java | 172 --
.../YarnRegistryViewForProviders.java | 226 --
.../apache/slider/util/RestApiConstants.java | 63 -
.../slider/util/RestApiErrorMessages.java | 92 -
.../src/main/proto/SliderClusterMessages.proto | 392 ---
.../src/main/proto/SliderClusterProtocol.proto | 140 -
.../org.apache.hadoop.security.SecurityInfo | 15 -
.../org/apache/slider/log4j.properties | 52 -
.../slider/providers/agent/conf/agent.txt | 19 -
.../slider/providers/agent/conf/command.json | 168 --
.../providers/agent/conf/command_template.json | 168 --
.../apache/slider/providers/agent/role-node.xml | 65 -
.../providers/dynamic/application.properties | 25 -
.../providers/slideram/instance/appconf.json | 19 -
.../providers/slideram/instance/internal.json | 17 -
.../providers/slideram/instance/resources.json | 18 -
.../src/main/resources/webapps/slideram/.keep | 0
.../src/scripts/slider_keytabs.sh | 67 -
.../src/scripts/yarnservice.py | 383 ---
.../hadoop-yarn-slider-core/src/site/site.xml | 26 -
.../hadoop/yarn/service/MockServiceAM.java | 9 +-
.../hadoop/yarn/service/ServiceTestUtils.java | 12 +-
.../hadoop/yarn/service/TestServiceApiUtil.java | 54 +-
.../yarn/service/TestYarnNativeServices.java | 30 +-
.../client/TestBuildExternalComponents.java | 8 +-
.../yarn/service/client/TestServiceCLI.java | 15 +-
.../yarn/service/conf/ExampleAppJson.java | 5 +-
.../yarn/service/conf/TestAppJsonResolve.java | 62 +-
.../service/conf/TestLoadExampleAppJson.java | 11 +-
.../service/conf/TestValidateServiceNames.java | 2 +-
.../providers/TestAbstractClientProvider.java | 9 +-
.../service/providers/TestProviderFactory.java | 5 +-
.../servicemonitor/TestServiceMonitor.java | 37 +-
.../TestServiceTimelinePublisher.java | 76 +-
.../org/apache/slider/api/TestRPCBinding.java | 50 -
.../apache/slider/client/TestClientBadArgs.java | 229 --
.../slider/client/TestClientBasicArgs.java | 81 -
.../slider/client/TestCommonArgParsing.java | 522 ----
.../slider/client/TestKeytabCommandOptions.java | 414 ---
.../slider/client/TestSliderClientMethods.java | 142 -
.../slider/client/TestSliderTokensCommand.java | 124 -
.../slider/common/tools/TestConfigHelper.java | 57 -
.../common/tools/TestConfigHelperHDFS.java | 57 -
.../common/tools/TestExecutionEnvironment.java | 67 -
.../slider/common/tools/TestPortScan.java | 184 --
.../common/tools/TestSliderFileSystem.java | 62 -
.../common/tools/TestSliderTestUtils.java | 97 -
.../slider/common/tools/TestSliderUtils.java | 134 -
.../slider/common/tools/TestWindowsSupport.java | 177 --
.../slider/common/tools/TestZKIntegration.java | 187 --
.../TestPublishedConfigurationOutputter.java | 220 --
.../slider/other/TestFilesystemPermissions.java | 263 --
.../apache/slider/other/TestLocalDirStatus.java | 166 --
.../providers/TestAbstractClientProvider.java | 122 -
.../slider/providers/TestProviderFactory.java | 75 -
.../slider/registry/TestConfigSetNaming.java | 85 -
.../slider/registry/TestRegistryPaths.java | 74 -
.../server/appmaster/actions/TestActions.java | 246 --
.../model/appstate/BaseMockAppStateAATest.java | 73 -
.../TestMockAppStateAAOvercapacity.java | 112 -
.../appstate/TestMockAppStateAAPlacement.java | 380 ---
.../TestMockAppStateContainerFailure.java | 387 ---
.../appstate/TestMockAppStateDependencies.java | 163 -
.../TestMockAppStateDynamicHistory.java | 208 --
.../appstate/TestMockAppStateDynamicRoles.java | 243 --
.../TestMockAppStateFlexDynamicRoles.java | 157 -
.../model/appstate/TestMockAppStateFlexing.java | 201 --
.../appstate/TestMockAppStateRMOperations.java | 430 ---
.../TestMockAppStateRebuildOnAMRestart.java | 117 -
.../appstate/TestMockAppStateRolePlacement.java | 122 -
.../appstate/TestMockAppStateRoleRelease.java | 82 -
.../appstate/TestMockAppStateUniqueNames.java | 149 -
.../TestMockContainerResourceAllocations.java | 100 -
.../appstate/TestMockLabelledAAPlacement.java | 156 -
.../TestOutstandingRequestValidation.java | 110 -
.../model/history/TestRoleHistoryAA.java | 269 --
.../history/TestRoleHistoryContainerEvents.java | 447 ---
...TestRoleHistoryFindNodesForNewInstances.java | 177 --
.../history/TestRoleHistoryNIComparators.java | 133 -
...estRoleHistoryOutstandingRequestTracker.java | 385 ---
.../model/history/TestRoleHistoryRW.java | 371 ---
.../history/TestRoleHistoryRWOrdering.java | 162 -
.../history/TestRoleHistoryRequestTracking.java | 298 --
.../history/TestRoleHistoryUpdateBlacklist.java | 117 -
.../server/appmaster/model/mock/Allocator.java | 123 -
.../model/mock/BaseMockAppStateTest.java | 539 ----
.../server/appmaster/model/mock/MockAM.java | 26 -
.../appmaster/model/mock/MockAppState.java | 82 -
.../model/mock/MockApplicationAttemptId.java | 61 -
.../appmaster/model/mock/MockApplicationId.java | 67 -
.../model/mock/MockClusterServices.java | 38 -
.../appmaster/model/mock/MockContainer.java | 131 -
.../appmaster/model/mock/MockContainerId.java | 104 -
.../appmaster/model/mock/MockFactory.java | 273 --
.../appmaster/model/mock/MockFileSystem.java | 32 -
.../server/appmaster/model/mock/MockNodeId.java | 62 -
.../appmaster/model/mock/MockPriority.java | 46 -
.../model/mock/MockRMOperationHandler.java | 120 -
.../appmaster/model/mock/MockRecordFactory.java | 27 -
.../model/mock/MockRegistryOperations.java | 83 -
.../appmaster/model/mock/MockResource.java | 75 -
.../appmaster/model/mock/MockRoleHistory.java | 53 -
.../server/appmaster/model/mock/MockRoles.java | 30 -
.../appmaster/model/mock/MockYarnCluster.java | 342 ---
.../appmaster/model/mock/MockYarnEngine.java | 188 --
.../appmaster/model/monkey/TestMockMonkey.java | 208 --
.../security/TestSecurityConfiguration.java | 215 --
.../web/rest/registry/PathEntryMarshalling.java | 28 -
.../registry/TestRegistryRestMarshalling.java | 51 -
.../web/view/TestClusterSpecificationBlock.java | 74 -
.../web/view/TestContainerStatsBlock.java | 251 --
.../appmaster/web/view/TestIndexBlock.java | 171 --
.../slider/server/management/TestGauges.java | 55 -
.../server/services/workflow/MockService.java | 80 -
.../workflow/ParentWorkflowTestBase.java | 70 -
.../workflow/ProcessCommandFactory.java | 96 -
.../services/workflow/SimpleRunnable.java | 46 -
.../workflow/TestWorkflowClosingService.java | 116 -
.../workflow/TestWorkflowCompositeService.java | 113 -
.../workflow/TestWorkflowExecutorService.java | 66 -
.../workflow/TestWorkflowRpcService.java | 107 -
.../workflow/TestWorkflowSequenceService.java | 151 -
.../TestWorkflowServiceTerminatingRunnable.java | 64 -
.../workflow/WorkflowServiceTestBase.java | 139 -
.../apache/slider/utils/ContractTestUtils.java | 901 ------
.../org/apache/slider/utils/KeysForTests.java | 38 -
.../org/apache/slider/utils/MicroZKCluster.java | 87 -
.../java/org/apache/slider/utils/Outcome.java | 46 -
.../org/apache/slider/utils/SliderTestBase.java | 60 -
.../apache/slider/utils/SliderTestUtils.java | 1065 -------
.../org/apache/slider/utils/TestAssertions.java | 60 -
.../org/apache/slider/utils/TestUtility.java | 181 --
.../slider/utils/YarnMiniClusterTestBase.java | 873 ------
.../slider/utils/YarnZKMiniClusterTestBase.java | 176 --
.../hadoop/yarn/service/conf/examples/app.json | 2 +-
.../appmaster/web/rest/registry/sample.json | 9 -
.../slider/server/avro/history-v01-3-role.json | 6 -
.../slider/server/avro/history-v01-6-role.json | 8 -
.../slider/server/avro/history_v01b_1_role.json | 38 -
.../src/main/webapp/app/routes/yarn-services.js | 2 +-
587 files changed, 13825 insertions(+), 67195 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml
new file mode 100644
index 0000000..5de45a9
--- /dev/null
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml
@@ -0,0 +1,30 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.01
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>hadoop-yarn-slider-dist</id>
+ <formats>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ </dependencySet>
+ </dependencySets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml
deleted file mode 100644
index 5de45a9..0000000
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.01
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
- <id>hadoop-yarn-slider-dist</id>
- <formats>
- <format>dir</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <dependencySets>
- <dependencySet>
- <useProjectArtifact>false</useProjectArtifact>
- </dependencySet>
- </dependencySets>
-</assembly>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index c15a906..c455b29 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -48,7 +48,7 @@ function hadoop_usage
hadoop_add_subcommand "rmadmin" admin "admin tools"
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
- hadoop_add_subcommand "servicesapi" "run yarn-service rest server"
+ hadoop_add_subcommand "servicesapi" "run yarn-native-service rest server"
hadoop_add_subcommand "sharedcachemanager" admin "run the SharedCacheManager daemon"
hadoop_add_subcommand "service" "run a service"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
@@ -151,7 +151,7 @@ function yarncmd_case
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*'
- HADOOP_CLASSNAME='org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp'
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApplicationApiWebApp'
local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
@@ -160,7 +160,7 @@ ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
hadoop_translate_cygwin_path sld
- hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}"
+ hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
;;
sharedcachemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -177,7 +177,7 @@ ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
hadoop_translate_cygwin_path sld
- hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}"
+ hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
;;
timelinereader)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java
new file mode 100644
index 0000000..30fc5ef
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.webapp;
+
+import com.google.inject.Singleton;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.ApplicationState;
+import org.apache.hadoop.yarn.service.api.records.ApplicationStatus;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
+
+@Singleton
+@Path(CONTEXT_ROOT)
+public class ApplicationApiService {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ApplicationApiService.class);
+ private static Configuration YARN_CONFIG = new YarnConfiguration();
+ private static ServiceClient SERVICE_CLIENT;
+
+ static {
+ init();
+ }
+
+ // initialize all the common resources - order is important
+ private static void init() {
+ SERVICE_CLIENT = new ServiceClient();
+ SERVICE_CLIENT.init(YARN_CONFIG);
+ SERVICE_CLIENT.start();
+ }
+
+ @GET
+ @Path(VERSION)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
+ public Response getVersion() {
+ String version = VersionInfo.getBuildVersion();
+ LOG.info(version);
+ return Response.ok(version).build();
+ }
+
+ @POST
+ @Path(APP_ROOT_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response createApplication(Application application) {
+ LOG.info("POST: createApplication = {}", application);
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+ try {
+ ApplicationId applicationId = SERVICE_CLIENT.actionCreate(application);
+ LOG.info("Successfully created application " + application.getName()
+ + " applicationId = " + applicationId);
+ applicationStatus.setState(ApplicationState.ACCEPTED);
+ applicationStatus.setUri(
+ CONTEXT_ROOT + APP_ROOT_PATH + "/" + application
+ .getName());
+ return Response.status(Status.CREATED).entity(applicationStatus).build();
+ } catch (IllegalArgumentException e) {
+ applicationStatus.setDiagnostics(e.getMessage());
+ return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
+ .build();
+ } catch (Exception e) {
+ String message = "Failed to create application " + application.getName();
+ LOG.error(message, e);
+ applicationStatus.setDiagnostics(message + ": " + e.getMessage());
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(applicationStatus).build();
+ }
+ }
+
+ @GET
+ @Path(APP_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response getApplication(@PathParam(APP_NAME) String appName) {
+ LOG.info("GET: getApplication for appName = {}", appName);
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+
+ // app name validation
+ if (!SliderUtils.isClusternameValid(appName)) {
+ applicationStatus.setDiagnostics("Invalid application name: " + appName);
+ applicationStatus.setCode(ERROR_CODE_APP_NAME_INVALID);
+ return Response.status(Status.NOT_FOUND).entity(applicationStatus)
+ .build();
+ }
+
+ try {
+ Application app = SERVICE_CLIENT.getStatus(appName);
+ return Response.ok(app).build();
+ } catch (Exception e) {
+ LOG.error("Get application failed", e);
+ applicationStatus
+ .setDiagnostics("Failed to retrieve application: " + e.getMessage());
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(applicationStatus).build();
+ }
+ }
+
+ @DELETE
+ @Path(APP_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response deleteApplication(@PathParam(APP_NAME) String appName) {
+ LOG.info("DELETE: deleteApplication for appName = {}", appName);
+ return stopApplication(appName, true);
+ }
+
+ private Response stopApplication(String appName, boolean destroy) {
+ try {
+ SERVICE_CLIENT.actionStop(appName, destroy);
+ if (destroy) {
+ SERVICE_CLIENT.actionDestroy(appName);
+ LOG.info("Successfully deleted application {}", appName);
+ } else {
+ LOG.info("Successfully stopped application {}", appName);
+ }
+ return Response.status(Status.NO_CONTENT).build();
+ } catch (ApplicationNotFoundException e) {
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+ applicationStatus.setDiagnostics(
+ "Application " + appName + " not found " + e.getMessage());
+ return Response.status(Status.NOT_FOUND).entity(applicationStatus)
+ .build();
+ } catch (Exception e) {
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+ applicationStatus.setDiagnostics(e.getMessage());
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(applicationStatus).build();
+ }
+ }
+
+ @PUT
+ @Path(COMPONENT_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
+ public Response updateComponent(@PathParam(APP_NAME) String appName,
+ @PathParam(COMPONENT_NAME) String componentName, Component component) {
+
+ if (component.getNumberOfContainers() < 0) {
+ return Response.status(Status.BAD_REQUEST).entity(
+ "Application = " + appName + ", Component = " + component.getName()
+ + ": Invalid number of containers specified " + component
+ .getNumberOfContainers()).build();
+ }
+ try {
+ Map<String, Long> original = SERVICE_CLIENT.flexByRestService(appName,
+ Collections.singletonMap(component.getName(),
+ component.getNumberOfContainers()));
+ return Response.ok().entity(
+ "Updating component " + componentName + " size from " + original
+ .get(componentName) + " to " + component.getNumberOfContainers())
+ .build();
+ } catch (YarnException | IOException e) {
+ ApplicationStatus status = new ApplicationStatus();
+ status.setDiagnostics(e.getMessage());
+ return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status)
+ .build();
+ }
+ }
+
+ @PUT
+ @Path(APP_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response updateApplication(@PathParam(APP_NAME) String appName,
+ Application updateAppData) {
+ LOG.info("PUT: updateApplication for app = {} with data = {}", appName,
+ updateAppData);
+
+ // Ignore the app name provided in updateAppData and always use appName
+ // path param
+ updateAppData.setName(appName);
+
+ // For STOP the app should be running. If already stopped then this
+ // operation will be a no-op. For START it should be in stopped state.
+ // If already running then this operation will be a no-op.
+ if (updateAppData.getState() != null
+ && updateAppData.getState() == ApplicationState.STOPPED) {
+ return stopApplication(appName, false);
+ }
+
+ // If a START is requested
+ if (updateAppData.getState() != null
+ && updateAppData.getState() == ApplicationState.STARTED) {
+ return startApplication(appName);
+ }
+
+ // If new lifetime value specified then update it
+ if (updateAppData.getLifetime() != null
+ && updateAppData.getLifetime() > 0) {
+ return updateLifetime(appName, updateAppData);
+ }
+
+ // flex a single component app
+ if (updateAppData.getNumberOfContainers() != null && !ServiceApiUtil
+ .hasComponent(updateAppData)) {
+ Component defaultComp = ServiceApiUtil.createDefaultComponent(updateAppData);
+ return updateComponent(updateAppData.getName(), defaultComp.getName(),
+ defaultComp);
+ }
+
+ // If nothing happens consider it a no-op
+ return Response.status(Status.NO_CONTENT).build();
+ }
+
+ private Response updateLifetime(String appName, Application updateAppData) {
+ try {
+ String newLifeTime =
+ SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime());
+ return Response.ok("Application " + appName + " lifeTime is successfully updated to "
+ + updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build();
+ } catch (Exception e) {
+ String message =
+ "Failed to update application (" + appName + ") lifetime ("
+ + updateAppData.getLifetime() + ")";
+ LOG.error(message, e);
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(message + " : " + e.getMessage()).build();
+ }
+ }
+
+ private Response startApplication(String appName) {
+ try {
+ SERVICE_CLIENT.actionStart(appName);
+ LOG.info("Successfully started application " + appName);
+ return Response.ok("Application " + appName + " is successfully started").build();
+ } catch (Exception e) {
+ String message = "Failed to start application " + appName;
+ LOG.info(message, e);
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(message + ": " + e.getMessage()).build();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java
new file mode 100644
index 0000000..7225209
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.webapp;
+
+import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.URI;
+import java.util.Arrays;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.eclipse.jetty.webapp.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class launches the web application using Hadoop HttpServer2 (which uses
+ * an embedded Jetty container). This is the entry point to your application.
+ * The Java command used to launch this app should call the main method.
+ */
+public class ApplicationApiWebApp extends AbstractService {
+ private static final Logger logger = LoggerFactory
+ .getLogger(ApplicationApiWebApp.class);
+ private static final String SEP = ";";
+
+ // REST API server for YARN native services
+ private HttpServer2 applicationApiServer;
+
+ public static void main(String[] args) throws IOException {
+ ApplicationApiWebApp apiWebApp = new ApplicationApiWebApp();
+ try {
+ apiWebApp.startWebApp();
+ } catch (Exception e) {
+ if (apiWebApp != null) {
+ apiWebApp.close();
+ }
+ }
+ }
+
+ public ApplicationApiWebApp() {
+ super(ApplicationApiWebApp.class.getName());
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ startWebApp();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (applicationApiServer != null) {
+ applicationApiServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ protected void startWebApp() throws IOException {
+ // The port that we should run on can be set into an environment variable
+ // Look for that variable and default to 9191 if it isn't there.
+ String webPort = System.getenv(PROPERTY_REST_SERVICE_PORT);
+ if (StringUtils.isEmpty(webPort)) {
+ webPort = "9191";
+ }
+
+ String webHost = System.getenv(PROPERTY_REST_SERVICE_HOST);
+ if (StringUtils.isEmpty(webHost)) {
+ webHost = InetAddress.getLocalHost().getHostName();
+ }
+ logger.info("YARN native services REST API running on host {} and port {}",
+ webHost, webPort);
+ logger.info("Configuration = {}", getConfig());
+
+ applicationApiServer = new HttpServer2.Builder()
+ .setName("services-rest-api")
+ .addEndpoint(URI.create("http://" + webHost + ":" + webPort)).build();
+
+ String apiPackages =
+ ApplicationApiService.class.getPackage().getName() + SEP
+ + GenericExceptionHandler.class.getPackage().getName() + SEP
+ + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
+ applicationApiServer.addJerseyResourcePackage(apiPackages, "/*");
+
+ try {
+ logger.info("Application starting up. Logging start...");
+ applicationApiServer.start();
+ logger.info("Server status = {}", applicationApiServer.toString());
+ for (Configuration conf : applicationApiServer.getWebAppContext()
+ .getConfigurations()) {
+ logger.info("Configurations = {}", conf);
+ }
+ logger.info("Context Path = {}", Arrays.asList(applicationApiServer
+ .getWebAppContext().getContextPath()));
+ logger.info("ResourceBase = {}", Arrays.asList(applicationApiServer
+ .getWebAppContext().getResourceBase()));
+ logger.info("War = {}",
+ Arrays.asList(applicationApiServer.getWebAppContext().getWar()));
+ } catch (Exception ex) {
+ logger.error("Hadoop HttpServer2 App **failed**", ex);
+ throw ex;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java
deleted file mode 100644
index 0f4bdae..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.services.api;
-
-import javax.ws.rs.core.Response;
-
-import org.apache.slider.api.resource.Application;
-
-/**
- * Apache Hadoop YARN Services REST API interface.
- *
- */
-public interface ApplicationApi {
- Response createApplication(Application application);
-
- Response getApplications(String state);
-
- Response deleteApplication(String appName);
-
- Response updateApplication(String appName, Application updateAppData);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
deleted file mode 100644
index decd849..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.services.api.impl;
-
-import com.google.inject.Singleton;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.ApplicationState;
-import org.apache.slider.api.resource.ApplicationStatus;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-
-import static org.apache.slider.util.RestApiConstants.*;
-
-@Singleton
-@Path(APPLICATIONS_API_RESOURCE_PATH)
-@Consumes({ MediaType.APPLICATION_JSON })
-@Produces({ MediaType.APPLICATION_JSON })
-public class ApplicationApiService {
- private static final Logger LOG =
- LoggerFactory.getLogger(ApplicationApiService.class);
- private static Configuration YARN_CONFIG = new YarnConfiguration();
- private static ServiceClient SERVICE_CLIENT;
-
- static {
- init();
- }
-
- // initialize all the common resources - order is important
- private static void init() {
- SERVICE_CLIENT = new ServiceClient();
- SERVICE_CLIENT.init(YARN_CONFIG);
- SERVICE_CLIENT.start();
- }
-
- @GET
- @Path("/versions/yarn-service-version")
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON }) public Response getSliderVersion() {
- String version = VersionInfo.getBuildVersion();
- LOG.info(version);
- return Response.ok(version).build();
- }
-
- @POST @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON })
- public Response createApplication(Application application) {
- LOG.info("POST: createApplication = {}", application);
- ApplicationStatus applicationStatus = new ApplicationStatus();
- try {
- ApplicationId applicationId = SERVICE_CLIENT.actionCreate(application);
- LOG.info("Successfully created application " + application.getName()
- + " applicationId = " + applicationId);
- applicationStatus.setState(ApplicationState.ACCEPTED);
- applicationStatus.setUri(
- CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH + "/" + application
- .getName());
- return Response.status(Status.CREATED).entity(applicationStatus).build();
- } catch (IllegalArgumentException e) {
- applicationStatus.setDiagnostics(e.getMessage());
- return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
- .build();
- } catch (Exception e) {
- String message = "Failed to create application " + application.getName();
- LOG.error(message, e);
- applicationStatus.setDiagnostics(message + ": " + e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(applicationStatus).build();
- }
- }
-
- @GET @Path("/{app_name}")
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON })
- public Response getApplication(@PathParam("app_name") String appName) {
- LOG.info("GET: getApplication for appName = {}", appName);
- ApplicationStatus applicationStatus = new ApplicationStatus();
-
- // app name validation
- if (!SliderUtils.isClusternameValid(appName)) {
- applicationStatus.setDiagnostics("Invalid application name: " + appName);
- applicationStatus.setCode(ERROR_CODE_APP_NAME_INVALID);
- return Response.status(Status.NOT_FOUND).entity(applicationStatus)
- .build();
- }
-
- try {
- Application app = SERVICE_CLIENT.getStatus(appName);
- ApplicationReport report = SERVICE_CLIENT.getYarnClient()
- .getApplicationReport(ApplicationId.fromString(app.getId()));
- if (report != null) {
- app.setLifetime(
- report.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME)
- .getRemainingTime());
- LOG.info("Application = {}", app);
- return Response.ok(app).build();
- } else {
- String message = "Application " + appName + " does not exist.";
- LOG.info(message);
- applicationStatus.setCode(ERROR_CODE_APP_DOES_NOT_EXIST);
- applicationStatus.setDiagnostics(message);
- return Response.status(Status.NOT_FOUND).entity(applicationStatus)
- .build();
- }
- } catch (Exception e) {
- LOG.error("Get application failed", e);
- applicationStatus
- .setDiagnostics("Failed to retrieve application: " + e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(applicationStatus).build();
- }
- }
-
- @DELETE
- @Path("/{app_name}")
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON })
- public Response deleteApplication(@PathParam("app_name") String appName) {
- LOG.info("DELETE: deleteApplication for appName = {}", appName);
- return stopApplication(appName, true);
- }
-
- private Response stopApplication(String appName, boolean destroy) {
- try {
- SERVICE_CLIENT.actionStop(appName);
- if (destroy) {
- SERVICE_CLIENT.actionDestroy(appName);
- LOG.info("Successfully deleted application {}", appName);
- } else {
- LOG.info("Successfully stopped application {}", appName);
- }
- return Response.status(Status.NO_CONTENT).build();
- } catch (ApplicationNotFoundException e) {
- ApplicationStatus applicationStatus = new ApplicationStatus();
- applicationStatus.setDiagnostics(
- "Application " + appName + " not found " + e.getMessage());
- return Response.status(Status.NOT_FOUND).entity(applicationStatus)
- .build();
- } catch (Exception e) {
- ApplicationStatus applicationStatus = new ApplicationStatus();
- applicationStatus.setDiagnostics(e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(applicationStatus).build();
- }
- }
-
- @PUT @Path("/{app_name}/components/{component_name}")
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON })
- public Response updateComponent(@PathParam("app_name") String appName,
- @PathParam("component_name") String componentName, Component component) {
-
- if (component.getNumberOfContainers() < 0) {
- return Response.status(Status.BAD_REQUEST).entity(
- "Application = " + appName + ", Component = " + component.getName()
- + ": Invalid number of containers specified " + component
- .getNumberOfContainers()).build();
- }
- try {
- Map<String, Long> original = SERVICE_CLIENT.flexByRestService(appName,
- Collections.singletonMap(component.getName(),
- component.getNumberOfContainers()));
- return Response.ok().entity("Updating " + componentName + " size from "
- + original.get(componentName) + " to "
- + component.getNumberOfContainers()).build();
- } catch (YarnException | IOException e) {
- ApplicationStatus status = new ApplicationStatus();
- status.setDiagnostics(e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status)
- .build();
- }
- }
-
- @PUT @Path("/{app_name}")
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON })
- public Response updateApplication(@PathParam("app_name") String appName,
- Application updateAppData) {
- LOG.info("PUT: updateApplication for app = {} with data = {}", appName,
- updateAppData);
-
- // Ignore the app name provided in updateAppData and always use appName
- // path param
- updateAppData.setName(appName);
-
- // For STOP the app should be running. If already stopped then this
- // operation will be a no-op. For START it should be in stopped state.
- // If already running then this operation will be a no-op.
- if (updateAppData.getState() != null
- && updateAppData.getState() == ApplicationState.STOPPED) {
- return stopApplication(appName, false);
- }
-
- // If a START is requested
- if (updateAppData.getState() != null
- && updateAppData.getState() == ApplicationState.STARTED) {
- return startApplication(appName);
- }
-
- // If new lifetime value specified then update it
- if (updateAppData.getLifetime() != null
- && updateAppData.getLifetime() > 0) {
- return updateLifetime(appName, updateAppData);
- }
-
- // flex a single component app
- if (updateAppData.getNumberOfContainers() != null && !ServiceApiUtil
- .hasComponent(
- updateAppData)) {
- Component defaultComp = ServiceApiUtil.createDefaultComponent(updateAppData);
- return updateComponent(updateAppData.getName(), defaultComp.getName(),
- defaultComp);
- }
-
- // If nothing happens consider it a no-op
- return Response.status(Status.NO_CONTENT).build();
- }
-
- private Response updateLifetime(String appName, Application updateAppData) {
- try {
- String newLifeTime =
- SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime());
- return Response.ok("Application " + appName + " lifeTime is successfully updated to "
- + updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build();
- } catch (Exception e) {
- String message =
- "Failed to update application (" + appName + ") lifetime ("
- + updateAppData.getLifetime() + ")";
- LOG.error(message, e);
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(message + " : " + e.getMessage()).build();
- }
- }
-
- private Response startApplication(String appName) {
- try {
- SERVICE_CLIENT.actionStart(appName);
- LOG.info("Successfully started application " + appName);
- return Response.ok("Application " + appName + " is successfully started").build();
- } catch (Exception e) {
- String message = "Failed to start application " + appName;
- LOG.info(message, e);
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(message + ": " + e.getMessage()).build();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
deleted file mode 100644
index 7fc01a1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.services.webapp;
-
-import static org.apache.slider.util.RestApiConstants.*;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.URI;
-import java.util.Arrays;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.services.api.impl.ApplicationApiService;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.eclipse.jetty.webapp.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class launches the web application using Hadoop HttpServer2 (which uses
- * an embedded Jetty container). This is the entry point to your application.
- * The Java command used to launch this app should call the main method.
- */
-public class ApplicationApiWebApp extends AbstractService {
- private static final Logger logger = LoggerFactory
- .getLogger(ApplicationApiWebApp.class);
- private static final String SEP = ";";
-
- // REST API server for YARN native services
- private HttpServer2 applicationApiServer;
-
- public static void main(String[] args) throws IOException {
- ApplicationApiWebApp apiWebApp = new ApplicationApiWebApp();
- try {
- apiWebApp.startWebApp();
- } catch (Exception e) {
- if (apiWebApp != null) {
- apiWebApp.close();
- }
- }
- }
-
- public ApplicationApiWebApp() {
- super(ApplicationApiWebApp.class.getName());
- }
-
- @Override
- protected void serviceStart() throws Exception {
- startWebApp();
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (applicationApiServer != null) {
- applicationApiServer.stop();
- }
- super.serviceStop();
- }
-
- protected void startWebApp() throws IOException {
- // The port that we should run on can be set into an environment variable
- // Look for that variable and default to 9191 if it isn't there.
- String webPort = System.getenv(PROPERTY_REST_SERVICE_PORT);
- if (StringUtils.isEmpty(webPort)) {
- webPort = "9191";
- }
-
- String webHost = System.getenv(PROPERTY_REST_SERVICE_HOST);
- if (StringUtils.isEmpty(webHost)) {
- webHost = InetAddress.getLocalHost().getHostName();
- }
- logger.info("YARN native services REST API running on host {} and port {}",
- webHost, webPort);
- logger.info("Configuration = {}", getConfig());
-
- applicationApiServer = new HttpServer2.Builder()
- .setName("services-rest-api")
- .addEndpoint(URI.create("http://" + webHost + ":" + webPort)).build();
-
- String apiPackages =
- ApplicationApiService.class.getPackage().getName() + SEP
- + GenericExceptionHandler.class.getPackage().getName() + SEP
- + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
- applicationApiServer.addJerseyResourcePackage(apiPackages, CONTEXT_ROOT
- + "/*");
-
- try {
- logger.info("Application starting up. Logging start...");
- applicationApiServer.start();
- logger.info("Server status = {}", applicationApiServer.toString());
- for (Configuration conf : applicationApiServer.getWebAppContext()
- .getConfigurations()) {
- logger.info("Configurations = {}", conf);
- }
- logger.info("Context Path = {}", Arrays.asList(applicationApiServer
- .getWebAppContext().getContextPath()));
- logger.info("ResourceBase = {}", Arrays.asList(applicationApiServer
- .getWebAppContext().getResourceBase()));
- logger.info("War = {}",
- Arrays.asList(applicationApiServer.getWebAppContext().getWar()));
- } catch (Exception ex) {
- logger.error("Hadoop HttpServer2 App **failed**", ex);
- throw ex;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index 2f090a2..88f74ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -277,9 +277,6 @@ definitions:
type: integer
format: int64
description: Number of containers for this app-component (optional). If not specified, the application level global number_of_containers takes effect.
- unique_component_support:
- type: boolean
- description: Certain applications need to define multiple components using the same artifact and resource profile, differing only in configurations. In such cases, this field helps app owners to avoid creating multiple component definitions with repeated information. The number_of_containers field dictates the initial number of components created. Component names typically differ with a trailing id, but assumptions should not be made on that, as the algorithm can change at any time. Configurations section will be able to use placeholders like ${USER}, ${CLUSTER_NAME} and ${COMPONENT_NAME} to be replaced at runtime with user the app is submitted as, application name and application component name respectively. Launch command can use placeholders like ${APP_COMPONENT_NAME} and ${APP_NAME} to get its component name and app name respectively at runtime. The best part of this feature is that when the component is flexed up, entirely new components (with new trailing ids) are crea
ted.
run_privileged_container:
type: boolean
description: Run all containers of this component in privileged mode (YARN-4262).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
index 31e3051..1282c9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
@@ -21,7 +21,7 @@
<servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
<init-param>
<param-name>com.sun.jersey.config.property.packages</param-name>
- <param-value>org.apache.hadoop.yarn.services.webapp,org.apache.hadoop.yarn.services.api,org.apache.slider.api.resource,org.apache.hadoop.yarn.services.api.impl</param-value>
+ <param-value>org.apache.hadoop.yarn.service.webapp,org.apache.hadoop.yarn.service.api,org.apache.hadoop.yarn.service.api.records</param-value>
</init-param>
<init-param>
<param-name>com.sun.jersey.api.json.POJOMappingFeature</param-name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
index f2bf582..2814cca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
@@ -23,99 +23,26 @@
<class name="org.apache.hadoop.yarn.service.utils.ServiceApiUtil" />
<Bug pattern="MS_CANNOT_BE_FINAL" />
</Match>
-
- <Match>
- <Package name="org.apache.slider.api.proto" />
- </Match>
- <Match>
- <class name="~org\.apache\.slider\.*" />
- <Bug pattern="IS2_INCONSISTENT_SYNC" />
- </Match>
- <Match>
- <Class name="org.apache.slider.core.zk.BlockingZKWatcher" />
- <Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" />
- </Match>
- <Match>
- <Class name="org.apache.slider.server.appmaster.state.ProviderAppState" />
- <Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" />
- </Match>
- <Match>
- <Class name="org.apache.slider.providers.ProviderUtils" />
- <Bug pattern="SWL_SLEEP_WITH_LOCK_HELD" />
- </Match>
- <Match>
- <Class name="org.apache.slider.server.appmaster.SliderAppMaster" />
- <Bug pattern="WA_AWAIT_NOT_IN_LOOP" />
- </Match>
- <Match>
- <Class name="org.apache.slider.core.zk.ZKIntegration" />
- <Bug pattern="SBSC_USE_STRINGBUFFER_CONCATENATION" />
- </Match>
- <Match>
- <Bug pattern="NP_PARAMETER_MUST_BE_NONNULL_BUT_MARKED_AS_NULLABLE" />
- </Match>
- <Match>
- <Class name="org.apache.slider.server.servicemonitor.ProbeWorker" />
- <Bug pattern="SF_SWITCH_FALLTHROUGH" />
- </Match>
- <Match>
- <Class name="org.apache.slider.core.persist.JsonSerDeser" />
- <Bug pattern="UI_INHERITANCE_UNSAFE_GETRESOURCE" />
- </Match>
- <Match>
- <Class name="org.apache.slider.server.appmaster.rpc.SliderAMPolicyProvider" />
- <Bug pattern="EI_EXPOSE_REP" />
- </Match>
- <Match>
- <Class name="org.apache.slider.server.appmaster.state.OutstandingRequest" />
- <Bug pattern="EQ_DOESNT_OVERRIDE_EQUALS" />
- </Match>
- <Match>
- <Class name="org.apache.slider.server.appmaster.web.SliderAMWebApp" />
- <Bug pattern="LG_LOST_LOGGER_DUE_TO_WEAK_REFERENCE" />
- </Match>
- <Match>
- <Class name="org.apache.slider.server.servicemonitor.Probe" />
- <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
- </Match>
<Match>
- <Class name="org.apache.slider.server.appmaster.web.rest.registry.PathEntryResource" />
- <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
- </Match>
- <Match>
- <Class name="org.apache.slider.server.appmaster.web.rest.application.actions.StopResponse" />
- <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
- </Match>
- <Match>
- <Class name="org.apache.slider.server.appmaster.state.ContainerAllocationResults" />
- <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
- </Match>
- <Match>
- <Class name="org.apache.slider.core.persist.JsonSerDeser" />
- <Method name="save" />
+ <Class name="org.apache.hadoop.yarn.service.utils.JsonSerDeser" />
<Bug pattern="OBL_UNSATISFIED_OBLIGATION" />
</Match>
<Match>
- <Class name="org.apache.slider.server.services.workflow.ForkedProcessService" />
- <Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" />
+ <Class name="org.apache.hadoop.yarn.service.utils.JsonSerDeser" />
+ <Bug pattern="UI_INHERITANCE_UNSAFE_GETRESOURCE" />
</Match>
<Match>
- <Class name="org.apache.slider.server.appmaster.state.RoleInstance"/>
+ <Package name="org.apache.hadoop.yarn.service.client.params"/>
<Bug pattern="UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD"/>
</Match>
<Match>
- <Class name="org.apache.slider.client.SliderClient" />
- <Method name="actionRegistryListConfigsYarn" />
- <Bug pattern="OS_OPEN_STREAM" />
+ <Package name="org.apache.hadoop.yarn.service.client.params"/>
+ <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"/>
</Match>
<Match>
- <Class name="org.apache.slider.client.SliderClient" />
- <Method name="actionRegistryListExports" />
- <Bug pattern="OS_OPEN_STREAM" />
- </Match>
- <Match>
- <Class name="org.apache.slider.common.tools.SliderUtils" />
- <Method name="getApplicationResourceInputStream" />
- <Bug pattern="OS_OPEN_STREAM" />
+ <Class name="org.apache.hadoop.yarn.service.client.ServiceClient"/>
+ <Field name="registryClient" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC"/>
</Match>
+
</FindBugsFilter>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java
deleted file mode 100644
index 1964459..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.persist;
-
-public class PersistKeys {
-
- public static final String SCHEMA =
- "http://example.org/specification/v2.0.0";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
deleted file mode 100644
index ac8fca5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-
-/**
- * Miscellaneous methods to assist slider registry work
- *
- */
-public class SliderRegistryUtils {
-
-
- /**
- * Get the registry path for an instance under the user's home node
- * @param instanceName application instance
- * @return a path to the registry location for this application instance.
- */
- public static String registryPathForInstance(String instanceName) {
- return RegistryUtils.servicePath(
- RegistryUtils.currentUser(), SliderKeys.APP_TYPE, instanceName
- );
- }
-
- /**
- * Process a path expanding it if needed.
- * Validation is delegated to later as the core registry will need
- * to do that anyway
- * @param path path
- * @return a path maybe with some expansion
- */
- public static String resolvePath(String path) {
- Preconditions.checkArgument(path!=null, "null path");
- Preconditions.checkArgument(!path.isEmpty(), "empty path");
- String newpath = path;
- if (path.startsWith("~/")) {
- // add user expansion
- newpath = RegistryUtils.homePathForCurrentUser() + path.substring(1);
- } else if (path.equals("~")) {
- newpath = RegistryUtils.homePathForCurrentUser();
- }
- return newpath;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
deleted file mode 100644
index d311fee..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.slider.client.SliderYarnClientImpl;
-import org.apache.slider.api.types.SliderInstanceDescription;
-import org.apache.slider.common.tools.CoreFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Client code for interacting with a list of service instances.
- * The initial logic just enumerates service instances in the YARN RM
- */
-public class YarnAppListClient {
-
- private final SliderYarnClientImpl yarnClient;
- private final String username;
- private final Configuration conf;
- private static final Logger log =
- LoggerFactory.getLogger(YarnAppListClient.class);
-
- public YarnAppListClient(SliderYarnClientImpl yarnClient,
- String username,
- Configuration conf) {
-
- Preconditions.checkArgument(yarnClient != null,
- "yarn client is null: is app inited?");
- Preconditions.checkArgument(username != null,
- "username is null");
- Preconditions.checkArgument(conf != null,
- "conf parameter is null");
- this.yarnClient = yarnClient;
- this.username = username;
- this.conf = conf;
- }
-
- /**
- * find all live instances of a specific app -if there is more than one
- * in the cluster, this returns them all. State should be running or earlier
- * in the lifecycle
- * @param appname application name
- * @return the list of all matching application instances
- */
- public List<ApplicationReport> findAllLiveInstances(String appname)
- throws YarnException, IOException {
- return yarnClient.findAllLiveInstances(username, appname);
- }
-
-
- /**
- * Find an instance of a application belong to the current user.
- * @param appname application name
- * @return the app report or null if none is found
- * @throws YarnException YARN issues
- * @throws IOException IO problems
- */
- public ApplicationReport findInstance(String appname) throws
- YarnException,
- IOException {
- return findInstance(appname, null);
- }
-
- /**
- * Find an instance of a application belong to the current user in specific
- * app states.
- * @param appname application name
- * @param appStates list of states in which application should be in
- * @return the app report or null if none is found
- * @throws YarnException YARN issues
- * @throws IOException IO problems
- */
- public ApplicationReport findInstance(String appname,
- EnumSet<YarnApplicationState> appStates)
- throws YarnException, IOException {
- List<ApplicationReport> instances = listInstances(null, appname, appStates);
- return yarnClient.findClusterInInstanceList(instances, appname);
- }
-
- /**
- * List instances belonging to the specific user
- * @return a possibly empty list of AMs
- */
- public List<ApplicationReport> listInstances()
- throws YarnException, IOException {
- return listInstances(null);
- }
-
- /**
- * List instances belonging to a specific user
- * @return a possibly empty list of AMs
- * @param user user if not the default. null means default, "" means all users,
- * otherwise it is the name of a user
- */
- public List<ApplicationReport> listInstances(String user)
- throws YarnException, IOException {
- return listInstances(user, null);
- }
-
- /**
- * List all instances belonging to a specific user with a specific app name.
- *
- * @param user
- * user if not the default. null means default, "" means all users,
- * otherwise it is the name of a user
- * @param appName
- * application name set as an application tag
- * @return a possibly empty list of AMs
- * @throws YarnException
- * @throws IOException
- */
- public List<ApplicationReport> listInstances(String user, String appName)
- throws YarnException, IOException {
- return listInstances(user, appName, null);
- }
-
- /**
- * List all instances belonging to a specific user, with a specific app name
- * and in specific app states.
- *
- * @param user
- * user if not the default. null means default, "" means all users,
- * otherwise it is the name of a user
- * @param appName
- * application name set as an application tag
- * @param appStates
- * a set of application states within which the app should be in
- * @return a possibly empty list of AMs
- * @throws YarnException
- * @throws IOException
- */
- public List<ApplicationReport> listInstances(String user, String appName,
- EnumSet<YarnApplicationState> appStates)
- throws YarnException, IOException {
- log.debug("listInstances called with user: {}, appName: {}, appStates: {}",
- user, appName, appStates);
- String listUser = user == null ? username : user;
- return yarnClient.listDeployedInstances(listUser, appStates, appName);
- }
-
- /**
- * Enumerate slider instances for the current user, and the
- * most recent app report, where available.
- * @param listOnlyInState boolean to indicate that the instances should
- * only include those in a YARN state
- * <code> minAppState <= currentState <= maxAppState </code>
- *
- * @param minAppState minimum application state to include in enumeration.
- * @param maxAppState maximum application state to include
- * @return a map of application instance name to description
- * @throws IOException Any IO problem
- * @throws YarnException YARN problems
- */
- public Map<String, SliderInstanceDescription> enumSliderInstances(
- boolean listOnlyInState,
- YarnApplicationState minAppState,
- YarnApplicationState maxAppState)
- throws IOException, YarnException {
-
- CoreFileSystem sliderFileSystem = new CoreFileSystem(conf);
- Preconditions.checkArgument(!listOnlyInState || minAppState != null,
- "null minAppState when listOnlyInState set");
- Preconditions.checkArgument(!listOnlyInState || maxAppState != null,
- "null maxAppState when listOnlyInState set");
- if (!listOnlyInState) {
- // if there's not filtering, ask for the entire range of states
- minAppState = YarnApplicationState.NEW;
- maxAppState = YarnApplicationState.KILLED;
- }
- // get the complete list of persistent instances
- Map<String, Path> persistentInstances =
- sliderFileSystem.listPersistentInstances();
- Map<String, SliderInstanceDescription> descriptions =
- new HashMap<String, SliderInstanceDescription>(persistentInstances.size());
-
- if (persistentInstances.isEmpty()) {
- // an empty listing is a success if no cluster was named
- log.debug("No application instances found");
- return descriptions;
- }
-
- // enum those the RM knows about
- List<ApplicationReport> rmInstances = listInstances();
- SliderUtils.sortApplicationsByMostRecent(rmInstances);
- Map<String, ApplicationReport> reportMap =
- SliderUtils.buildApplicationReportMap(rmInstances, minAppState,
- maxAppState);
- log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}",
- persistentInstances.size(),
- rmInstances.size(),
- minAppState, maxAppState,
- reportMap.size());
-
- // at this point there is a list of all persistent instances, and
- // a (possibly filtered) list of application reports
-
- for (Map.Entry<String, Path> entry : persistentInstances.entrySet()) {
- // loop through the persistent values
- String name = entry.getKey();
-
- // look up any report from the (possibly filtered) report set
- ApplicationReport report = reportMap.get(name);
- if (!listOnlyInState || report != null) {
- // if the enum wants to filter in state, only add it if there is
- // a report in that range. Otherwise: include all values
- SliderInstanceDescription sid = new SliderInstanceDescription(
- name, entry.getValue(), report);
- descriptions.put(name, sid);
- }
- }
-
- return descriptions;
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
deleted file mode 100644
index 081688b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.docstore;
-
-import java.util.Locale;
-
-public enum ConfigFormat {
-
- JSON("json"),
- PROPERTIES("properties"),
- XML("xml"),
- HADOOP_XML("hadoop_xml"),
- ENV("env"),
- TEMPLATE("template"),
- YAML("yaml"),
- ;
- ConfigFormat(String suffix) {
- this.suffix = suffix;
- }
-
- private final String suffix;
-
- public String getSuffix() {
- return suffix;
- }
-
-
- @Override
- public String toString() {
- return suffix;
- }
-
- /**
- * Get a matching format or null
- * @param type
- * @return the format
- */
- public static ConfigFormat resolve(String type) {
- for (ConfigFormat format: values()) {
- if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) {
- return format;
- }
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
deleted file mode 100644
index 2e1615b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.core.registry.docstore;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.common.tools.SliderFileSystem;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class ConfigUtils {
- public static final String TEMPLATE_FILE = "template.file";
-
- public static String replaceProps(Map<String, String> config, String content) {
- Map<String, String> tokens = new HashMap<>();
- for (Entry<String, String> entry : config.entrySet()) {
- tokens.put("${" + entry.getKey() + "}", entry.getValue());
- tokens.put("{{" + entry.getKey() + "}}", entry.getValue());
- }
- String value = content;
- for (Map.Entry<String,String> token : tokens.entrySet()) {
- value = value.replaceAll(Pattern.quote(token.getKey()),
- Matcher.quoteReplacement(token.getValue()));
- }
- return value;
- }
-
- public static Map<String, String> replacePropsInConfig(
- Map<String, String> config, Map<String, String> env) {
- Map<String, String> tokens = new HashMap<>();
- for (Entry<String, String> entry : env.entrySet()) {
- tokens.put("${" + entry.getKey() + "}", entry.getValue());
- }
- Map<String, String> newConfig = new HashMap<>();
- for (Entry<String, String> entry : config.entrySet()) {
- String value = entry.getValue();
- for (Map.Entry<String,String> token : tokens.entrySet()) {
- value = value.replaceAll(Pattern.quote(token.getKey()),
- Matcher.quoteReplacement(token.getValue()));
- }
- newConfig.put(entry.getKey(), entry.getValue());
- }
- return newConfig;
- }
-
- public static void prepConfigForTemplateOutputter(ConfigFormat configFormat,
- Map<String, String> config, SliderFileSystem fileSystem,
- String clusterName, String fileName) throws IOException {
- if (!configFormat.equals(ConfigFormat.TEMPLATE)) {
- return;
- }
- Path templateFile = null;
- if (config.containsKey(TEMPLATE_FILE)) {
- templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE));
- if (!fileSystem.isFile(templateFile)) {
- templateFile = fileSystem.buildResourcePath(clusterName,
- config.get(TEMPLATE_FILE));
- }
- if (!fileSystem.isFile(templateFile)) {
- throw new IOException("config specified template file " + config
- .get(TEMPLATE_FILE) + " but " + templateFile + " doesn't exist");
- }
- }
- if (templateFile == null && fileName != null) {
- templateFile = fileSystem.buildResourcePath(fileName);
- if (!fileSystem.isFile(templateFile)) {
- templateFile = fileSystem.buildResourcePath(clusterName,
- fileName);
- }
- }
- if (fileSystem.isFile(templateFile)) {
- config.put("content", fileSystem.cat(templateFile));
- } else {
- config.put("content", "");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
deleted file mode 100644
index dd6e034..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.docstore;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-/**
- * JSON-serializable description of a published key-val configuration.
- *
- * The values themselves are not serialized in the external view; they have to be served up by the far end
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class ExportEntry {
-
- /**
- * The value of the export
- */
- private String value;
- /**
- * The container id of the container that is responsible for the export
- */
- private String containerId;
- /**
- * Tag associated with the container - its usually an identifier different than container id
- * that allows a soft serial id to all containers of a component - e.g. 1, 2, 3, ...
- */
- private String tag;
- /**
- * An export can be at the level of a component or an application
- */
- private String level;
- /**
- * The time when the export was updated
- */
- private String updatedTime;
- /**
- * The time when the export expires
- */
- private String validUntil;
-
- public ExportEntry() {
- }
-
- public String getValue() {
- return value;
- }
-
- public void setValue(String value) {
- this.value = value;
- }
-
- public String getContainerId() {
- return containerId;
- }
-
- public void setContainerId(String containerId) {
- this.containerId = containerId;
- }
-
- public String getTag() {
- return tag;
- }
-
- public void setTag(String tag) {
- this.tag = tag;
- }
-
- public String getLevel() {
- return level;
- }
-
- public void setLevel(String level) {
- this.level = level;
- }
- public String getUpdatedTime() {
- return updatedTime;
- }
-
- public void setUpdatedTime(String updatedTime) {
- this.updatedTime = updatedTime;
- }
-
- public String getValidUntil() {
- return validUntil;
- }
-
- public void setValidUntil(String validUntil) {
- this.validUntil = validUntil;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- ExportEntry that = (ExportEntry) o;
-
- if (value != null ? !value.equals(that.value) : that.value != null)
- return false;
- return containerId != null ? containerId.equals(that.containerId) :
- that.containerId == null;
- }
-
- @Override
- public int hashCode() {
- int result = value != null ? value.hashCode() : 0;
- result = 31 * result + (containerId != null ? containerId.hashCode() : 0);
- return result;
- }
-
- @Override
- public String toString() {
- return new StringBuilder("ExportEntry{").
- append("value='").append(value).append("',").
- append("containerId='").append(containerId).append("',").
- append("tag='").append(tag).append("',").
- append("level='").append(level).append("'").
- append("updatedTime='").append(updatedTime).append("'").
- append("validUntil='").append(validUntil).append("'").
- append(" }").toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
deleted file mode 100644
index edc129e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.docstore;
-
-import org.apache.slider.server.appmaster.web.rest.RestPaths;
-import org.apache.slider.server.services.utility.PatternValidator;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-/**
- * Represents a set of configurations for an application, component, etc.
- * Json serialisable; accessors are synchronized
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class PublishedConfigSet {
-
- private static final PatternValidator validator = new PatternValidator(
- RestPaths.PUBLISHED_CONFIGURATION_REGEXP);
-
- public Map<String, PublishedConfiguration> configurations =
- new HashMap<>();
-
- public PublishedConfigSet() {
- }
-
- /**
- * Put a name -it will be converted to lower case before insertion.
- * Any existing entry will be overwritten (that includes an entry
- * with a different case in the original name)
- * @param name name of entry
- * @param conf configuration
- * @throws IllegalArgumentException if not a valid name
- */
- public void put(String name, PublishedConfiguration conf) {
- String name1 = name.toLowerCase(Locale.ENGLISH);
- validateName(name1);
- configurations.put(name1, conf);
- }
-
- /**
- * Validate the name -restricting it to the set defined in
- * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP}
- * @param name name to validate
- * @throws IllegalArgumentException if not a valid name
- */
- public static void validateName(String name) {
- validator.validate(name);
-
- }
-
- public PublishedConfiguration get(String name) {
- return configurations.get(name);
- }
-
- public boolean contains(String name) {
- return configurations.containsKey(name);
- }
-
- public int size() {
- return configurations.size();
- }
-
- public Set<String> keys() {
- TreeSet<String> keys = new TreeSet<>();
- keys.addAll(configurations.keySet());
- return keys;
- }
-
- public PublishedConfigSet shallowCopy() {
- PublishedConfigSet that = new PublishedConfigSet();
- for (Map.Entry<String, PublishedConfiguration> entry :
- configurations.entrySet()) {
- that.put(entry.getKey(), entry.getValue().shallowCopy());
- }
- return that;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
deleted file mode 100644
index 50b522f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.docstore;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.slider.common.tools.ConfigHelper;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.SerializationConfig;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.io.IOException;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-/**
- * JSON-serializable description of a published key-val configuration.
- *
- * The values themselves are not serialized in the external view; they have
- * to be served up by the far end
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class PublishedConfiguration {
-
- public String description;
- public long updated;
-
- public String updatedTime;
-
- public Map<String, String> entries = new HashMap<>();
-
- public PublishedConfiguration() {
- }
-
- /**
- * build an empty published configuration
- * @param description configuration description
- */
- public PublishedConfiguration(String description) {
- this.description = description;
- }
-
- /**
- * Build a configuration from the entries
- * @param description configuration description
- * @param entries entries to put
- */
- public PublishedConfiguration(String description,
- Iterable<Map.Entry<String, String>> entries) {
- this.description = description;
- putValues(entries);
- }
-
- /**
- * Build a published configuration, using the keys from keysource,
- * but resolving the values from the value source, via Configuration.get()
- * @param description configuration description
- * @param keysource source of keys
- * @param valuesource source of values
- */
- public PublishedConfiguration(String description,
- Iterable<Map.Entry<String, String>> keysource,
- Configuration valuesource) {
- this.description = description;
- putValues(ConfigHelper.resolveConfiguration(keysource, valuesource));
- }
-
-
- /**
- * Is the configuration empty. This means either that it has not
- * been given any values, or it is stripped down copy set down over the
- * wire.
- * @return true if it is empty
- */
- public boolean isEmpty() {
- return entries.isEmpty();
- }
-
-
- public void setUpdated(long updated) {
- this.updated = updated;
- this.updatedTime = new Date(updated).toString();
- }
-
- public long getUpdated() {
- return updated;
- }
-
- /**
- * Set the values from an iterable (this includes a Hadoop Configuration
- * and Java properties object).
- * Any existing value set is discarded
- * @param entries entries to put
- */
- public void putValues(Iterable<Map.Entry<String, String>> entries) {
- this.entries = new HashMap<String, String>();
- for (Map.Entry<String, String> entry : entries) {
- this.entries.put(entry.getKey(), entry.getValue());
- }
-
- }
-
- /**
- * Convert to Hadoop XML
- * @return the configuration as a Hadoop Configuratin
- */
- public Configuration asConfiguration() {
- Configuration conf = new Configuration(false);
- try {
- ConfigHelper.addConfigMap(conf, entries, "");
- } catch (BadConfigException e) {
- // triggered on a null value; switch to a runtime (and discard the stack)
- throw new RuntimeException(e.toString());
- }
- return conf;
- }
-
- public String asConfigurationXML() throws IOException {
- return ConfigHelper.toXml(asConfiguration());
- }
-
- /**
- * Convert values to properties
- * @return a property file
- */
- public Properties asProperties() {
- Properties props = new Properties();
- props.putAll(entries);
- return props;
- }
-
- /**
- * Return the values as json string
- * @return the JSON representation
- * @throws IOException marshalling failure
- */
- public String asJson() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
- String json = mapper.writeValueAsString(entries);
- return json;
- }
-
-
- /**
- * This makes a copy without the nested content -so is suitable
- * for returning as part of the list of a parent's values
- * @return the copy
- */
- public PublishedConfiguration shallowCopy() {
- PublishedConfiguration that = new PublishedConfiguration();
- that.description = this.description;
- that.updated = this.updated;
- that.updatedTime = this.updatedTime;
- return that;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("PublishedConfiguration{");
- sb.append("description='").append(description).append('\'');
- sb.append(" entries = ").append(entries.size());
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * Create an outputter for a given format
- * @param format format to use
- * @return an instance of output
- */
- public PublishedConfigurationOutputter createOutputter(ConfigFormat format) {
- return PublishedConfigurationOutputter.createOutputter(format, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
deleted file mode 100644
index 4ec513c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.docstore;
-
-import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.slider.common.tools.ConfigHelper;
-import org.yaml.snakeyaml.DumperOptions;
-import org.yaml.snakeyaml.DumperOptions.FlowStyle;
-import org.yaml.snakeyaml.Yaml;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.StringWriter;
-import java.util.Properties;
-
-/**
- * Output a published configuration
- */
-public abstract class PublishedConfigurationOutputter {
-
- private static final String COMMENTS = "Generated by Apache Slider";
-
- protected final PublishedConfiguration owner;
-
- protected PublishedConfigurationOutputter(PublishedConfiguration owner) {
- this.owner = owner;
- }
-
- /**
- * Save the config to a destination file, in the format of this outputter
- * @param dest destination file
- * @throws IOException
- */
-/* JDK7
- public void save(File dest) throws IOException {
- try(FileOutputStream out = new FileOutputStream(dest)) {
- save(out);
- out.close();
- }
- }
-*/
- public void save(File dest) throws IOException {
- FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
- }
-
- /**
- * Save the content. The default saves the asString() value
- * to the output stream
- * @param out output stream
- * @throws IOException
- */
- public void save(OutputStream out) throws IOException {
- IOUtils.write(asString(), out, Charsets.UTF_8);
- }
- /**
- * Convert to a string
- * @return the string form
- * @throws IOException
- */
- public abstract String asString() throws IOException;
-
- /**
- * Create an outputter for the chosen format
- * @param format format enumeration
- * @param owner owning config
- * @return the outputter
- */
-
- public static PublishedConfigurationOutputter createOutputter(ConfigFormat format,
- PublishedConfiguration owner) {
- Preconditions.checkNotNull(owner);
- switch (format) {
- case XML:
- case HADOOP_XML:
- return new XmlOutputter(owner);
- case PROPERTIES:
- return new PropertiesOutputter(owner);
- case JSON:
- return new JsonOutputter(owner);
- case ENV:
- return new EnvOutputter(owner);
- case TEMPLATE:
- return new TemplateOutputter(owner);
- case YAML:
- return new YamlOutputter(owner);
- default:
- throw new RuntimeException("Unsupported format :" + format);
- }
- }
-
- public static class XmlOutputter extends PublishedConfigurationOutputter {
-
-
- private final Configuration configuration;
-
- public XmlOutputter(PublishedConfiguration owner) {
- super(owner);
- configuration = owner.asConfiguration();
- }
-
- @Override
- public void save(OutputStream out) throws IOException {
- configuration.writeXml(out);
- }
-
- @Override
- public String asString() throws IOException {
- return ConfigHelper.toXml(configuration);
- }
-
- public Configuration getConfiguration() {
- return configuration;
- }
- }
-
- public static class PropertiesOutputter extends PublishedConfigurationOutputter {
-
- private final Properties properties;
-
- public PropertiesOutputter(PublishedConfiguration owner) {
- super(owner);
- properties = owner.asProperties();
- }
-
- @Override
- public void save(OutputStream out) throws IOException {
- properties.store(out, COMMENTS);
- }
-
-
- public String asString() throws IOException {
- StringWriter sw = new StringWriter();
- properties.store(sw, COMMENTS);
- return sw.toString();
- }
- }
-
-
- public static class JsonOutputter extends PublishedConfigurationOutputter {
-
- public JsonOutputter(PublishedConfiguration owner) {
- super(owner);
- }
-
- @Override
- public String asString() throws IOException {
- return owner.asJson();
- }
- }
-
-
- public static class EnvOutputter extends PublishedConfigurationOutputter {
-
- public EnvOutputter(PublishedConfiguration owner) {
- super(owner);
- }
-
- @Override
- public String asString() throws IOException {
- if (!owner.entries.containsKey("content")) {
- throw new IOException("Configuration has no content field and cannot " +
- "be retrieved as type 'env'");
- }
- String content = owner.entries.get("content");
- return ConfigUtils.replaceProps(owner.entries, content);
- }
- }
-
- public static class TemplateOutputter extends EnvOutputter {
- public TemplateOutputter(PublishedConfiguration owner) {
- super(owner);
- }
- }
-
- public static class YamlOutputter extends PublishedConfigurationOutputter {
-
- private final Yaml yaml;
-
- public YamlOutputter(PublishedConfiguration owner) {
- super(owner);
- DumperOptions options = new DumperOptions();
- options.setDefaultFlowStyle(FlowStyle.BLOCK);
- yaml = new Yaml(options);
- }
-
- public String asString() throws IOException {
- return yaml.dump(owner.entries);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
deleted file mode 100644
index 58e67ee..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.docstore;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.SerializationConfig;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.io.IOException;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
-
-/**
- * JSON-serializable description of a published key-val configuration.
- *
- * The values themselves are not serialized in the external view; they have to be served up by the far end
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class PublishedExports {
-
- public String description;
- public long updated;
- public String updatedTime;
- public Map<String, Set<ExportEntry>> entries = new HashMap<>();
-
- public PublishedExports() {
- }
-
- /**
- * build an empty published configuration
- *
- * @param description configuration description
- */
- public PublishedExports(String description) {
- this.description = description;
- }
-
- /**
- * Build a configuration from the entries
- *
- * @param description configuration description
- * @param entries entries to put
- */
- public PublishedExports(String description,
- Iterable<Entry<String, Set<ExportEntry>>> entries) {
- this.description = description;
- putValues(entries);
- }
-
- /**
- * Is the configuration empty. This means either that it has not been given any values,
- * or it is stripped down copy
- * set down over the wire.
- *
- * @return true if it is empty
- */
- public boolean isEmpty() {
- return entries.isEmpty();
- }
-
- public long getUpdated() {
- return updated;
- }
-
- public void setUpdated(long updated) {
- this.updated = updated;
- this.updatedTime = new Date(updated).toString();
- }
-
-
- public Map<String, Set<ExportEntry>> sortedEntries() {
- Map<String, Set<ExportEntry>> sortedEntries = new TreeMap<>();
- sortedEntries.putAll(entries);
- return sortedEntries;
- }
-
- /**
- * Set the values from an iterable (this includes a Hadoop Configuration and Java properties
- * object). Any existing value set is discarded
- *
- * @param values values to put
- */
- public void putValues(Iterable<Map.Entry<String, Set<ExportEntry>>> values) {
- this.entries = new HashMap<>();
- for (Map.Entry<String, Set<ExportEntry>> entry : values) {
- this.entries.put(entry.getKey(), entry.getValue());
- }
- }
-
- /**
- * Return the values as json string
- *
- * @return the JSON form
- *
- * @throws IOException mapping problems
- */
- public String asJson() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
- String json = mapper.writeValueAsString(entries);
- return json;
- }
-
- /**
- * This makes a copy without the nested content -so is suitable for returning as part of the list of a parent's
- * values
- *
- * @return the copy
- */
- public PublishedExports shallowCopy() {
- PublishedExports that = new PublishedExports();
- that.description = this.description;
- that.updated = this.updated;
- that.updatedTime = this.updatedTime;
- return that;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("PublishedConfiguration{");
- sb.append("description='").append(description).append('\'');
- sb.append(" entries = ").append(entries.size());
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java
deleted file mode 100644
index 67cb094..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.docstore;
-
-import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-
-/** Output a published configuration */
-public abstract class PublishedExportsOutputter {
-
- protected final PublishedExports exports;
-
- protected PublishedExportsOutputter(PublishedExports exports) {
- this.exports = exports;
- }
-
- /**
- * Create an outputter for the chosen format
- *
- * @param format format enumeration
- * @param exports owning config
- * @return the outputter
- */
-
- public static PublishedExportsOutputter createOutputter(ConfigFormat format,
- PublishedExports exports) {
- Preconditions.checkNotNull(exports);
- switch (format) {
- case JSON:
- return new JsonOutputter(exports);
- default:
- throw new RuntimeException("Unsupported format :" + format);
- }
- }
-
- public void save(File dest) throws IOException {
- FileOutputStream out = null;
- try {
- out = new FileOutputStream(dest);
- save(out);
- out.close();
- } finally {
- org.apache.hadoop.io.IOUtils.closeStream(out);
- }
- }
-
- /**
- * Save the content. The default saves the asString() value to the output stream
- *
- * @param out output stream
- * @throws IOException
- */
- public void save(OutputStream out) throws IOException {
- IOUtils.write(asString(), out, Charsets.UTF_8);
- }
-
- /**
- * Convert to a string
- *
- * @return the string form
- * @throws IOException
- */
- public abstract String asString() throws IOException;
-
- public static class JsonOutputter extends PublishedExportsOutputter {
-
- public JsonOutputter(PublishedExports exports) {
- super(exports);
- }
-
- @Override
- public void save(File dest) throws IOException {
- FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
- }
-
- @Override
- public String asString() throws IOException {
- return exports.asJson();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
deleted file mode 100644
index 339d3d6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.docstore;
-
-import org.apache.slider.server.appmaster.web.rest.RestPaths;
-import org.apache.slider.server.services.utility.PatternValidator;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-/**
- * Represents a set of configurations for an application, component, etc.
- * Json serialisable; accessors are synchronized
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class PublishedExportsSet {
-
- private static final PatternValidator validator = new PatternValidator(
- RestPaths.PUBLISHED_CONFIGURATION_REGEXP);
-
- public Map<String, PublishedExports> exports = new HashMap<>();
-
- public PublishedExportsSet() {
- }
-
- /**
- * Put a name -it will be converted to lower case before insertion.
- * Any existing entry will be overwritten (that includes an entry
- * with a different case in the original name)
- * @param name name of entry
- * @param export published export
- * @throws IllegalArgumentException if not a valid name
- */
- public void put(String name, PublishedExports export) {
- String name1 = name.toLowerCase(Locale.ENGLISH);
- validateName(name1);
- exports.put(name1, export);
- }
-
- /**
- * Validate the name -restricting it to the set defined in
- * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP}
- * @param name name to validate
- * @throws IllegalArgumentException if not a valid name
- */
- public static void validateName(String name) {
- validator.validate(name);
-
- }
-
- public PublishedExports get(String name) {
- return exports.get(name);
- }
-
- public boolean contains(String name) {
- return exports.containsKey(name);
- }
-
- public int size() {
- return exports.size();
- }
-
- public Set<String> keys() {
- TreeSet<String> keys = new TreeSet<>();
- keys.addAll(exports.keySet());
- return keys;
- }
-
- public PublishedExportsSet shallowCopy() {
- PublishedExportsSet that = new PublishedExportsSet();
- for (Map.Entry<String, PublishedExports> entry : exports.entrySet()) {
- that.put(entry.getKey(), entry.getValue().shallowCopy());
- }
- return that;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
deleted file mode 100644
index 120966f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.docstore;
-
-import org.codehaus.jackson.annotate.JsonIgnore;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.HashMap;
-import java.util.Map;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class UriMap {
-
- public Map<String, String> uris = new HashMap<>();
-
- @JsonIgnore
- public void put(String key, String value) {
- uris.put(key, value);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
deleted file mode 100644
index 13ad5c5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.info;
-
-/**
- * These are constants unique to the Slider AM
- */
-public class CustomRegistryConstants {
-
- public static final String MANAGEMENT_REST_API =
- "classpath:org.apache.slider.management";
-
- public static final String REGISTRY_REST_API =
- "classpath:org.apache.slider.registry";
-
- public static final String PUBLISHER_REST_API =
- "classpath:org.apache.slider.publisher";
-
- public static final String PUBLISHER_CONFIGURATIONS_API =
- "classpath:org.apache.slider.publisher.configurations";
-
- public static final String PUBLISHER_EXPORTS_API =
- "classpath:org.apache.slider.publisher.exports";
-
- public static final String PUBLISHER_DOCUMENTS_API =
- "classpath:org.apache.slider.publisher.documents";
-
- public static final String AGENT_SECURE_REST_API =
- "classpath:org.apache.slider.agents.secure";
-
- public static final String AGENT_ONEWAY_REST_API =
- "classpath:org.apache.slider.agents.oneway";
-
- public static final String AM_IPC_PROTOCOL =
- "classpath:org.apache.slider.appmaster.ipc";
-
- public static final String AM_REST_BASE =
- "classpath:org.apache.slider.client.rest";
-
- public static final String WEB_UI = "http://";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
deleted file mode 100644
index e204178..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.retrieve;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.GenericType;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.api.json.JSONConfiguration;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.ssl.SSLFactory;
-import org.apache.slider.client.rest.BaseRestClient;
-import org.apache.slider.core.restclient.HttpVerb;
-import org.apache.slider.core.restclient.UgiJerseyBinding;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLSocketFactory;
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.MediaType;
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URI;
-import java.net.URL;
-
-/**
- * Class to retrieve artifacts from the AM's web site. This sets up
- * the redirection and security logic properly
- */
-public class AMWebClient {
-
-
- private final BaseRestClient restClient;
- private static final Logger
- log = LoggerFactory.getLogger(AMWebClient.class);
-
-
- public AMWebClient(Configuration conf) {
- UgiJerseyBinding binding = new UgiJerseyBinding(conf);
-
- restClient = new BaseRestClient(binding.createJerseyClient());
-
- }
- public WebResource resource(String url) {
- return restClient.resource(url);
- }
-
- /**
- * Execute the operation. Failures are raised as IOException subclasses
- * @param method method to execute
- * @param resource resource to work against
- * @param c class to build
- * @param <T> type expected
- * @return an instance of the type T
- * @throws IOException on any failure
- */
- public <T> T exec(HttpVerb method, WebResource resource, Class<T> c) throws IOException {
- return restClient.exec(method, resource, c);
- }
-
- /**
- * Execute the operation. Failures are raised as IOException subclasses
- * @param method method to execute
- * @param resource resource to work against
- * @param t type to work with
- * @param <T> type expected
- * @return an instance of the type T
- * @throws IOException on any failure
- */
- public <T> T exec(HttpVerb method, WebResource resource, GenericType<T> t)
- throws IOException {
- return restClient.exec(method, resource, t);
- }
-
- /**
- * Execute the GET operation. Failures are raised as IOException subclasses
- * @param resource resource to work against
- * @param c class to build
- * @param <T> type expected
- * @return an instance of the type T
- * @throws IOException on any failure
- */
- public <T> T get(WebResource resource, Class<T> c) throws IOException {
- return restClient.get(resource, c);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
deleted file mode 100644
index b0eddb8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.retrieve;
-
-import com.beust.jcommander.Strings;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.registry.client.exceptions.RegistryIOException;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import static org.apache.slider.client.ClientRegistryBinder.*;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.ExceptionConverter;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.docstore.PublishedExports;
-import org.apache.slider.core.registry.docstore.PublishedExportsSet;
-import static org.apache.slider.core.registry.info.CustomRegistryConstants.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
-/**
- * Registry retriever.
- * This hides the HTTP operations that take place to
- * get the actual content
- */
-public class RegistryRetriever extends AMWebClient {
- private static final Logger log = LoggerFactory.getLogger(RegistryRetriever.class);
-
- private final String externalConfigurationURL;
- private final String internalConfigurationURL;
- private final String externalExportsURL;
- private final String internalExportsURL;
-
- /**
- * Retrieve from a service by locating the
- * exported {@link CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API}
- * and working off it.
- *
- * @param conf configuration to work from
- * @param record service record
- * @throws RegistryIOException the address type of the endpoint does
- * not match that expected (i.e. not a list of URLs), missing endpoint...
- */
- public RegistryRetriever(Configuration conf, ServiceRecord record) throws RegistryIOException {
- super(conf);
- externalConfigurationURL = lookupRestAPI(record,
- PUBLISHER_CONFIGURATIONS_API, true);
- internalConfigurationURL = lookupRestAPI(record,
- PUBLISHER_CONFIGURATIONS_API, false);
- externalExportsURL = lookupRestAPI(record,
- PUBLISHER_EXPORTS_API, true);
- internalExportsURL = lookupRestAPI(record,
- PUBLISHER_EXPORTS_API, false);
- }
-
- /**
- * Does a bonded registry retriever have a configuration?
- * @param external flag to indicate that it is the external entries to fetch
- * @return true if there is a URL to the configurations defined
- */
- public boolean hasConfigurations(boolean external) {
- return !Strings.isStringEmpty(
- external ? externalConfigurationURL : internalConfigurationURL);
- }
-
- /**
- * Get the configurations of the registry
- * @param external flag to indicate that it is the external entries to fetch
- * @return the configuration sets
- */
- public PublishedConfigSet getConfigurations(boolean external) throws
- FileNotFoundException, IOException {
-
- String confURL = getConfigurationURL(external);
- WebResource webResource = resource(confURL);
- return get(webResource, PublishedConfigSet.class);
- }
-
- protected String getConfigurationURL(boolean external) throws FileNotFoundException {
- String confURL = external ? externalConfigurationURL: internalConfigurationURL;
- if (Strings.isStringEmpty(confURL)) {
- throw new FileNotFoundException("No configuration URL");
- }
- return confURL;
- }
-
- protected String getExportURL(boolean external) throws FileNotFoundException {
- String confURL = external ? externalExportsURL: internalExportsURL;
- if (Strings.isStringEmpty(confURL)) {
- throw new FileNotFoundException("No configuration URL");
- }
- return confURL;
- }
-
- /**
- * Get the configurations of the registry
- * @param external flag to indicate that it is the external entries to fetch
- * @return the configuration sets
- */
- public PublishedExportsSet getExports(boolean external) throws
- FileNotFoundException, IOException {
-
- String exportsUrl = getExportURL(external);
- WebResource webResource = resource(exportsUrl);
- return get(webResource, PublishedExportsSet.class);
- }
-
-
- /**
- * Get a complete configuration, with all values
- * @param configSet config set to ask for
- * @param name name of the configuration
- * @param external flag to indicate that it is an external configuration
- * @return the retrieved config
- * @throws IOException IO problems
- */
- public PublishedConfiguration retrieveConfiguration(PublishedConfigSet configSet,
- String name,
- boolean external) throws IOException {
- String confURL = getConfigurationURL(external);
- if (!configSet.contains(name)) {
- throw new FileNotFoundException("Unknown configuration " + name);
- }
- confURL = SliderUtils.appendToURL(confURL, name);
- WebResource webResource = resource(confURL);
- return get(webResource, PublishedConfiguration.class);
- }
-
- /**
- * Get a complete export, with all values
- * @param exportSet
- * @param name name of the configuration
- * @param external flag to indicate that it is an external configuration
- * @return the retrieved config
- * @throws IOException IO problems
- */
- public PublishedExports retrieveExports(PublishedExportsSet exportSet,
- String name,
- boolean external) throws IOException {
- if (!exportSet.contains(name)) {
- throw new FileNotFoundException("Unknown export " + name);
- }
- String exportsURL = getExportURL(external);
- exportsURL = SliderUtils.appendToURL(exportsURL, name);
- return get(resource(exportsURL), PublishedExports.class);
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("RegistryRetriever{");
- sb.append("externalConfigurationURL='")
- .append(externalConfigurationURL)
- .append('\'');
- sb.append(", internalConfigurationURL='")
- .append(internalConfigurationURL)
- .append('\'');
- sb.append(", externalExportsURL='").append(externalExportsURL).append('\'');
- sb.append(", internalExportsURL='").append(internalExportsURL).append('\'');
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java
deleted file mode 100644
index c040345..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.restclient;
-
-/**
- * Http verbs with details on what they support in terms of submit and
- * response bodies.
- * <p>
- * Those verbs which do support bodies in the response MAY NOT return it;
- * if the response code is 204 then the answer is "no body", but the operation
- * is considered a success.
- */
-public enum HttpVerb {
- GET("GET", false, true),
- POST("POST", true, true),
- PUT("PUT", true, true),
- DELETE("DELETE", false, true),
- HEAD("HEAD", false, false);
-
- private final String verb;
- private final boolean hasUploadBody;
- private final boolean hasResponseBody;
-
- HttpVerb(String verb, boolean hasUploadBody, boolean hasResponseBody) {
- this.verb = verb;
- this.hasUploadBody = hasUploadBody;
- this.hasResponseBody = hasResponseBody;
- }
-
- public String getVerb() {
- return verb;
- }
-
- public boolean hasUploadBody() {
- return hasUploadBody;
- }
-
- public boolean hasResponseBody() {
- return hasResponseBody;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java
deleted file mode 100644
index e453f52..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.restclient;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.web.KerberosUgiAuthenticator;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
-import org.apache.hadoop.security.ssl.SSLFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLSocketFactory;
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.net.URLConnection;
-import java.security.GeneralSecurityException;
-
-/**
- * Factory for URL connections; used behind the scenes in the Jersey integration.
- * <p>
- * Derived from the WebHDFS implementation.
- */
-public class SliderURLConnectionFactory {
- private static final Logger log =
- LoggerFactory.getLogger(SliderURLConnectionFactory.class);
-
- /**
- * Timeout for socket connects and reads
- */
- public final static int DEFAULT_SOCKET_TIMEOUT = 60 * 1000; // 1 minute
- private final ConnectionConfigurator connConfigurator;
-
- private static final ConnectionConfigurator DEFAULT_CONFIGURATOR = new BasicConfigurator();
-
- /**
- * Construct a new URLConnectionFactory based on the configuration. It will
- * try to load SSL certificates when it is specified.
- */
- public static SliderURLConnectionFactory newInstance(Configuration conf) {
- ConnectionConfigurator conn;
- try {
- conn = newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
- } catch (Exception e) {
- log.debug("Cannot load customized SSL configuration.", e);
- conn = DEFAULT_CONFIGURATOR;
- }
- return new SliderURLConnectionFactory(conn);
- }
-
- private SliderURLConnectionFactory(ConnectionConfigurator connConfigurator) {
- this.connConfigurator = connConfigurator;
- }
-
- /**
- * Create a new ConnectionConfigurator for SSL connections
- */
- private static ConnectionConfigurator newSslConnConfigurator(final int timeout,
- Configuration conf) throws IOException, GeneralSecurityException {
- final SSLFactory factory;
- final SSLSocketFactory sf;
- final HostnameVerifier hv;
-
- factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
- factory.init();
- sf = factory.createSSLSocketFactory();
- hv = factory.getHostnameVerifier();
-
- return new ConnectionConfigurator() {
- @Override
- public HttpURLConnection configure(HttpURLConnection conn)
- throws IOException {
- if (conn instanceof HttpsURLConnection) {
- HttpsURLConnection c = (HttpsURLConnection) conn;
- c.setSSLSocketFactory(sf);
- c.setHostnameVerifier(hv);
- }
- SliderURLConnectionFactory.setupConnection(conn, timeout);
- return conn;
- }
- };
- }
-
- /**
- * Opens a url with read and connect timeouts
- *
- * @param url
- * to open
- * @return URLConnection
- * @throws IOException
- */
- public URLConnection openConnection(URL url) throws IOException {
- try {
- return openConnection(url, false);
- } catch (AuthenticationException e) {
- // Unreachable
- return null;
- }
- }
-
- /**
- * Opens a url with read and connect timeouts
- *
- * @param url
- * URL to open
- * @param isSpnego
- * whether the url should be authenticated via SPNEGO
- * @return URLConnection
- * @throws IOException
- * @throws AuthenticationException
- */
- public URLConnection openConnection(URL url, boolean isSpnego)
- throws IOException, AuthenticationException {
- if (isSpnego) {
- log.debug("open AuthenticatedURL connection {}", url);
- UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
- final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
- return new AuthenticatedURL(new KerberosUgiAuthenticator(),
- connConfigurator).openConnection(url, authToken);
- } else {
- log.debug("open URL connection {}", url);
- URLConnection connection = url.openConnection();
- if (connection instanceof HttpURLConnection) {
- connConfigurator.configure((HttpURLConnection) connection);
- }
- return connection;
- }
- }
-
- /**
- * Sets connection parameters on the given URLConnection
- *
- * @param connection
- * URLConnection to set
- * @param socketTimeout
- * the connection and read timeout of the connection.
- */
- private static void setupConnection(URLConnection connection, int socketTimeout) {
- connection.setConnectTimeout(socketTimeout);
- connection.setReadTimeout(socketTimeout);
- connection.setUseCaches(false);
- if (connection instanceof HttpURLConnection) {
- ((HttpURLConnection) connection).setInstanceFollowRedirects(true);
- }
- }
-
- private static class BasicConfigurator implements ConnectionConfigurator {
- @Override
- public HttpURLConnection configure(HttpURLConnection conn)
- throws IOException {
- SliderURLConnectionFactory.setupConnection(conn, DEFAULT_SOCKET_TIMEOUT);
- return conn;
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[54/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
deleted file mode 100644
index c2866cf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
-import java.io.File;
-
-/**
- * Registry actions
- *
- * --instance {app name}, if a / is in it, refers underneath?
- * --dest {destfile}
- * --list : list instances of slider service
- * --listfiles
- */
-@Parameters(commandNames = {ACTION_REGISTRY},
- commandDescription = DESCRIBE_ACTION_REGISTRY)
-
-public class ActionRegistryArgs extends AbstractActionArgs {
-
- public static final String USAGE =
- "Usage: " + SliderActions.ACTION_REGISTRY
- + " ("
- + Arguments.ARG_LIST + "|"
- + Arguments.ARG_LISTCONF + "|"
- + Arguments.ARG_LISTEXP + "|"
- + Arguments.ARG_LISTFILES + "|"
- + Arguments.ARG_GETCONF + "|"
- + Arguments.ARG_GETEXP + "> "
- + Arguments.ARG_NAME + " <name> "
- + " )"
- + "[" + Arguments.ARG_VERBOSE + "] "
- + "[" + Arguments.ARG_USER + "] "
- + "[" + Arguments.ARG_OUTPUT + " <filename> ] "
- + "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] "
- + "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] "
- + System.getProperty("line.separator")
- + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
- ;
- public ActionRegistryArgs() {
- }
-
- public ActionRegistryArgs(String name) {
- this.name = name;
- }
-
- @Override
- public String getActionName() {
- return ACTION_REGISTRY;
- }
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-
- @Parameter(names = {ARG_LIST},
- description = "list services")
- public boolean list;
-
- @Parameter(names = {ARG_LISTCONF},
- description = "list configurations")
- public boolean listConf;
-
- @Parameter(names = {ARG_GETCONF},
- description = "get configuration")
- public String getConf;
-
- @Parameter(names = {ARG_LISTEXP},
- description = "list exports")
- public boolean listExports;
-
- @Parameter(names = {ARG_GETEXP},
- description = "get export")
- public String getExport;
-
- @Parameter(names = {ARG_LISTFILES},
- description = "list files")
- public String listFiles;
-
- @Parameter(names = {ARG_GETFILES},
- description = "get files")
- public String getFiles;
-
- //--format
- @Parameter(names = ARG_FORMAT,
- description = "Format for a response: <xml|json|properties>")
- public String format = ConfigFormat.XML.toString() ;
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
- description = "Output destination")
- public File out;
-
- @Parameter(names = {ARG_NAME},
- description = "name of an instance")
- public String name;
-
- @Parameter(names = {ARG_SERVICETYPE},
- description = "optional service type")
- public String serviceType = YarnServiceConstants.APP_TYPE;
-
- @Parameter(names = {ARG_VERBOSE},
- description = "verbose output")
- public boolean verbose;
-
- @Parameter(names = {ARG_INTERNAL},
- description = "fetch internal registry entries")
- public boolean internal;
-
- @Parameter(names = {ARG_USER},
- description = "the name of the user whose application is being resolved")
- public String user;
-
- /**
- * validate health of all the different operations
- * @throws BadCommandArgumentsException
- */
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
-
- //verify that at most one of the operations is set
- int gets = s(getConf) + s(getFiles) + s(getExport);
- int lists = s(list) + s(listConf) + s(listFiles) + s(listExports);
- int set = lists + gets;
- if (set > 1) {
- throw new UsageException(USAGE);
- }
-
- if (out != null && ( set == 0)) {
- throw new UsageException("output path"
- + " is only supported on 'get' operations: ");
- }
- if (!list && !is(name)) {
- throw new UsageException("Argument " + ARG_NAME
- +" missing: ");
-
- }
- }
-
- private int s(String arg) {
- return is(arg) ? 1 : 0;
- }
-
- private boolean is(String arg) {
- return arg != null;
- }
-
- private int s(boolean arg) {
- return arg ? 1 : 0;
- }
-
- private String ifdef(String arg, boolean val) {
- return val ? (arg + " "): "";
- }
-
- private String ifdef(String arg, String val) {
- if (is(val)) {
- return arg + " " + val + " ";
- } else {
- return "";
- }
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder(ACTION_REGISTRY);
- sb.append(' ');
- sb.append(ifdef(ARG_LIST, list));
- sb.append(ifdef(ARG_LISTCONF, listConf));
- sb.append(ifdef(ARG_LISTFILES, listFiles));
- sb.append(ifdef(ARG_GETCONF, getConf));
- sb.append(ifdef(ARG_GETFILES, getFiles));
-
- sb.append(ifdef(ARG_NAME, name));
- sb.append(ifdef(ARG_SERVICETYPE, serviceType));
-
-
- sb.append(ifdef(ARG_VERBOSE, verbose));
- sb.append(ifdef(ARG_INTERNAL, internal));
-
- if (out != null) {
- sb.append(ifdef(ARG_OUTPUT, out.toString()));
- }
- sb.append(ifdef(ARG_FORMAT, format));
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
deleted file mode 100644
index 65f0472..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import java.io.File;
-
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_RESOLVE;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
-
-/**
- * Resolve registry entries
- *
- * --path {path}
- * --out {destfile}
- * --verbose
- * --list
- */
-@Parameters(commandNames = {ACTION_RESOLVE},
- commandDescription = DESCRIBE_ACTION_REGISTRY)
-public class ActionResolveArgs extends AbstractActionArgs {
-
- public static final String USAGE =
- "Usage: " + SliderActions.ACTION_RESOLVE
- + " "
- + ARG_PATH + " <path> "
- + "[" + ARG_LIST + "] "
- + "[" + ARG_OUTPUT + " <filename> ] "
- + "[" + ARG_DESTDIR + " <directory> ] "
- ;
- public ActionResolveArgs() {
- }
-
- @Override
- public String getActionName() {
- return ACTION_RESOLVE;
- }
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-
- @Parameter(names = {ARG_LIST},
- description = "list services")
- public boolean list;
-
- @Parameter(names = {ARG_PATH},
- description = "resolve a path")
- public String path;
-
- @Parameter(names = {ARG_DESTDIR},
- description = "destination directory for operations")
- public File destdir;
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "dest file")
- public File out;
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder(ACTION_RESOLVE).append(" ");
- sb.append(ARG_PATH).append(" ").append(path).append(" ");
- if (list) {
- sb.append(ARG_LIST).append(" ");
- }
- if (destdir != null) {
- sb.append(ARG_DESTDIR).append(" ").append(destdir).append(" ");
- }
- if (out != null) {
- sb.append(ARG_OUTPUT).append(" ").append(out).append(" ");
- }
- return sb.toString();
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
- if (StringUtils.isEmpty(path)) {
- throw new BadCommandArgumentsException("Missing mandatory argument "
- + ARG_PATH);
- }
- if (list && out != null) {
- throw new BadCommandArgumentsException("Argument "
- + ARG_OUTPUT +
- " not supported for " + ARG_LIST);
- }
- if (out != null && destdir != null) {
- throw new BadCommandArgumentsException(
- ARG_OUTPUT + " and " + ARG_DESTDIR + " cannot be used together"
- );
- }
- }
-
- public String getPath() {
- return path;
- }
-
- public void setPath(String path) {
- this.path = path;
- }
-
- public boolean isList() {
- return list;
- }
-
- public void setList(boolean list) {
- this.list = list;
- }
-
- public File getDestdir() {
- return destdir;
- }
-
- public void setDestdir(File destdir) {
- this.destdir = destdir;
- }
-
- public File getOut() {
- return out;
- }
-
- public void setOut(File out) {
- this.out = out;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
deleted file mode 100644
index b03dc92..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_RESOURCE},
- commandDescription = SliderActions.DESCRIBE_ACTION_RESOURCE)
-
-public class ActionResourceArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_RESOURCE;
- }
-
- @Parameter(names = {ARG_INSTALL},
- description = "Install the resource(s)")
- public boolean install;
-
- @Parameter(names = {ARG_DELETE},
- description = "Delete the file")
- public boolean delete;
-
- @Parameter(names = {ARG_LIST},
- description = "List of installed files")
- public boolean list;
-
- @Parameter(names = {ARG_RESOURCE},
- description = "Name of the file or directory")
- public String resource;
-
- @Parameter(names = {ARG_DESTDIR},
- description = "The name of the folder in which to store the resources")
- public String folder;
-
- @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing resource(s)")
- public boolean overwrite = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 3;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
deleted file mode 100644
index 622e77d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_STATUS},
- commandDescription = SliderActions.DESCRIBE_ACTION_STATUS)
-
-public class ActionStatusArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_STATUS;
- }
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "Output file for the status information")
- public String output;
-
- @Parameter(names = {ARG_LIFETIME},
- description = "Lifetime of the application from the time of request")
- public boolean lifetime;
-
- public String getOutput() {
- return output;
- }
-
- public void setOutput(String output) {
- this.output = output;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
deleted file mode 100644
index 2b90479..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_START },
- commandDescription = SliderActions.DESCRIBE_ACTION_THAW)
-public class ActionThawArgs extends AbstractActionArgs implements
- WaitTimeAccessor,
- LaunchArgsAccessor {
-
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_START;
- }
-
- @Override
- public int getWaittime() {
- return launchArgs.getWaittime();
- }
-
- @ParametersDelegate
- LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
-
- @Parameter(names = {ARG_LIFETIME},
- description = "Life time of the application since application started at"
- + " running state")
- public long lifetime;
-
- @Override
- public String getRmAddress() {
- return launchArgs.getRmAddress();
- }
-
- @Override
- public void setWaittime(int waittime) {
- launchArgs.setWaittime(waittime);
- }
-
-
- @Override
- public File getOutputFile() {
- return launchArgs.getOutputFile();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
deleted file mode 100644
index cf48513..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_TOKENS},
- commandDescription = "save tokens to a file or list tokens in a file")
-public class ActionTokensArgs extends AbstractActionArgs {
-
- public static final String DUPLICATE_ARGS = "Only one of " +
- ARG_SOURCE + " and " + ARG_OUTPUT + " allowed";
-
- public static final String MISSING_KT_PROVIDER =
- "Both " + ARG_KEYTAB + " and " + ARG_PRINCIPAL
- + " must be provided";
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_TOKENS;
- }
-
- @Parameter(names = {ARG_OUTPUT},
- description = "File to write")
- public File output;
-
- @Parameter(names = {ARG_SOURCE},
- description = "source file")
- public File source;
-
- @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
- public File keytab;
-
- @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
- public String principal="";
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
- if (output != null && source != null) {
- throw new BadCommandArgumentsException(DUPLICATE_ARGS);
- }
-
- // this is actually a !xor
- if (keytab != null ^ !principal.isEmpty()) {
- throw new BadCommandArgumentsException(MISSING_KT_PROVIDER);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
deleted file mode 100644
index e310f45..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_UPDATE},
- commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE)
-
-public class ActionUpdateArgs extends AbstractClusterBuildingActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_UPDATE;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
deleted file mode 100644
index 00151f4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Static argument manipulation operations
- */
-public class ArgOps {
-
- private static final Logger
- log = LoggerFactory.getLogger(ArgOps.class);
-
- /**
- * create a 3-tuple
- */
- public static List<Object> triple(String msg, int min, int max) {
- List<Object> l = new ArrayList<>(3);
- l.add(msg);
- l.add(min);
- l.add(max);
- return l;
- }
-
- public static void applyFileSystemBinding(String filesystemBinding,
- Configuration conf) {
- if (filesystemBinding != null) {
- //filesystem argument was set -this overwrites any defaults in the
- //configuration
- FileSystem.setDefaultUri(conf, filesystemBinding);
- }
- }
-
- public static void splitPairs(Collection<String> pairs,
- Map<String, String> dest) {
- for (String prop : pairs) {
- String[] keyval = prop.split("=", 2);
- if (keyval.length == 2) {
- dest.put(keyval[0], keyval[1]);
- }
- }
- }
-
-
- public static void applyDefinitions(Map<String, String> definitionMap,
- Configuration conf) {
- for (Map.Entry<String, String> entry : definitionMap.entrySet()) {
- String key = entry.getKey();
- String val = entry.getValue();
- log.debug("configuration[{}]<=\"{}\"", key, val);
- conf.set(key, val, "command line");
- }
- }
-
- /**
- * Create a map from a tuple list like ['worker','2','master','1] into a map
- * ['worker':'2',"master":'1'];
- * Duplicate entries also trigger errors
- * @param description description for errors
- * @param list list to conver to tuples
- * @return the map of key value pairs -unordered.
- * @throws BadCommandArgumentsException odd #of arguments received
- */
- public static Map<String, String> convertTupleListToMap(String description,
- List<String> list) throws
- BadCommandArgumentsException {
- Map<String, String> results = new HashMap<>();
- if (list != null && !list.isEmpty()) {
- int size = list.size();
- if (size % 2 != 0) {
- //odd number of elements, not permitted
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_PARSE_FAILURE + description);
- }
- for (int count = 0; count < size; count += 2) {
- String key = list.get(count);
- String val = list.get(count + 1);
- if (results.get(key) != null) {
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_DUPLICATE_ENTRY + description
- + ": " + key);
- }
- results.put(key, val);
- }
- }
- return results;
- }
-
- /**
- * Create a map from a tuple list like
- * ['worker','heapsize','5G','master','heapsize','2M'] into a map
- * ['worker':'2',"master":'1'];
- * Duplicate entries also trigger errors
-
- * @throws BadCommandArgumentsException odd #of arguments received
- */
- public static Map<String, Map<String, String>> convertTripleListToMaps(String description,
- List<String> list) throws BadCommandArgumentsException {
-
- Map<String, Map<String, String>> results = new HashMap<>();
- if (list != null && !list.isEmpty()) {
- int size = list.size();
- if (size % 3 != 0) {
- //wrong number of elements, not permitted
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_PARSE_FAILURE + description);
- }
- for (int count = 0; count < size; count += 3) {
- String role = list.get(count);
- String key = list.get(count + 1);
- String val = list.get(count + 2);
- Map<String, String> roleMap = results.get(role);
- if (roleMap == null) {
- //demand create new role map
- roleMap = new HashMap<>();
- results.put(role, roleMap);
- }
- if (roleMap.get(key) != null) {
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_DUPLICATE_ENTRY + description
- + ": for key " + key + " under " + role);
- }
- roleMap.put(key, val);
- }
- }
- return results;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
deleted file mode 100644
index 204149b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-/**
- * Here are all the arguments that may be parsed by the client or server
- * command lines.
- *
- * Important: Please keep the main list in alphabetical order
- * so it is easier to see what arguments are there
- */
-public interface Arguments {
-
- String ARG_APPDEF = "--appdef";
- String ARG_BASE_PATH = "--basepath";
- String ARG_COMPONENT = "--component";
- String ARG_COMPONENT_SHORT = "--comp";
- String ARG_COMPONENTS = "--components";
- String ARG_COMP_OPT= "--compopt";
- String ARG_COMP_OPT_SHORT = "--co";
- String ARG_CONFIG = "--config";
- String ARG_CONTAINERS = "--containers";
- String ARG_DEBUG = "--debug";
- String ARG_DEFINE = "-D";
- String ARG_DELETE = "--delete";
- String ARG_DEST = "--dest";
- String ARG_DESTDIR = "--destdir";
- String ARG_FILESYSTEM = "--fs";
- String ARG_FILESYSTEM_LONG = "--filesystem";
- String ARG_FOLDER = "--folder";
- String ARG_FORCE = "--force";
- String ARG_FORMAT = "--format";
- String ARG_GETCONF = "--getconf";
- String ARG_GETEXP = "--getexp";
- String ARG_GETFILES = "--getfiles";
- String ARG_HELP = "--help";
- String ARG_IMAGE = "--image";
- String ARG_INSTALL = "--install";
- String ARG_INTERNAL = "--internal";
- String ARG_KEYLEN = "--keylen";
- String ARG_KEYTAB = "--keytab";
- String ARG_KEYTABINSTALL = ARG_INSTALL;
- String ARG_KEYTABDELETE = ARG_DELETE;
- String ARG_KEYTABLIST = "--list";
- String ARG_LIST = "--list";
- String ARG_LISTCONF = "--listconf";
- String ARG_LISTEXP = "--listexp";
- String ARG_LISTFILES = "--listfiles";
- String ARG_LIVE = "--live";
- String ARG_MANAGER = "--manager";
- String ARG_MANAGER_SHORT = "--m";
- String ARG_MESSAGE = "--message";
- String ARG_NAME = "--name";
- String ARG_OPTION = "--option";
- String ARG_OPTION_SHORT = "-O";
- String ARG_OUTPUT = "--out";
- String ARG_OUTPUT_SHORT = "-o";
- String ARG_OVERWRITE = "--overwrite";
- String ARG_PACKAGE = "--package";
- String ARG_PATH = "--path";
- String ARG_PRINCIPAL = "--principal";
- String ARG_QUEUE = "--queue";
- String ARG_LIFETIME = "--lifetime";
- String ARG_RESOURCE = "--resource";
- String ARG_RESOURCE_MANAGER = "--rm";
- String ARG_SECURE = "--secure";
- String ARG_SERVICETYPE = "--servicetype";
- String ARG_SERVICES = "--services";
- String ARG_SOURCE = "--source";
- String ARG_STATE = "--state";
- String ARG_SYSPROP = "-S";
- String ARG_USER = "--user";
- String ARG_UPLOAD = "--upload";
- String ARG_VERBOSE = "--verbose";
- String ARG_VERSION = "--version";
- String ARG_WAIT = "--wait";
-/*
- STOP: DO NOT ADD YOUR ARGUMENTS HERE. GO BACK AND INSERT THEM IN THE
- RIGHT PLACE IN THE LIST
- */
-
- /**
- * server: URI for the cluster
- */
- String ARG_CLUSTER_URI = "-cluster-uri";
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
deleted file mode 100644
index 7b957fa..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-
-import java.util.Collection;
-
-/**
- * Client CLI Args
- */
-
-public class ClientArgs extends CommonArgs {
-
- // =========================================================
- // Keep all of these in alphabetical order. Thanks.
- // =========================================================
-
- private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
- private final ActionClientArgs actionClientArgs = new ActionClientArgs();
- private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
- private final ActionDependencyArgs actionDependencyArgs = new ActionDependencyArgs();
- private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs();
- private final ActionExistsArgs actionExistsArgs = new ActionExistsArgs();
- private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs();
- private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
- private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
- private final ActionKDiagArgs actionKDiagArgs = new ActionKDiagArgs();
- private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs();
- private final ActionListArgs actionListArgs = new ActionListArgs();
- private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
- private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
- private final ActionResourceArgs actionResourceArgs = new ActionResourceArgs();
- private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
- private final ActionThawArgs actionThawArgs = new ActionThawArgs();
- private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs();
- private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
-
- public ClientArgs(String[] args) {
- super(args);
- }
-
- public ClientArgs(Collection args) {
- super(args);
- }
-
- @Override
- protected void addActionArguments() {
-
- addActions(
- actionBuildArgs,
- actionCreateArgs,
- actionDependencyArgs,
- actionDestroyArgs,
- actionFlexArgs,
- actionFreezeArgs,
- actionHelpArgs,
- actionStatusArgs,
- actionThawArgs
- );
- }
-
- @Override
- public void applyDefinitions(Configuration conf) throws
- BadCommandArgumentsException {
- super.applyDefinitions(conf);
- //RM
- if (getManager() != null) {
- log.debug("Setting RM to {}", getManager());
- conf.set(YarnConfiguration.RM_ADDRESS, getManager());
- }
- if (getBasePath() != null) {
- log.debug("Setting basePath to {}", getBasePath());
- conf.set(YarnServiceConf.YARN_SERVICE_BASE_PATH,
- getBasePath().toString());
- }
- }
-
-
- public ActionBuildArgs getActionBuildArgs() {
- return actionBuildArgs;
- }
-
- public ActionUpdateArgs getActionUpdateArgs() {
- return actionUpdateArgs;
- }
-
- public ActionCreateArgs getActionCreateArgs() {
- return actionCreateArgs;
- }
-
- public ActionDependencyArgs getActionDependencyArgs() {
- return actionDependencyArgs;
- }
-
- public ActionDestroyArgs getActionDestroyArgs() {
- return actionDestroyArgs;
- }
-
- public ActionExistsArgs getActionExistsArgs() {
- return actionExistsArgs;
- }
-
- public ActionFlexArgs getActionFlexArgs() {
- return actionFlexArgs;
- }
-
- public ActionFreezeArgs getActionFreezeArgs() {
- return actionFreezeArgs;
- }
-
- public ActionListArgs getActionListArgs() {
- return actionListArgs;
- }
-
-
- public ActionRegistryArgs getActionRegistryArgs() {
- return actionRegistryArgs;
- }
-
- public ActionResolveArgs getActionResolveArgs() {
- return actionResolveArgs;
- }
-
- public ActionResourceArgs getActionResourceArgs() {
- return actionResourceArgs;
- }
-
- public ActionStatusArgs getActionStatusArgs() {
- return actionStatusArgs;
- }
-
- public ActionThawArgs getActionThawArgs() {
- return actionThawArgs;
- }
-
- public ActionTokensArgs getActionTokenArgs() {
- return actionTokenArgs;
- }
-
- /**
- * Look at the chosen action and bind it as the core action for the operation.
- * @throws SliderException bad argument or similar
- */
- @Override
- public void applyAction() throws SliderException {
- String action = getAction();
- if (SliderUtils.isUnset(action)) {
- action = ACTION_HELP;
- }
- switch (action) {
- case ACTION_BUILD:
- bindCoreAction(actionBuildArgs);
- break;
-
- case ACTION_CREATE:
- bindCoreAction(actionCreateArgs);
- break;
-
- case ACTION_STOP:
- bindCoreAction(actionFreezeArgs);
- break;
-
- case ACTION_START:
- bindCoreAction(actionThawArgs);
- break;
-
- case ACTION_DEPENDENCY:
- bindCoreAction(actionDependencyArgs);
- break;
-
- case ACTION_DESTROY:
- bindCoreAction(actionDestroyArgs);
- break;
-
- case ACTION_EXISTS:
- bindCoreAction(actionExistsArgs);
- break;
-
- case ACTION_FLEX:
- bindCoreAction(actionFlexArgs);
- break;
-
- case ACTION_HELP:
- bindCoreAction(actionHelpArgs);
- break;
-
- case ACTION_KDIAG:
- bindCoreAction(actionKDiagArgs);
- break;
-
- case ACTION_KEYTAB:
- bindCoreAction(actionKeytabArgs);
- break;
-
- case ACTION_LIST:
- bindCoreAction(actionListArgs);
- break;
-
- case ACTION_REGISTRY:
- bindCoreAction(actionRegistryArgs);
- break;
-
- case ACTION_RESOLVE:
- bindCoreAction(actionResolveArgs);
- break;
-
- case ACTION_RESOURCE:
- bindCoreAction(actionResourceArgs);
- break;
-
- case ACTION_STATUS:
- bindCoreAction(actionStatusArgs);
- break;
-
- case ACTION_TOKENS:
- bindCoreAction(actionTokenArgs);
- break;
-
- case ACTION_UPDATE:
- bindCoreAction(actionUpdateArgs);
- break;
-
- default:
- throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
- + " " + action);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
deleted file mode 100644
index d44412a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
+++ /dev/null
@@ -1,282 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterDescription;
-import com.beust.jcommander.ParameterException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * This class contains the common argument set for all tne entry points,
- * and the core parsing logic to verify that the action is on the list
- * of allowed actions -and that the remaining number of arguments is
- * in the range allowed
- */
-
-public abstract class CommonArgs extends ArgOps implements SliderActions,
- Arguments {
-
- protected static final Logger log = LoggerFactory.getLogger(CommonArgs.class);
-
-
- private static final int DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME = 30;
-
-
- @Parameter(names = ARG_HELP, help = true)
- public boolean help;
-
-
- /**
- -D name=value
-
- Define an HBase configuration option which overrides any options in
- the configuration XML files of the image or in the image configuration
- directory. The values will be persisted.
- Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
-
- */
-
- public Map<String, String> definitionMap = new HashMap<String, String>();
- /**
- * System properties
- */
- public Map<String, String> syspropsMap = new HashMap<String, String>();
-
-
- /**
- * fields
- */
- public final JCommander commander;
- private final String[] args;
-
- private AbstractActionArgs coreAction;
-
- /**
- * get the name: relies on arg 1 being the cluster name in all operations
- * @return the name argument, null if there is none
- */
- public String getClusterName() {
- return coreAction.getClusterName();
- }
-
- protected CommonArgs(String[] args) {
- this.args = args;
- commander = new JCommander(this);
- }
-
- protected CommonArgs(Collection args) {
- List<String> argsAsStrings = SliderUtils.collectionToStringList(args);
- this.args = argsAsStrings.toArray(new String[argsAsStrings.size()]);
- commander = new JCommander(this);
- }
-
- public String usage() {
- return usage(this, null);
- }
-
- public static String usage(CommonArgs serviceArgs, String commandOfInterest) {
- String result = null;
- StringBuilder helperMessage = new StringBuilder();
- if (commandOfInterest == null) {
- // JCommander.usage is too verbose for a command with many options like
- // slider no short version of that is found Instead, we compose our msg by
- helperMessage.append("\nUsage: service COMMAND [options]\n");
- helperMessage.append("where COMMAND is one of\n");
- for (String jcommand : serviceArgs.commander.getCommands().keySet()) {
- helperMessage.append(String.format("\t%-"
- + DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", jcommand,
- serviceArgs.commander.getCommandDescription(jcommand) + "\n"));
- }
- helperMessage
- .append("Most commands print help when invoked without parameters or with --help");
- result = helperMessage.toString();
- } else {
- helperMessage.append("\nUsage: service ").append(commandOfInterest);
- helperMessage.append(serviceArgs.coreAction.getMinParams() > 0 ? " <application>" : "");
- helperMessage.append("\n");
- for (ParameterDescription paramDesc : serviceArgs.commander.getCommands()
- .get(commandOfInterest).getParameters()) {
- String optional = paramDesc.getParameter().required() ? " (required)"
- : " (optional)";
- String paramName = paramDesc.getParameterized().getType() == Boolean.TYPE ? paramDesc
- .getLongestName() : paramDesc.getLongestName() + " <"
- + paramDesc.getParameterized().getName() + ">";
- helperMessage.append(String.format("\t%-"
- + DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", paramName,
- paramDesc.getDescription() + optional + "\n"));
- result = helperMessage.toString();
- }
- }
- return result;
- }
-
- public static String usage(CommonArgs serviceArgs) {
- return usage(serviceArgs, null);
- }
-
- /**
- * Parse routine -includes registering the action-specific argument classes
- * and postprocess it
- * @throws SliderException on any problem
- */
- public void parse() throws SliderException {
- addActionArguments();
- try {
- commander.parse(args);
- } catch (ParameterException e) {
- throw new BadCommandArgumentsException(e, "%s in %s",
- e.toString(),
- (args != null
- ? (SliderUtils.join(args,
- " ", false))
- : "[]"));
- }
- //now copy back to this class some of the attributes that are common to all
- //actions
- postProcess();
- }
-
- /**
- * Add a command
- * @param name action
- * @param arg value
- */
- protected void addAction(String name, Object arg) {
- commander.addCommand(name, arg);
- }
-
- protected void addActions(Object... actions) {
- for (Object action : actions) {
- commander.addCommand(action);
- }
- }
-
- /**
- * Override point to add a set of actions
- */
- protected void addActionArguments() {
-
- }
-
- /**
- * validate args via {@link #validate()}
- * then postprocess the arguments
- */
- public void postProcess() throws SliderException {
- applyAction();
- validate();
-
- //apply entry set
- for (Map.Entry<String, String> entry : syspropsMap.entrySet()) {
- System.setProperty(entry.getKey(), entry.getValue());
- }
- }
-
-
- /**
- * Implementors must implement their action apply routine here
- */
- public abstract void applyAction() throws SliderException;
-
-
- /**
- * Bind the core action; this extracts any attributes that are used
- * across routines
- * @param action action to bind
- */
- protected void bindCoreAction(AbstractActionArgs action) {
- coreAction = action;
-
- splitPairs(coreAction.definitions, definitionMap);
- splitPairs(coreAction.sysprops, syspropsMap);
- }
-
- /**
- * Validate the arguments against the action requested
- */
- public void validate() throws BadCommandArgumentsException, UsageException {
- if (coreAction == null) {
- throw new UsageException(ErrorStrings.ERROR_NO_ACTION + usage());
- }
- log.debug("action={}", getAction());
- // let the action validate itself
- try {
- coreAction.validate();
- } catch (BadCommandArgumentsException e) {
- String badArgMsgBuilder =
- e.getMessage() + System.lineSeparator() + usage(this,
- coreAction.getActionName());
- throw new BadCommandArgumentsException(badArgMsgBuilder);
- }
- }
-
- /**
- * Apply all the definitions on the command line to the configuration
- * @param conf config
- */
- public void applyDefinitions(Configuration conf) throws
- BadCommandArgumentsException {
- applyDefinitions(definitionMap, conf);
- }
-
-
- /**
- * If the Filesystem binding was provided, it overrides anything in
- * the configuration
- * @param conf configuration
- */
- public void applyFileSystemBinding(Configuration conf) {
- ArgOps.applyFileSystemBinding(getFilesystemBinding(), conf);
- }
-
- public boolean isDebug() {
- return coreAction.debug;
- }
-
-
- public String getFilesystemBinding() {
- return coreAction.filesystemBinding;
- }
-
- public Path getBasePath() { return coreAction.basePath; }
-
- public String getManager() {
- return coreAction.manager;
- }
-
- public String getAction() {
- return commander.getParsedCommand();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
deleted file mode 100644
index b6cd0a1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-public class ComponentArgsDelegate extends AbstractArgsDelegate {
-
- /**
- * This is a listing of the roles to create
- */
- @Parameter(names = {ARG_COMPONENT, ARG_COMPONENT_SHORT},
- arity = 2,
- description = "--component <name> <count> e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3",
- splitter = DontSplitArguments.class)
- public List<String> componentTuples = new ArrayList<>(0);
-
-
- /**
- * Get the role mapping (may be empty, but never null)
- * @return role mapping
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, String> getComponentMap() throws BadCommandArgumentsException {
- return convertTupleListToMap("component", componentTuples);
- }
-
- public List<String> getComponentTuples() {
- return componentTuples;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
deleted file mode 100644
index 85de615..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.converters.IParameterSplitter;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class DontSplitArguments implements IParameterSplitter {
-
- @Override
- public List<String> split(String value) {
- List<String> list = new ArrayList<>(1);
- list.add(value);
- return list;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
deleted file mode 100644
index bf194b6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import java.io.File;
-
-/**
- * Launch args for create and start and anything else that can start something
- */
-public interface LaunchArgsAccessor extends WaitTimeAccessor {
- String getRmAddress();
-
- File getOutputFile();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
deleted file mode 100644
index 157fb61..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-
-import java.io.File;
-
-/**
- * Any launch-time args
- */
-public class LaunchArgsDelegate extends WaitArgsDelegate implements
- LaunchArgsAccessor {
-
-
- //TODO: do we need this?
- @Parameter(names = ARG_RESOURCE_MANAGER,
- description = "Resource manager hostname:port ",
- required = false)
- private String rmAddress;
-
- @Override
- public String getRmAddress() {
- return rmAddress;
- }
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for any application report")
- public File outputFile;
-
- @Override
- public File getOutputFile() {
- return outputFile;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
deleted file mode 100644
index 7972716..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Delegate for application and resource options.
- */
-public class OptionArgsDelegate extends AbstractArgsDelegate {
-
- /**
- * Options key value.
- */
- @Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
- description = ARG_OPTION + "<name> <value>",
- splitter = DontSplitArguments.class)
- public List<String> optionTuples = new ArrayList<>(0);
-
-
- /**
- * All the app component option triples.
- */
- @Parameter(names = {ARG_COMP_OPT, ARG_COMP_OPT_SHORT}, arity = 3,
- description = "Component option " + ARG_COMP_OPT +
- " <component> <name> <option>",
- splitter = DontSplitArguments.class)
- public List<String> compOptTriples = new ArrayList<>(0);
-
- public Map<String, String> getOptionsMap() throws
- BadCommandArgumentsException {
- return convertTupleListToMap(ARG_OPTION, optionTuples);
- }
-
- /**
- * Get the role heap mapping (may be empty, but never null).
- * @return role heap mapping
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, Map<String, String>> getCompOptionMap()
- throws BadCommandArgumentsException {
- return convertTripleListToMaps(ARG_COMP_OPT, compOptTriples);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
deleted file mode 100644
index 040ac64..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.converters.BaseConverter;
-import org.apache.hadoop.fs.Path;
-
-public class PathArgumentConverter extends BaseConverter<Path> {
-
- public PathArgumentConverter(String optionName) {
- super(optionName);
- }
-
- @Override
- public Path convert(String value) {
- return new Path(value);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
deleted file mode 100644
index 1c38213..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.hadoop.yarn.service.client.params;
-
-/**
- * Parameters sent by the Client to the AM
- */
-public class SliderAMArgs extends CommonArgs {
-
- SliderAMCreateAction createAction = new SliderAMCreateAction();
-
- public SliderAMArgs(String[] args) {
- super(args);
- }
-
- @Override
- protected void addActionArguments() {
- addActions(createAction);
- }
-
- public String getImage() {
- return createAction.image;
- }
-
- /**
- * This is the URI in the FS to the Slider cluster; the conf file (and any
- * other cluster-specifics) can be picked up here
- */
- public String getAppDefPath() {
- return createAction.sliderClusterURI;
- }
-
- /**
- * Am binding is simple: there is only one action
- */
- @Override
- public void applyAction() {
- bindCoreAction(createAction);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
deleted file mode 100644
index a446665..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-
-import java.io.File;
-
-
-@Parameters(commandNames = { SliderActions.ACTION_CREATE},
- commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
-
-public class SliderAMCreateAction extends AbstractActionArgs implements
- LaunchArgsAccessor {
-
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_CREATE;
- }
-
- @Parameter(names = ARG_IMAGE, description = "image", required = false)
- public String image;
-
- /**
- * This is the URI in the FS to the Slider cluster; the conf file (and any
- * other cluster-specifics) can be picked up here
- */
- @Parameter(names = ARG_CLUSTER_URI,
- description = "URI to the Slider cluster", required = true)
- public String sliderClusterURI;
-
- @ParametersDelegate LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
-
- @Override
- public String getRmAddress() {
- return launchArgs.getRmAddress();
- }
-
- @Override
- public int getWaittime() {
- return launchArgs.getWaittime();
- }
-
- @Override
- public void setWaittime(int waittime) {
- launchArgs.setWaittime(waittime);
- }
-
- @Override
- public File getOutputFile() {
- return launchArgs.getOutputFile();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
deleted file mode 100644
index fc3c5a1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-/**
- * Actions.
- * Only some of these are supported by specific Slider Services; they
- * are listed here to ensure the names are consistent
- */
-public interface SliderActions {
- String ACTION_BUILD = "build";
- String ACTION_CLIENT = "client";
- String ACTION_CREATE = "create";
- String ACTION_DEPENDENCY = "dependency";
- String ACTION_UPDATE = "update";
- String ACTION_UPGRADE = "upgrade";
- String ACTION_DESTROY = "destroy";
- String ACTION_EXISTS = "exists";
- String ACTION_FLEX = "flex";
- String ACTION_STOP = "stop";
- String ACTION_HELP = "help";
- String ACTION_INSTALL_KEYTAB = "install-keytab";
- String ACTION_KDIAG = "kdiag";
- String ACTION_KEYTAB = "keytab";
- String ACTION_LIST = "list";
-
- String ACTION_REGISTRY = "registry";
- String ACTION_RESOLVE = "resolve";
- String ACTION_RESOURCE = "resource";
- String ACTION_STATUS = "status";
- String ACTION_START = "start";
- String ACTION_TOKENS = "tokens";
-
- String DESCRIBE_ACTION_BUILD =
- "Build a service specification, but do not start it";
- String DESCRIBE_ACTION_CREATE =
- "Build and start a service, it's equivalent to first invoke build and then start";
- String DESCRIBE_ACTION_DEPENDENCY =
- "Yarn service framework dependency (libraries) management";
- String DESCRIBE_ACTION_UPDATE =
- "Update template for service";
- String DESCRIBE_ACTION_UPGRADE =
- "Rolling upgrade/downgrade the component/containerto a newer/previous version";
- String DESCRIBE_ACTION_DESTROY =
- "Destroy a stopped service, service must be stopped first before destroying.";
- String DESCRIBE_ACTION_EXISTS =
- "Probe for an application running";
- String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
- String DESCRIBE_ACTION_FREEZE =
- "Stop a running service";
- String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems";
- String DESCRIBE_ACTION_HELP = "Print help information";
- String DESCRIBE_ACTION_LIST =
- "List running services";
- String DESCRIBE_ACTION_REGISTRY =
- "Query the registry of a service";
- String DESCRIBE_ACTION_STATUS =
- "Get the status of a service";
- String DESCRIBE_ACTION_THAW =
- "Start a service with pre-built specification or a previously stopped service";
- String DESCRIBE_ACTION_CLIENT = "Install the application client in the specified directory or obtain a client keystore or truststore";
- String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
- String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory";
-
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
deleted file mode 100644
index 86f3709..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-
-public class WaitArgsDelegate extends AbstractArgsDelegate implements
- WaitTimeAccessor {
-
-
- //--wait [timeout]
- @Parameter(names = {ARG_WAIT},
- description = "time to wait for an action to complete")
- public int waittime = 0;
-
-
- @Override
- public int getWaittime() {
- return waittime;
- }
-
- @Override
- public void setWaittime(int waittime) {
- this.waittime = waittime;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
deleted file mode 100644
index f6afae6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-public interface WaitTimeAccessor {
- int getWaittime();
- void setWaittime(int waittime);
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 0ed4860..1049698 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -22,6 +22,8 @@ import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.RetryNTimes;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -40,6 +42,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsReque
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -58,34 +61,32 @@ import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
import org.apache.hadoop.yarn.service.ClientAMProtocol;
import org.apache.hadoop.yarn.service.ServiceMaster;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
import org.apache.hadoop.yarn.service.client.params.Arguments;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
import org.apache.hadoop.yarn.service.client.params.CommonArgs;
import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
import org.apache.hadoop.yarn.service.provider.ProviderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.Times;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.exceptions.UsageException;
-import org.apache.slider.core.launch.ClasspathConstructor;
-import org.apache.slider.core.launch.JavaCommandLineBuilder;
-import org.apache.slider.core.registry.SliderRegistryUtils;
-import org.apache.slider.core.zk.ZKIntegration;
-import org.apache.slider.core.zk.ZookeeperUtils;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.containerlaunch.JavaCommandLineBuilder;
+import org.apache.hadoop.yarn.service.utils.ZookeeperUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -105,17 +106,21 @@ import java.util.concurrent.ConcurrentHashMap;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_CREATE;
import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_FLEX;
-import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
-import static org.apache.slider.common.tools.SliderUtils.*;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_QUEUE;
+import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
public class ServiceClient extends CompositeService
- implements SliderExitCodes, SliderKeys {
+ implements SliderExitCodes, YarnServiceConstants {
private static final Logger LOG =
LoggerFactory.getLogger(ServiceClient.class);
private SliderFileSystem fs;
private YarnClient yarnClient;
// Avoid looking up applicationId from fs all the time.
private Map<String, ApplicationId> cachedAppIds = new ConcurrentHashMap<>();
+ private Map<String, ClientAMProtocol> cachedAMProxies = new ConcurrentHashMap<>();
+
private RegistryOperations registryClient;
private CuratorFramework curatorClient;
private YarnRPC rpc;
@@ -293,7 +298,12 @@ public class ServiceClient extends CompositeService
ServiceApiUtil.jsonSerDeser
.save(fs.getFileSystem(), ServiceApiUtil.getAppJsonPath(fs, appName),
persistedApp, true);
- ClientAMProtocol proxy = connectToAM(appName);
+ ClientAMProtocol proxy = getAMProxy(appName);
+ if (proxy == null) {
+ String message = appName + " is not running";
+ LOG.error(message);
+ throw new YarnException(message);
+ }
proxy.flexComponents(requestBuilder.build());
for (Map.Entry<String, Long> entry : original.entrySet()) {
LOG.info("[COMPONENT {}]: number of containers changed from {} to {}",
@@ -303,9 +313,10 @@ public class ServiceClient extends CompositeService
return original;
}
- public int actionStop(String appName) throws YarnException, IOException {
+ public int actionStop(String appName, boolean waitForAppStopped)
+ throws YarnException, IOException {
validateClusterName(appName);
- getAppIdFromPersistedApp(appName);
+ getAppId(appName);
ApplicationId currentAppId = cachedAppIds.get(appName);
ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
if (terminatedStates.contains(report.getYarnApplicationState())) {
@@ -315,17 +326,29 @@ public class ServiceClient extends CompositeService
}
LOG.info("Stopping application {}, with appId = {}", appName, currentAppId);
try {
- // try to stop the app gracefully.
- ClientAMProtocol proxy = connectToAM(appName);
- StopRequestProto request = StopRequestProto.newBuilder().build();
- proxy.stop(request);
- LOG.info("Application " + appName + " is being gracefully stopped...");
+ ClientAMProtocol proxy = getAMProxy(appName, report);
+ cachedAppIds.remove(appName);
+ cachedAMProxies.remove(appName);
+ if (proxy != null) {
+ // try to stop the app gracefully.
+ StopRequestProto request = StopRequestProto.newBuilder().build();
+ proxy.stop(request);
+ LOG.info("Application " + appName + " is being gracefully stopped...");
+ } else {
+ yarnClient.killApplication(currentAppId,
+ appName + " is forcefully killed by user!");
+ LOG.info("Forcefully kill the application: " + appName);
+ return EXIT_SUCCESS;
+ }
+ if (!waitForAppStopped) {
+ return EXIT_SUCCESS;
+ }
// Wait until the app is killed.
long startTime = System.currentTimeMillis();
int pollCount = 0;
while (true) {
- Thread.sleep(1000);
+ Thread.sleep(2000);
report = yarnClient.getApplicationReport(currentAppId);
if (terminatedStates.contains(report.getYarnApplicationState())) {
LOG.info("Application " + appName + " is stopped.");
@@ -358,6 +381,7 @@ public class ServiceClient extends CompositeService
FileSystem fileSystem = fs.getFileSystem();
// remove from the appId cache
cachedAppIds.remove(appName);
+ cachedAMProxies.remove(appName);
if (fileSystem.exists(appDir)) {
if (fileSystem.delete(appDir, true)) {
LOG.info("Successfully deleted application dir for " + appName + ": "
@@ -370,7 +394,7 @@ public class ServiceClient extends CompositeService
}
}
deleteZKNode(appName);
- String registryPath = SliderRegistryUtils.registryPathForInstance(appName);
+ String registryPath = ServiceRegistryUtils.registryPathForInstance(appName);
try {
getRegistryClient().delete(registryPath, true);
} catch (IOException e) {
@@ -395,7 +419,7 @@ public class ServiceClient extends CompositeService
private void deleteZKNode(String clusterName) throws Exception {
CuratorFramework curatorFramework = getCuratorClient();
String user = RegistryUtils.currentUser();
- String zkPath = ZKIntegration.mkClusterPath(user, clusterName);
+ String zkPath = ServiceRegistryUtils.mkClusterPath(user, clusterName);
if (curatorFramework.checkExists().forPath(zkPath) != null) {
curatorFramework.delete().deletingChildrenIfNeeded().forPath(zkPath);
LOG.info("Deleted zookeeper path: " + zkPath);
@@ -418,7 +442,7 @@ public class ServiceClient extends CompositeService
if (curatorClient == null) {
curatorClient =
CuratorFrameworkFactory.builder().connectString(registryQuorum)
- .sessionTimeoutMs(10000).retryPolicy(new RetryNTimes(10, 2000))
+ .sessionTimeoutMs(10000).retryPolicy(new RetryNTimes(5, 2000))
.build();
curatorClient.start();
}
@@ -433,7 +457,7 @@ public class ServiceClient extends CompositeService
private void verifyNoLiveAppInRM(String appname, String action)
throws IOException, YarnException {
Set<String> types = new HashSet<>(1);
- types.add(SliderKeys.APP_TYPE);
+ types.add(YarnServiceConstants.APP_TYPE);
Set<String> tags = null;
if (appname != null) {
tags = Collections.singleton(SliderUtils.createNameTag(appname));
@@ -469,12 +493,13 @@ public class ServiceClient extends CompositeService
appTimeout.put(ApplicationTimeoutType.LIFETIME, app.getLifetime());
submissionContext.setApplicationTimeouts(appTimeout);
}
- submissionContext.setMaxAppAttempts(conf.getInt(KEY_AM_RESTART_LIMIT, 2));
+ submissionContext.setMaxAppAttempts(conf.getInt(
+ YarnServiceConf.AM_RESTART_MAX, 2));
Map<String, LocalResource> localResources = new HashMap<>();
// copy local slideram-log4j.properties to hdfs and add to localResources
- boolean hasSliderAMLog4j =
+ boolean hasAMLog4j =
addAMLog4jResource(appName, conf, localResources);
// copy jars to hdfs and add to localResources
addJarResource(appName, localResources);
@@ -487,17 +512,17 @@ public class ServiceClient extends CompositeService
// create AM CLI
String cmdStr =
- buildCommandLine(appName, conf, appRootDir, hasSliderAMLog4j);
+ buildCommandLine(appName, conf, appRootDir, hasAMLog4j);
submissionContext.setResource(Resource.newInstance(YarnServiceConf
- .getLong(KEY_AM_RESOURCE_MEM, DEFAULT_KEY_AM_RESOURCE_MEM,
+ .getLong(YarnServiceConf.AM_RESOURCE_MEM, YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM,
app.getConfiguration(), conf), 1));
String queue = app.getQueue();
if (StringUtils.isEmpty(queue)) {
- queue = conf.get(KEY_YARN_QUEUE, "default");
+ queue = conf.get(YARN_QUEUE, "default");
}
submissionContext.setQueue(queue);
submissionContext.setApplicationName(appName);
- submissionContext.setApplicationType(SliderKeys.APP_TYPE);
+ submissionContext.setApplicationType(YarnServiceConstants.APP_TYPE);
Set<String> appTags =
AbstractClientProvider.createApplicationTags(appName, null, null);
if (!appTags.isEmpty()) {
@@ -531,7 +556,7 @@ public class ServiceClient extends CompositeService
//TODO CLI.setJVMHeap
//TODO CLI.addJVMOPTS
if (hasSliderAMLog4j) {
- CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME);
+ CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, YARN_SERVICE_LOG4J_FILENAME);
CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
}
CLI.add(ServiceMaster.class.getCanonicalName());
@@ -553,15 +578,15 @@ public class ServiceClient extends CompositeService
private Map<String, String> addAMEnv(Configuration conf) throws IOException {
Map<String, String> env = new HashMap<>();
ClasspathConstructor classpath =
- buildClasspath(SliderKeys.SUBMITTED_CONF_DIR, "lib", fs, getConfig()
+ buildClasspath(YarnServiceConstants.SUBMITTED_CONF_DIR, "lib", fs, getConfig()
.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false));
env.put("CLASSPATH", classpath.buildClasspath());
env.put("LANG", "en_US.UTF-8");
env.put("LC_ALL", "en_US.UTF-8");
env.put("LANGUAGE", "en_US.UTF-8");
- String jaas = System.getenv(HADOOP_JAAS_DEBUG);
+ String jaas = System.getenv("HADOOP_JAAS_DEBUG");
if (jaas != null) {
- env.put(HADOOP_JAAS_DEBUG, jaas);
+ env.put("HADOOP_JAAS_DEBUG", jaas);
}
if (!UserGroupInformation.isSecurityEnabled()) {
String userName = UserGroupInformation.getCurrentUser().getUserName();
@@ -579,11 +604,11 @@ public class ServiceClient extends CompositeService
throws IOException, SliderException {
Path libPath = fs.buildClusterDirPath(appName);
ProviderUtils
- .addProviderJar(localResources, ServiceMaster.class, SLIDER_JAR, fs,
+ .addProviderJar(localResources, ServiceMaster.class, SERVICE_CORE_JAR, fs,
libPath, "lib", false);
Path dependencyLibTarGzip = fs.getDependencyTarGzip();
if (fs.isFile(dependencyLibTarGzip)) {
- LOG.info("Loading lib tar from " + fs.getFileSystem().getScheme() + ": "
+ LOG.info("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
+ dependencyLibTarGzip);
SliderUtils.putAmTarGzipAndUpdate(localResources, fs);
} else {
@@ -599,27 +624,29 @@ public class ServiceClient extends CompositeService
private boolean addAMLog4jResource(String appName, Configuration conf,
Map<String, LocalResource> localResources)
throws IOException, BadClusterStateException {
- boolean hasSliderAMLog4j = false;
+ boolean hasAMLog4j = false;
String hadoopConfDir =
System.getenv(ApplicationConstants.Environment.HADOOP_CONF_DIR.name());
if (hadoopConfDir != null) {
File localFile =
- new File(hadoopConfDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
+ new File(hadoopConfDir, YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
if (localFile.exists()) {
Path localFilePath = createLocalPath(localFile);
Path appDirPath = fs.buildClusterDirPath(appName);
Path remoteConfPath =
- new Path(appDirPath, SliderKeys.SUBMITTED_CONF_DIR);
+ new Path(appDirPath, YarnServiceConstants.SUBMITTED_CONF_DIR);
Path remoteFilePath =
- new Path(remoteConfPath, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
+ new Path(remoteConfPath, YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
copy(conf, localFilePath, remoteFilePath);
LocalResource localResource =
fs.createAmResource(remoteConfPath, LocalResourceType.FILE);
localResources.put(localFilePath.getName(), localResource);
- hasSliderAMLog4j = true;
+ hasAMLog4j = true;
+ } else {
+ LOG.warn("AM log4j property file doesn't exist: " + localFile);
}
}
- return hasSliderAMLog4j;
+ return hasAMLog4j;
}
public int actionStart(String appName) throws YarnException, IOException {
@@ -674,22 +701,22 @@ public class ServiceClient extends CompositeService
return;
}
String keytabPreInstalledOnHost =
- conf.get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
+ conf.get(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
if (StringUtils.isEmpty(keytabPreInstalledOnHost)) {
String amKeytabName =
- conf.get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
- String keytabDir = conf.get(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
+ conf.get(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
+ String keytabDir = conf.get(YarnServiceConf.KEY_HDFS_KEYTAB_DIR);
Path keytabPath =
fileSystem.buildKeytabPath(keytabDir, amKeytabName, appName);
if (fileSystem.getFileSystem().exists(keytabPath)) {
LocalResource keytabRes =
fileSystem.createAmResource(keytabPath, LocalResourceType.FILE);
localResource
- .put(SliderKeys.KEYTAB_DIR + "/" + amKeytabName, keytabRes);
+ .put(YarnServiceConstants.KEYTAB_DIR + "/" + amKeytabName, keytabRes);
LOG.info("Adding AM keytab on hdfs: " + keytabPath);
} else {
LOG.warn("No keytab file was found at {}.", keytabPath);
- if (conf.getBoolean(KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) {
+ if (conf.getBoolean(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) {
throw new BadConfigException("No keytab file was found at %s.",
keytabPath);
} else {
@@ -704,7 +731,7 @@ public class ServiceClient extends CompositeService
public String updateLifetime(String appName, long lifetime)
throws YarnException, IOException {
- getAppIdFromPersistedApp(appName);
+ getAppId(appName);
ApplicationId currentAppId = cachedAppIds.get(appName);
ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
if (report == null) {
@@ -729,11 +756,25 @@ public class ServiceClient extends CompositeService
public Application getStatus(String appName)
throws IOException, YarnException {
- ClientAMProtocol proxy = connectToAM(appName);
- GetStatusResponseProto response =
- proxy.getStatus(GetStatusRequestProto.newBuilder().build());
- return ServiceApiUtil.jsonSerDeser.fromJson(response.getStatus());
-
+ validateClusterName(appName);
+ ApplicationId currentAppId = getAppId(appName);
+ ApplicationReport appReport = yarnClient.getApplicationReport(currentAppId);
+ ClientAMProtocol amProxy = getAMProxy(appName, appReport);
+ Application appSpec;
+ if (amProxy != null) {
+ GetStatusResponseProto response =
+ amProxy.getStatus(GetStatusRequestProto.newBuilder().build());
+ appSpec = ServiceApiUtil.jsonSerDeser.fromJson(response.getStatus());
+ } else {
+ appSpec = new Application();
+ appSpec.setName(appName);
+ }
+ ApplicationTimeout lifetime =
+ appReport.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME);
+ if (lifetime != null) {
+ appSpec.setLifetime(lifetime.getRemainingTime());
+ }
+ return appSpec;
}
public YarnClient getYarnClient() {
@@ -760,71 +801,61 @@ public class ServiceClient extends CompositeService
String[] libDirs = SliderUtils.getLibDirs();
if (libDirs.length > 0) {
File tempLibTarGzipFile = File.createTempFile(
- SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME + "_",
- SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT);
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
// copy all jars
tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter());
- LOG.info("Uploading dependency for AM (version {}) from {} to {}",
- VersionInfo.getBuildVersion(), tempLibTarGzipFile.toURI(),
- dependencyLibTarGzip.toUri());
+ LOG.info("Version Info: " + VersionInfo.getBuildVersion());
fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip,
- new FsPermission(SliderKeys.SLIDER_DEPENDENCY_DIR_PERMISSIONS));
+ new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS));
return EXIT_SUCCESS;
} else {
return EXIT_FALSE;
}
}
- protected ClientAMProtocol connectToAM(String appName)
+ // Get AMProxy with the appReport provided
+ protected ClientAMProtocol getAMProxy(String appName, ApplicationReport report)
+ throws IOException {
+ if (!cachedAMProxies.containsKey(appName) && !StringUtils
+ .isEmpty(report.getHost())) {
+ insertAMProxy(appName, report.getHost(), report.getRpcPort());
+ }
+ return cachedAMProxies.get(appName);
+ }
+
+ // Get AMProxy without appReport provided - it'll getAppReport from RM
+ protected ClientAMProtocol getAMProxy(String appName)
throws IOException, YarnException {
- ApplicationId currentAppId = getAppIdFromPersistedApp(appName);
- // Wait until app becomes running.
- long startTime = System.currentTimeMillis();
- int pollCount = 0;
- ApplicationReport appReport = null;
- while (true) {
- appReport = yarnClient.getApplicationReport(currentAppId);
- YarnApplicationState state = appReport.getYarnApplicationState();
- if (state == RUNNING) {
- break;
- }
- if (terminatedStates.contains(state)) {
- throw new YarnException(
- "Failed to getStatus " + currentAppId + ": " + appReport
- .getDiagnostics());
- }
- long elapsedMillis = System.currentTimeMillis() - startTime;
- // if over 5 min, quit
- if (elapsedMillis >= 300000) {
- throw new YarnException(
- "Timed out while waiting for application " + currentAppId
- + " to be running");
- }
+ ApplicationId currentAppId = getAppId(appName);
- if (++pollCount % 10 == 0) {
- LOG.info(
- "Waiting for application {} to be running, current state is {}",
- currentAppId, state);
- }
- try {
- Thread.sleep(3000);
- } catch (InterruptedException ie) {
- String msg =
- "Interrupted while waiting for application " + currentAppId
- + " to be running.";
- throw new YarnException(msg, ie);
+ if (cachedAMProxies.containsKey(appName)) {
+ return cachedAMProxies.get(appName);
+ } else {
+ ApplicationReport appReport =
+ yarnClient.getApplicationReport(currentAppId);
+ String host = appReport.getHost();
+ int port = appReport.getRpcPort();
+ if (!StringUtils.isEmpty(host)) {
+ return insertAMProxy(appName, host, port);
}
+ return null;
}
+ }
- // Make the connection
- InetSocketAddress address = NetUtils
- .createSocketAddrForHost(appReport.getHost(), appReport.getRpcPort());
- return ClientAMProxy.createProxy(getConfig(), ClientAMProtocol.class,
+ private ClientAMProtocol insertAMProxy(String appName, String host, int port)
+ throws IOException {
+ InetSocketAddress address =
+ NetUtils.createSocketAddrForHost(host, port);
+ ClientAMProtocol amProxy =
+ ClientAMProxy.createProxy(getConfig(), ClientAMProtocol.class,
UserGroupInformation.getCurrentUser(), rpc, address);
+ cachedAMProxies.put(appName, amProxy);
+ return amProxy;
}
- private synchronized ApplicationId getAppIdFromPersistedApp(String appName)
+ private synchronized ApplicationId getAppId(String appName)
throws IOException, YarnException {
if (cachedAppIds.containsKey(appName)) {
return cachedAppIds.get(appName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
index 05c6501..ea3bb0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
@@ -20,10 +20,9 @@ package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import org.apache.hadoop.fs.Path;
-import org.apache.slider.common.params.PathArgumentConverter;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.UsageException;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,7 +48,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
public String filesystemBinding;
@Parameter(names = {ARG_BASE_PATH},
- description = "Slider base path on the filesystem",
+ description = "Service base path on the filesystem",
converter = PathArgumentConverter.class)
public Path basePath;
@@ -129,7 +128,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
if (minArgs > actionArgSize) {
throw new BadCommandArgumentsException(
ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName() +
- " Expected minimum " + minArgs + " but got " + actionArgSize);
+ ", Expected minimum " + minArgs + " but got " + actionArgSize);
}
int maxArgs = getMaxParams();
if (maxArgs == -1) {
@@ -146,7 +145,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
StringBuilder buf = new StringBuilder(message);
for (String actionArg : parameters) {
log.error("[{}] \"{}\"", index++, actionArg);
- buf.append(" \"" + actionArg + "\" ");
+ buf.append(" \"").append(actionArg).append("\" ");
}
throw new BadCommandArgumentsException(buf.toString());
}
@@ -156,25 +155,4 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
public String toString() {
return super.toString() + ": " + getActionName();
}
-
- /**
- * Override point:
- * Flag to indicate that core hadoop API services are needed (HDFS, YARN, etc)
- * —and that validation of the client state should take place.
- *
- * @return a flag to indicate that the core hadoop services will be needed.
- */
- public boolean getHadoopServicesRequired() {
- return true;
- }
-
- /**
- * Flag to disable secure login.
- * This MUST only be set if the action is bypassing security or setting
- * it itself
- * @return true if login at slider client init time is to be skipped
- */
- public boolean disableSecureLogin() {
- return false;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
new file mode 100644
index 0000000..457e357
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import org.apache.hadoop.yarn.service.client.params.ArgOps;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+
+/**
+ * Base class for all the delegates
+ */
+public class AbstractArgsDelegate extends ArgOps implements Arguments {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
new file mode 100644
index 0000000..017286f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParametersDelegate;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Abstract Action to build things; shares args across build and
+ * list
+ */
+public abstract class AbstractClusterBuildingActionArgs
+ extends AbstractActionArgs {
+ @Parameter(names = {ARG_APPDEF},
+ description = "Template application definition file in JSON format.")
+ public File appDef;
+
+ public File getAppDef() {
+ return appDef;
+ }
+
+ @Parameter(names = {
+ ARG_QUEUE }, description = "Queue to submit the application")
+ public String queue;
+
+ @Parameter(names = {
+ ARG_LIFETIME }, description = "Lifetime of the application from the time of request")
+ public long lifetime;
+
+ @ParametersDelegate
+ public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
+
+ @ParametersDelegate
+ public OptionArgsDelegate optionsDelegate =
+ new OptionArgsDelegate();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
index 28381cf..c2ff545 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
@Parameters(commandNames = { SliderActions.ACTION_BUILD},
commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
new file mode 100644
index 0000000..0097b4e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+import java.io.File;
+
+@Parameters(commandNames = { SliderActions.ACTION_CLIENT},
+ commandDescription = SliderActions.DESCRIBE_ACTION_CLIENT)
+
+public class ActionClientArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_CLIENT;
+ }
+
+ @Parameter(names = {ARG_INSTALL},
+ description = "Install client")
+ public boolean install;
+
+ @Parameter(names = {ARG_NAME},
+ description = "The name of the application")
+ public String name;
+
+ @Parameter(names = {ARG_PACKAGE},
+ description = "Path to app package")
+ public String packageURI;
+
+ @Parameter(names = {ARG_DEST},
+ description = "The location where to install the client")
+ public File installLocation;
+
+ @Parameter(names = {ARG_CONFIG},
+ description = "Client configuration")
+ public File clientConfig;
+
+ /**
+ * Get the min #of params expected
+ *
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
index 35cef5a..eecffb6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
@Parameters(commandNames = { SliderActions.ACTION_CREATE},
commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
index b41b2af..51e07c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
@@ -17,8 +17,8 @@
*/
package org.apache.hadoop.yarn.service.client.params;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.UsageException;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
new file mode 100644
index 0000000..ba3c5a9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+import java.io.File;
+
+@Parameters(commandNames = { SliderActions.ACTION_EXISTS},
+ commandDescription = SliderActions.DESCRIBE_ACTION_EXISTS)
+
+public class ActionExistsArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_EXISTS;
+ }
+
+ @Parameter(names = {ARG_LIVE},
+ description = "verify that the application is running")
+ public boolean live;
+
+ @Parameter(names = {ARG_STATE},
+ description = "verify that the application is in the specific YARN state")
+ public String state = "";
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for any application report")
+ public File out;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
index fcbb803..b7acf58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import java.util.List;
import java.util.Map;
@@ -47,9 +47,4 @@ public class ActionFlexArgs extends AbstractActionArgs {
BadCommandArgumentsException {
return componentDelegate.getComponentMap();
}
-
- public List<String> getComponentTuples() {
- return componentDelegate.getComponentTuples();
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
new file mode 100644
index 0000000..aecf0eb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+@Parameters(commandNames = { SliderActions.ACTION_STOP },
+ commandDescription = SliderActions.DESCRIBE_ACTION_FREEZE)
+
+public class ActionFreezeArgs extends AbstractActionArgs implements
+ WaitTimeAccessor {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_STOP;
+ }
+
+ public static final String FREEZE_COMMAND_ISSUED = "stop command issued";
+ @ParametersDelegate
+ public WaitArgsDelegate waitDelegate = new WaitArgsDelegate();
+
+ @Override
+ public int getWaittime() {
+ return waitDelegate.getWaittime();
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ waitDelegate.setWaittime(waittime);
+ }
+
+ @Parameter(names={ARG_MESSAGE},
+ description = "reason for the operation")
+ public String message = FREEZE_COMMAND_ISSUED;
+
+ @Parameter(names = {ARG_FORCE},
+ description = "force the operation")
+ public boolean force;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
new file mode 100644
index 0000000..51aa88a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+/**
+ * The Help command
+ */
+@Parameters(commandNames = { SliderActions.ACTION_HELP},
+ commandDescription = SliderActions.DESCRIBE_ACTION_HELP)
+public class ActionHelpArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_HELP;
+ }
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
new file mode 100644
index 0000000..061121e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+@Parameters(commandNames = { SliderActions.ACTION_KDIAG},
+ commandDescription = SliderActions.DESCRIBE_ACTION_KDIAG)
+
+public class ActionKDiagArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_KDIAG;
+ }
+
+ @Parameter(names = {ARG_SERVICES}, variableArity = true,
+ description =" list of services to check")
+ public List<String> services = new ArrayList<>();
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for report")
+ public File out;
+
+ @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
+ public File keytab;
+
+ @Parameter(names = {ARG_KEYLEN}, description = "minimum key length")
+ public int keylen = 256;
+
+ @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
+ public String principal;
+
+ @Parameter(names = {ARG_SECURE}, description = "Is security required")
+ public boolean secure = false;
+
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+ if (keytab != null && SliderUtils.isUnset(principal)) {
+ throw new UsageException("Missing argument " + ARG_PRINCIPAL);
+ }
+ if (keytab == null && SliderUtils.isSet(principal)) {
+ throw new UsageException("Missing argument " + ARG_KEYTAB);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
new file mode 100644
index 0000000..7e51457
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+@Parameters(commandNames = { SliderActions.ACTION_KEYTAB},
+ commandDescription = SliderActions.DESCRIBE_ACTION_KEYTAB)
+
+public class ActionKeytabArgs extends AbstractActionArgs {
+
+ public ActionKeytabArgs() {
+ super();
+ }
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_INSTALL_KEYTAB;
+ }
+
+ @Parameter(names = {ARG_KEYTABINSTALL},
+ description = "Install the keytab")
+ public boolean install;
+
+ @Parameter(names = {ARG_KEYTABDELETE},
+ description = "Delete the keytab")
+ public boolean delete;
+
+ @Parameter(names = {ARG_KEYTABLIST},
+ description = "List of installed keytabs")
+ public boolean list;
+
+ @Parameter(names = {ARG_KEYTAB},
+ description = "Path or name of the keytab")
+ public String keytab;
+
+ @Parameter(names = {ARG_FOLDER},
+ description = "The name of the folder in which to store the keytab")
+ public String folder;
+
+ @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing keytab")
+ public boolean overwrite = false;
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 3;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
new file mode 100644
index 0000000..005c172
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+@Parameters(commandNames = { SliderActions.ACTION_LIST},
+ commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
+
+public class ActionListArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_LIST;
+ }
+
+ @Parameter(names = {ARG_LIVE},
+ description = "List only live application instances")
+ public boolean live;
+
+ @Parameter(names = {ARG_STATE},
+ description = "list only applications in the specific YARN state")
+ public String state = "";
+
+ @Parameter(names = {ARG_VERBOSE},
+ description = "print out information in details")
+ public boolean verbose = false;
+
+ @Parameter(names = {ARG_CONTAINERS},
+ description = "List containers of an application instance")
+ public boolean containers;
+
+ @Parameter(names = {ARG_VERSION},
+ description = "Filter containers by app version (used with " +
+ ARG_CONTAINERS + ")")
+ public String version;
+
+ @Parameter(names = {ARG_COMPONENTS}, variableArity = true,
+ description = "Filter containers by component names (used with " +
+ ARG_CONTAINERS + ")")
+ public Set<String> components = new HashSet<>(0);
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
new file mode 100644
index 0000000..c2866cf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
+import java.io.File;
+
+/**
+ * Registry actions
+ *
+ * --instance {app name}, if a / is in it, refers underneath?
+ * --dest {destfile}
+ * --list : list instances of slider service
+ * --listfiles
+ */
+@Parameters(commandNames = {ACTION_REGISTRY},
+ commandDescription = DESCRIBE_ACTION_REGISTRY)
+
+public class ActionRegistryArgs extends AbstractActionArgs {
+
+ public static final String USAGE =
+ "Usage: " + SliderActions.ACTION_REGISTRY
+ + " ("
+ + Arguments.ARG_LIST + "|"
+ + Arguments.ARG_LISTCONF + "|"
+ + Arguments.ARG_LISTEXP + "|"
+ + Arguments.ARG_LISTFILES + "|"
+ + Arguments.ARG_GETCONF + "|"
+ + Arguments.ARG_GETEXP + "> "
+ + Arguments.ARG_NAME + " <name> "
+ + " )"
+ + "[" + Arguments.ARG_VERBOSE + "] "
+ + "[" + Arguments.ARG_USER + "] "
+ + "[" + Arguments.ARG_OUTPUT + " <filename> ] "
+ + "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] "
+ + "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] "
+ + System.getProperty("line.separator")
+ + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
+ ;
+ public ActionRegistryArgs() {
+ }
+
+ public ActionRegistryArgs(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String getActionName() {
+ return ACTION_REGISTRY;
+ }
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Parameter(names = {ARG_LIST},
+ description = "list services")
+ public boolean list;
+
+ @Parameter(names = {ARG_LISTCONF},
+ description = "list configurations")
+ public boolean listConf;
+
+ @Parameter(names = {ARG_GETCONF},
+ description = "get configuration")
+ public String getConf;
+
+ @Parameter(names = {ARG_LISTEXP},
+ description = "list exports")
+ public boolean listExports;
+
+ @Parameter(names = {ARG_GETEXP},
+ description = "get export")
+ public String getExport;
+
+ @Parameter(names = {ARG_LISTFILES},
+ description = "list files")
+ public String listFiles;
+
+ @Parameter(names = {ARG_GETFILES},
+ description = "get files")
+ public String getFiles;
+
+ //--format
+ @Parameter(names = ARG_FORMAT,
+ description = "Format for a response: <xml|json|properties>")
+ public String format = ConfigFormat.XML.toString() ;
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
+ description = "Output destination")
+ public File out;
+
+ @Parameter(names = {ARG_NAME},
+ description = "name of an instance")
+ public String name;
+
+ @Parameter(names = {ARG_SERVICETYPE},
+ description = "optional service type")
+ public String serviceType = YarnServiceConstants.APP_TYPE;
+
+ @Parameter(names = {ARG_VERBOSE},
+ description = "verbose output")
+ public boolean verbose;
+
+ @Parameter(names = {ARG_INTERNAL},
+ description = "fetch internal registry entries")
+ public boolean internal;
+
+ @Parameter(names = {ARG_USER},
+ description = "the name of the user whose application is being resolved")
+ public String user;
+
+ /**
+ * validate health of all the different operations
+ * @throws BadCommandArgumentsException
+ */
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+
+ //verify that at most one of the operations is set
+ int gets = s(getConf) + s(getFiles) + s(getExport);
+ int lists = s(list) + s(listConf) + s(listFiles) + s(listExports);
+ int set = lists + gets;
+ if (set > 1) {
+ throw new UsageException(USAGE);
+ }
+
+ if (out != null && ( set == 0)) {
+ throw new UsageException("output path"
+ + " is only supported on 'get' operations: ");
+ }
+ if (!list && !is(name)) {
+ throw new UsageException("Argument " + ARG_NAME
+ +" missing: ");
+
+ }
+ }
+
+ private int s(String arg) {
+ return is(arg) ? 1 : 0;
+ }
+
+ private boolean is(String arg) {
+ return arg != null;
+ }
+
+ private int s(boolean arg) {
+ return arg ? 1 : 0;
+ }
+
+ private String ifdef(String arg, boolean val) {
+ return val ? (arg + " "): "";
+ }
+
+ private String ifdef(String arg, String val) {
+ if (is(val)) {
+ return arg + " " + val + " ";
+ } else {
+ return "";
+ }
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder(ACTION_REGISTRY);
+ sb.append(' ');
+ sb.append(ifdef(ARG_LIST, list));
+ sb.append(ifdef(ARG_LISTCONF, listConf));
+ sb.append(ifdef(ARG_LISTFILES, listFiles));
+ sb.append(ifdef(ARG_GETCONF, getConf));
+ sb.append(ifdef(ARG_GETFILES, getFiles));
+
+ sb.append(ifdef(ARG_NAME, name));
+ sb.append(ifdef(ARG_SERVICETYPE, serviceType));
+
+
+ sb.append(ifdef(ARG_VERBOSE, verbose));
+ sb.append(ifdef(ARG_INTERNAL, internal));
+
+ if (out != null) {
+ sb.append(ifdef(ARG_OUTPUT, out.toString()));
+ }
+ sb.append(ifdef(ARG_FORMAT, format));
+
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
new file mode 100644
index 0000000..65f0472
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+
+import java.io.File;
+
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_RESOLVE;
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
+
+/**
+ * Resolve registry entries
+ *
+ * --path {path}
+ * --out {destfile}
+ * --verbose
+ * --list
+ */
+@Parameters(commandNames = {ACTION_RESOLVE},
+ commandDescription = DESCRIBE_ACTION_REGISTRY)
+public class ActionResolveArgs extends AbstractActionArgs {
+
+ public static final String USAGE =
+ "Usage: " + SliderActions.ACTION_RESOLVE
+ + " "
+ + ARG_PATH + " <path> "
+ + "[" + ARG_LIST + "] "
+ + "[" + ARG_OUTPUT + " <filename> ] "
+ + "[" + ARG_DESTDIR + " <directory> ] "
+ ;
+ public ActionResolveArgs() {
+ }
+
+ @Override
+ public String getActionName() {
+ return ACTION_RESOLVE;
+ }
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Parameter(names = {ARG_LIST},
+ description = "list services")
+ public boolean list;
+
+ @Parameter(names = {ARG_PATH},
+ description = "resolve a path")
+ public String path;
+
+ @Parameter(names = {ARG_DESTDIR},
+ description = "destination directory for operations")
+ public File destdir;
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "dest file")
+ public File out;
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder(ACTION_RESOLVE).append(" ");
+ sb.append(ARG_PATH).append(" ").append(path).append(" ");
+ if (list) {
+ sb.append(ARG_LIST).append(" ");
+ }
+ if (destdir != null) {
+ sb.append(ARG_DESTDIR).append(" ").append(destdir).append(" ");
+ }
+ if (out != null) {
+ sb.append(ARG_OUTPUT).append(" ").append(out).append(" ");
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+ if (StringUtils.isEmpty(path)) {
+ throw new BadCommandArgumentsException("Missing mandatory argument "
+ + ARG_PATH);
+ }
+ if (list && out != null) {
+ throw new BadCommandArgumentsException("Argument "
+ + ARG_OUTPUT +
+ " not supported for " + ARG_LIST);
+ }
+ if (out != null && destdir != null) {
+ throw new BadCommandArgumentsException(
+ ARG_OUTPUT + " and " + ARG_DESTDIR + " cannot be used together"
+ );
+ }
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ public boolean isList() {
+ return list;
+ }
+
+ public void setList(boolean list) {
+ this.list = list;
+ }
+
+ public File getDestdir() {
+ return destdir;
+ }
+
+ public void setDestdir(File destdir) {
+ this.destdir = destdir;
+ }
+
+ public File getOut() {
+ return out;
+ }
+
+ public void setOut(File out) {
+ this.out = out;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
new file mode 100644
index 0000000..b03dc92
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+@Parameters(commandNames = { SliderActions.ACTION_RESOURCE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_RESOURCE)
+
+public class ActionResourceArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_RESOURCE;
+ }
+
+ @Parameter(names = {ARG_INSTALL},
+ description = "Install the resource(s)")
+ public boolean install;
+
+ @Parameter(names = {ARG_DELETE},
+ description = "Delete the file")
+ public boolean delete;
+
+ @Parameter(names = {ARG_LIST},
+ description = "List of installed files")
+ public boolean list;
+
+ @Parameter(names = {ARG_RESOURCE},
+ description = "Name of the file or directory")
+ public String resource;
+
+ @Parameter(names = {ARG_DESTDIR},
+ description = "The name of the folder in which to store the resources")
+ public String folder;
+
+ @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing resource(s)")
+ public boolean overwrite = false;
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 3;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
new file mode 100644
index 0000000..622e77d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+@Parameters(commandNames = { SliderActions.ACTION_STATUS},
+ commandDescription = SliderActions.DESCRIBE_ACTION_STATUS)
+
+public class ActionStatusArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_STATUS;
+ }
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "Output file for the status information")
+ public String output;
+
+ @Parameter(names = {ARG_LIFETIME},
+ description = "Lifetime of the application from the time of request")
+ public boolean lifetime;
+
+ public String getOutput() {
+ return output;
+ }
+
+ public void setOutput(String output) {
+ this.output = output;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
new file mode 100644
index 0000000..2b90479
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+import java.io.File;
+
+@Parameters(commandNames = { SliderActions.ACTION_START },
+ commandDescription = SliderActions.DESCRIBE_ACTION_THAW)
+public class ActionThawArgs extends AbstractActionArgs implements
+ WaitTimeAccessor,
+ LaunchArgsAccessor {
+
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_START;
+ }
+
+ @Override
+ public int getWaittime() {
+ return launchArgs.getWaittime();
+ }
+
+ @ParametersDelegate
+ LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
+
+ @Parameter(names = {ARG_LIFETIME},
+ description = "Life time of the application since application started at"
+ + " running state")
+ public long lifetime;
+
+ @Override
+ public String getRmAddress() {
+ return launchArgs.getRmAddress();
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ launchArgs.setWaittime(waittime);
+ }
+
+
+ @Override
+ public File getOutputFile() {
+ return launchArgs.getOutputFile();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
new file mode 100644
index 0000000..cf48513
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+
+import java.io.File;
+
+@Parameters(commandNames = { SliderActions.ACTION_TOKENS},
+ commandDescription = "save tokens to a file or list tokens in a file")
+public class ActionTokensArgs extends AbstractActionArgs {
+
+ public static final String DUPLICATE_ARGS = "Only one of " +
+ ARG_SOURCE + " and " + ARG_OUTPUT + " allowed";
+
+ public static final String MISSING_KT_PROVIDER =
+ "Both " + ARG_KEYTAB + " and " + ARG_PRINCIPAL
+ + " must be provided";
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_TOKENS;
+ }
+
+ @Parameter(names = {ARG_OUTPUT},
+ description = "File to write")
+ public File output;
+
+ @Parameter(names = {ARG_SOURCE},
+ description = "source file")
+ public File source;
+
+ @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
+ public File keytab;
+
+ @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
+ public String principal="";
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+ if (output != null && source != null) {
+ throw new BadCommandArgumentsException(DUPLICATE_ARGS);
+ }
+
+ // this is actually a !xor
+ if (keytab != null ^ !principal.isEmpty()) {
+ throw new BadCommandArgumentsException(MISSING_KT_PROVIDER);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
deleted file mode 100644
index 864a1cf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-import com.codahale.metrics.JmxReporter;
-import com.codahale.metrics.Metric;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.MetricSet;
-import com.codahale.metrics.ScheduledReporter;
-import com.codahale.metrics.Slf4jReporter;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.slider.server.services.workflow.ClosingService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * YARN service which hooks up Codahale metrics to
- * JMX, and, if enabled Ganglia and/or an SLF4J log.
- */
-public class MetricsBindingService extends CompositeService
- implements MetricsKeys {
- protected static final Logger log =
- LoggerFactory.getLogger(MetricsBindingService.class);
- private final MetricRegistry metrics;
-
- private String reportingDetails = "not started";
-
-
- public MetricsBindingService(String name,
- MetricRegistry metrics) {
- super(name);
- Preconditions.checkArgument(metrics != null, "Null metrics");
- this.metrics = metrics;
- }
-
- /**
- * Instantiate...create a metric registry in the process
- * @param name service name
- */
- public MetricsBindingService(String name) {
- this(name, new MetricRegistry());
- }
-
- /**
- * Accessor for the metrics instance
- * @return the metrics
- */
- public MetricRegistry getMetrics() {
- return metrics;
- }
-
- @Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
-
- StringBuilder summary = new StringBuilder();
- Configuration conf = getConfig();
-
- summary.append("Reporting to JMX");
- // always start the JMX binding
- JmxReporter jmxReporter;
- jmxReporter = JmxReporter.forRegistry(metrics).build();
- jmxReporter.start();
- addService(new ClosingService<>(jmxReporter));
-
-
- // Ganglia
- if (conf.getBoolean(METRICS_GANGLIA_ENABLED, false)) {
- log.warn("Ganglia integration is not implemented");
-/*
- // This is all disabled due to transitive dependencies on an LGPL library
- com.codahale.metrics.ganglia.GangliaReporter gangliaReporter;
- String host = conf.getTrimmed(METRICS_GANGLIA_HOST, "");
- int port = conf.getInt(METRICS_GANGLIA_PORT, DEFAULT_GANGLIA_PORT);
- int interval = conf.getInt(METRICS_GANGLIA_REPORT_INTERVAL, 60);
- int ttl = 1;
- info.ganglia.gmetric4j.gmetric.GMetric.UDPAddressingMode
- mcast = info.ganglia.gmetric4j.gmetric.GMetric.UDPAddressingMode.getModeForAddress(host);
- boolean ganglia31 = conf.getBoolean(METRICS_GANGLIA_VERSION_31, true);
-
- final info.ganglia.gmetric4j.gmetric.GMetric ganglia =
- new info.ganglia.gmetric4j.gmetric.GMetric(
- host,
- port,
- mcast,
- ttl,
- ganglia31);
- gangliaReporter = com.codahale.metrics.ganglia.GangliaReporter.forRegistry(metrics)
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(
- TimeUnit.MILLISECONDS)
- .build(ganglia);
- gangliaReporter.start(interval, TimeUnit.SECONDS);
- addService(new ClosingService<ScheduledReporter>(gangliaReporter));
- summary.append(String.format(", Ganglia at %s:%d interval=%d",
- host, port, interval));
- */
- }
-
- // Logging
- if (conf.getBoolean(METRICS_LOGGING_ENABLED, false)) {
- ScheduledReporter reporter;
- String logName =
- conf.getTrimmed(METRICS_LOGGING_LOG, METRICS_DEFAULT_LOG);
- int interval = conf.getInt(METRICS_LOGGING_LOG_INTERVAL,
- METRICS_DEFAULT_LOG_INTERVAL);
- reporter = Slf4jReporter.forRegistry(metrics)
- .convertRatesTo(TimeUnit.SECONDS)
- .outputTo(LoggerFactory.getLogger(logName))
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .build();
- reporter.start(interval, TimeUnit.MINUTES);
- addService(new ClosingService<>(reporter));
- summary.append(String.format(", SLF4J to log %s interval=%d",
- logName, interval));
- }
- reportingDetails = summary.toString();
- log.info(reportingDetails);
- }
-
-
- @Override
- public String toString() {
- return super.toString() + " " + reportingDetails;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
deleted file mode 100644
index fa6bfc0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-/**
- * Constants used in slider for metrics registration and lookup
- */
-public class MetricsConstants {
-
- /**
- * {@value}
- */
- public static final String CONTAINERS_OUTSTANDING_REQUESTS = "containers.outstanding-requests";
-
- /**
- * {@value}
- */
- public static final String CONTAINERS_STARTED = "containers.started";
-
- /**
- * {@value}
- */
- public static final String CONTAINERS_SURPLUS = "containers.surplus";
-
- /**
- * {@value}
- */
- public static final String CONTAINERS_COMPLETED = "containers.completed";
-
- /**
- * {@value}
- */
- public static final String CONTAINERS_FAILED = "containers.failed";
-
- /**
- * {@value}
- */
- public static final String CONTAINERS_START_FAILED = "containers.start-failed";
-
- public static final String PREFIX_SLIDER_ROLES = "slider.roles.";
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsKeys.java
deleted file mode 100644
index 13b3b6b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsKeys.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-public interface MetricsKeys {
-
- /**
- * Prefix for metrics configuration options: {@value}
- */
- String METRICS_PREFIX = "slider.metrics.";
-
- /**
- * Boolean to enable Ganglia metrics reporting
- * {@value}
- */
- String METRICS_GANGLIA_ENABLED =
- METRICS_PREFIX + "ganglia.enabled";
- /**
- * {@value}
- */
- String METRICS_GANGLIA_HOST = METRICS_PREFIX + "ganglia.host";
- /**
- * {@value}
- */
- String METRICS_GANGLIA_PORT = METRICS_PREFIX + "ganglia.port";
- /**
- * {@value}
- */
- String METRICS_GANGLIA_VERSION_31 = METRICS_PREFIX + "ganglia.version-31";
- /**
- * {@value}
- */
- String METRICS_GANGLIA_REPORT_INTERVAL = METRICS_PREFIX + "ganglia.report.interval";
- /**
- * {@value}
- */
- int DEFAULT_GANGLIA_PORT = 8649;
-
-
- /**
- * Boolean to enable Logging metrics reporting
- * {@value}
- */
- String METRICS_LOGGING_ENABLED =
- METRICS_PREFIX + "logging.enabled";
-
- /**
- * String name of log to log to
- * {@value}
- */
- String METRICS_LOGGING_LOG =
- METRICS_PREFIX + "logging.log.name";
-
- /**
- * Default log name: {@value}
- */
- String METRICS_DEFAULT_LOG =
- "org.apache.slider.metrics.log";
-
-
- /**
- * Int log interval in seconds
- * {@value}
- */
- String METRICS_LOGGING_LOG_INTERVAL =
- METRICS_PREFIX + "logging.interval.minutes";
-
-
- /**
- * Default log interval: {@value}.
- * This is a big interval as in a long lived service, log overflows are easy
- * to create.
- */
- int METRICS_DEFAULT_LOG_INTERVAL = 60;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java
deleted file mode 100644
index e9ad46a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-import com.codahale.metrics.Metric;
-import com.codahale.metrics.MetricSet;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * From an existing metrics set, generate a new metrics set with the
- * prefix in front of every key.
- *
- * The prefix is added directly: if you want a '.' between prefix and metric
- * keys, include it in the prefix.
- */
-public class PrefixedMetricsSet implements MetricSet {
-
- private final String prefix;
- private final MetricSet source;
-
- public PrefixedMetricsSet(String prefix, MetricSet source) {
- this.prefix = prefix;
- this.source = source;
- }
-
- @Override
- public Map<String, Metric> getMetrics() {
- Map<String, Metric> sourceMetrics = source.getMetrics();
- Map<String, Metric> metrics = new HashMap<>(sourceMetrics.size());
- for (Map.Entry<String, Metric> entry : sourceMetrics.entrySet()) {
- metrics.put(prefix + "." + entry.getKey(), entry.getValue());
- }
- return metrics;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/Timestamp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/Timestamp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/Timestamp.java
deleted file mode 100644
index c30e749..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/Timestamp.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-/**
- * A timestamp metric
- */
-public class Timestamp extends LongGauge {
-
- public Timestamp(long val) {
- super(val);
- }
-
- public Timestamp() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/YarnServiceHealthCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/YarnServiceHealthCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/YarnServiceHealthCheck.java
deleted file mode 100644
index 936563c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/YarnServiceHealthCheck.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-import com.codahale.metrics.health.HealthCheck;
-import org.apache.hadoop.service.Service;
-
-public class YarnServiceHealthCheck extends HealthCheck {
-
- private final Service service;
-
- public YarnServiceHealthCheck(Service service) {
- this.service = service;
- }
-
- @Override
- protected Result check() throws Exception {
- return service.isInState(Service.STATE.STARTED)
- ? Result.healthy()
- : Result.unhealthy("Service is not running: %s", service);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
deleted file mode 100644
index a397e19..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.monkey;
-
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.MetricRegistry;
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
-import org.apache.slider.api.InternalKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Entry in the chaos list
- */
-public class ChaosEntry {
-
- protected static final Logger log =
- LoggerFactory.getLogger(ChaosEntry.class);
- public final String name;
- public final ChaosTarget target;
- public final long probability;
-
- private final Counter invocationCounter;
-
-
- /**
- * Constructor -includes validation of all arguments
- * @param name entry name
- * @param target target
- * @param probability probability of occurring
- */
- public ChaosEntry(String name, ChaosTarget target, long probability,
- MetricRegistry metrics) {
- Preconditions.checkArgument(!StringUtils.isEmpty(name), "missing name");
- Preconditions.checkArgument(target != null, "null target");
- Preconditions.checkArgument(probability > 0, "negative probability");
- Preconditions.checkArgument(probability <= InternalKeys.PROBABILITY_PERCENT_100,
- "probability over 100%: "+ probability);
- this.name = name;
- this.target = target;
- this.probability = probability;
- invocationCounter =
- metrics.counter(MetricRegistry.name(ChaosEntry.class, name));
- }
-
- /**
- * Trigger the chaos action
- */
- public void invokeChaos() {
- log.info("Invoking {}", name);
- invocationCounter.inc();
- target.chaosAction();
- }
-
- /**
- * Invoke Chaos if the trigger value is in range of the probability
- * @param value trigger value, 0-10K
- * @return true if the chaos method was invoked
- */
- public boolean maybeInvokeChaos(long value) {
- log.debug("Probability {} trigger={}", probability, value);
- if (value < probability) {
- invokeChaos();
- return true;
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
deleted file mode 100644
index 3c1a914..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.monkey;
-
-import org.apache.slider.server.appmaster.actions.ActionHalt;
-import org.apache.slider.server.appmaster.actions.QueueAccess;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Kill the AM
- */
-public class ChaosKillAM implements ChaosTarget {
-
- public static final int DELAY = 1000;
- private final QueueAccess queues;
- private final int exitCode;
-
- public ChaosKillAM(QueueAccess queues, int exitCode) {
- this.queues = queues;
- this.exitCode = exitCode;
- }
-
- /**
- * Trigger a delayed halt
- */
- @Override
- public void chaosAction() {
- queues.schedule(new ActionHalt(exitCode, "Chaos invoked halt", DELAY,
- TimeUnit.MILLISECONDS));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
deleted file mode 100644
index 022312c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.monkey;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.slider.server.appmaster.actions.ActionKillContainer;
-import org.apache.slider.server.appmaster.actions.QueueAccess;
-import org.apache.slider.server.appmaster.operations.RMOperationHandler;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Kill a container
- */
-public class ChaosKillContainer implements ChaosTarget {
- protected static final Logger log =
- LoggerFactory.getLogger(ChaosKillContainer.class);
- public static final int DELAY = 100;
- private final AppState appState;
- private final QueueAccess queues;
- private final Random random = new Random();
- private final RMOperationHandler operationHandler;
-
- public ChaosKillContainer(AppState appState,
- QueueAccess queues,
- RMOperationHandler operationHandler) {
- Preconditions.checkNotNull(appState);
- Preconditions.checkNotNull(queues);
- this.appState = appState;
- this.queues = queues;
- this.operationHandler = operationHandler;
- }
-
- /**
- * Trigger a container kill
- */
- @Override
- public void chaosAction() {
- List<RoleInstance> liveContainers =
- appState.cloneLiveContainerInfoList();
- // purge any and all components which declare that they are an AM
- ListIterator<RoleInstance> containers =
- liveContainers.listIterator();
- while (containers.hasNext()) {
- RoleInstance instance = containers.next();
- if (SliderKeys.COMPONENT_AM.equals(instance.role)) {
- containers.remove();
- }
- }
- int size = liveContainers.size();
- if (size > 0) {
- int target = random.nextInt(size);
- RoleInstance roleInstance = liveContainers.get(target);
- log.info("Killing {}", roleInstance);
-
- queues.schedule(new ActionKillContainer(roleInstance.getContainerId(),
- DELAY, TimeUnit.MILLISECONDS, operationHandler));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
deleted file mode 100644
index 8948f0d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.monkey;
-
-import com.codahale.metrics.MetricRegistry;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.server.appmaster.actions.QueueAccess;
-import org.apache.slider.server.appmaster.actions.RenewingAction;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
-/**
- * A chaos monkey service which will invoke ChaosTarget events
- */
-public class ChaosMonkeyService extends AbstractService {
- protected static final Logger log =
- LoggerFactory.getLogger(ChaosMonkeyService.class);
-
- private final MetricRegistry metrics;
- private final QueueAccess queues;
- private final Random random = new Random();
-
- private final List<ChaosEntry> chaosEntries =
- new ArrayList<ChaosEntry>();
-
- public ChaosMonkeyService(MetricRegistry metrics, QueueAccess queues) {
- super("ChaosMonkeyService");
- this.metrics = metrics;
- this.queues = queues;
- }
-
- /**
- * Add a target ... it is only added if <code>probability > 0</code>
- * @param name name
- * @param target chaos target
- * @param probability probability
- */
- public synchronized void addTarget(String name,
- ChaosTarget target, long probability) {
- if (probability > 0) {
- log.info("Adding {} with probability {}", name,
- ((double)probability) / InternalKeys.PROBABILITY_PERCENT_1);
- chaosEntries.add(new ChaosEntry(name, target, probability, metrics));
- } else {
- log.debug("Action {} not enabled", name);
- }
- }
-
- /**
- * Get the number of targets in the list
- * @return the count of added targets
- */
- public int getTargetCount() {
- return chaosEntries.size();
- }
-
- /**
- * Iterate through all the entries and invoke chaos on those wanted
- */
- public void play() {
- for (ChaosEntry chaosEntry : chaosEntries) {
- long p = randomPercentage();
- chaosEntry.maybeInvokeChaos(p);
- }
- }
-
- public int randomPercentage() {
- return random.nextInt(InternalKeys.PROBABILITY_PERCENT_100);
- }
-
- /**
- * Check for callers to see if chaos should be triggered; shares the
- * same random number source as the rest of the monkey entries
- * @param probability probability
- * @return true if the action should happen
- */
- public boolean chaosCheck(long probability) {
- return randomPercentage() < probability;
- }
-
- /**
- * Schedule the monkey
- *
- * @param delay initial delay
- * @param timeUnit time unit
- * @return true if it was scheduled (i.e. 1+ action) and interval > 0
- */
- public boolean schedule(long delay, long interval, TimeUnit timeUnit) {
- if (interval > 0 && !chaosEntries.isEmpty()) {
- queues.schedule(getChaosAction(delay, interval, timeUnit));
- return true;
- } else {
- return false;
- }
- }
-
- /**
- * Get the chaos action
- *
- * @param delay
- * @param timeUnit time unit
- * @return the action to schedule
- */
- public RenewingAction<MonkeyPlayAction> getChaosAction(long delay,
- long interval,
- TimeUnit timeUnit) {
- RenewingAction<MonkeyPlayAction> action = new RenewingAction<MonkeyPlayAction>(
- new MonkeyPlayAction(this, 0, TimeUnit.MILLISECONDS),
- delay,
- interval,
- timeUnit,
- 0
- );
- return action;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java
deleted file mode 100644
index 1c3a9ac..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.monkey;
-
-public interface ChaosTarget {
-
- public void chaosAction();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java
deleted file mode 100644
index 20e4466..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.monkey;
-
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.actions.AsyncAction;
-import org.apache.slider.server.appmaster.actions.QueueAccess;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Queueable action which calls {@link ChaosMonkeyService#play()} when
- * executed.
- */
-public class MonkeyPlayAction extends AsyncAction {
-
- private final ChaosMonkeyService monkey;
-
- public MonkeyPlayAction(ChaosMonkeyService monkey, long delay,
- TimeUnit timeUnit) {
- super("chaos monkey", delay, timeUnit);
- this.monkey = monkey;
- }
-
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
- monkey.play();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java
deleted file mode 100644
index ed3f197..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.operations;
-
-public abstract class AbstractRMOperation {
-
- /**
- * Execute the operation
- * @param asyncRMClient client
- * @param handler handler to perform the execution
- */
- public abstract void execute(RMOperationHandlerActions handler);
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
deleted file mode 100644
index 7173354..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.operations;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-import java.util.List;
-
-/**
- * Hands off RM operations to the Resource Manager.
- */
-public class AsyncRMOperationHandler extends RMOperationHandler {
- protected static final Logger log =
- LoggerFactory.getLogger(AsyncRMOperationHandler.class);
- private final AMRMClientAsync client;
- private final Resource maxResources;
-
- public AsyncRMOperationHandler(AMRMClientAsync client, Resource maxResources) {
- this.client = client;
- this.maxResources = maxResources;
- }
-
- @Override
- public int cancelContainerRequests(Priority priority1,
- Priority priority2,
- int count) {
- // need to revoke a previously issued container request
- // so enum the sets and pick some
- int remaining = cancelSinglePriorityRequests(priority1, count);
- if (priority2 != null) {
- remaining = cancelSinglePriorityRequests(priority2, remaining);
- }
-
- return remaining;
- }
-
- /**
- * Cancel just one of the priority levels
- * @param priority priority to cancel
- * @param count count to cancel
- * @return number of requests cancelled
- */
- @SuppressWarnings("unchecked")
- protected int cancelSinglePriorityRequests(Priority priority,
- int count) {
- List<Collection<AMRMClient.ContainerRequest>> requestSets =
- client.getMatchingRequests(priority, "", maxResources);
- if (count <= 0) {
- return 0;
- }
- int remaining = count;
- for (Collection<AMRMClient.ContainerRequest> requestSet : requestSets) {
- if (remaining == 0) {
- break;
- }
- for (AMRMClient.ContainerRequest request : requestSet) {
- if (remaining == 0) {
- break;
- }
- // a single release
- cancelSingleRequest(request);
- remaining --;
- }
- }
- return remaining;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void cancelSingleRequest(AMRMClient.ContainerRequest request) {
- // a single release
- client.removeContainerRequest(request);
- }
-
- @Override
- public void releaseAssignedContainer(ContainerId containerId) {
- log.debug("Releasing container {}", containerId);
-
- client.releaseAssignedContainer(containerId);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void addContainerRequest(AMRMClient.ContainerRequest req) {
- client.addContainerRequest(req);
- }
-
- @Override
- public void updateBlacklist(List<String> blacklistAdditions,
- List<String> blacklistRemovals) {
- client.updateBlacklist(blacklistAdditions, blacklistRemovals);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
deleted file mode 100644
index d7673d3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.operations;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.slider.server.appmaster.state.ContainerPriority;
-
-/**
- * Cancel a container request
- */
-public class CancelSingleRequest extends AbstractRMOperation {
-
- private final AMRMClient.ContainerRequest request;
-
- public CancelSingleRequest(AMRMClient.ContainerRequest request) {
- Preconditions.checkArgument(request != null, "Null container request");
- this.request = request;
- }
-
- @Override
- public void execute(RMOperationHandlerActions handler) {
- handler.cancelSingleRequest(request);
- }
-
- public AMRMClient.ContainerRequest getRequest() {
- return request;
- }
-
- @Override
- public String toString() {
- return "Cancel container request"
- + " for :" + ContainerPriority.toString(request.getPriority())
- + " request " + request;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java
deleted file mode 100644
index 4271d50..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.operations;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.server.appmaster.state.ContainerPriority;
-
-public class ContainerReleaseOperation extends AbstractRMOperation {
-
- private final ContainerId containerId;
-
- public ContainerReleaseOperation(ContainerId containerId) {
- Preconditions.checkArgument(containerId != null, "Null containerId");
- this.containerId = containerId;
- }
-
- public ContainerId getContainerId() {
- return containerId;
- }
-
- @Override
- public void execute(RMOperationHandlerActions handler) {
- handler.releaseAssignedContainer(containerId);
- }
-
- @Override
- public String toString() {
- return "release container " + containerId;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java
deleted file mode 100644
index e29ddd0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.operations;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.slider.server.appmaster.state.ContainerPriority;
-
-/**
- * A container request operation
- */
-public class ContainerRequestOperation extends AbstractRMOperation {
-
- private final AMRMClient.ContainerRequest request;
-
- public ContainerRequestOperation(AMRMClient.ContainerRequest request) {
- Preconditions.checkArgument(request != null, "Null container request");
- this.request = request;
- }
-
- public AMRMClient.ContainerRequest getRequest() {
- return request;
- }
-
- public Priority getPriority() {
- return request.getPriority();
- }
-
- public boolean getRelaxLocality() {
- return request.getRelaxLocality();
- }
-
- @Override
- public void execute(RMOperationHandlerActions handler) {
- handler.addContainerRequest(request);
- }
-
- @Override
- public String toString() {
- return "request container for role "
- + ContainerPriority.toString(getPriority())
- + " request " + request
- + " relaxLocality=" + getRelaxLocality();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java
deleted file mode 100644
index d0d038a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.operations;
-
-import java.util.List;
-
-public abstract class RMOperationHandler implements RMOperationHandlerActions {
-
- @Override
- public void execute(List<AbstractRMOperation> operations) {
- for (AbstractRMOperation operation : operations) {
- operation.execute(this);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
deleted file mode 100644
index bbaa933..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.operations;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-
-import java.util.List;
-
-public interface RMOperationHandlerActions {
-
- /**
- * Release an assigned container.
- * @param containerId container
- */
- void releaseAssignedContainer(ContainerId containerId);
-
- /**
- * Issue a container request.
- * @param request
- */
- void addContainerRequest(AMRMClient.ContainerRequest request);
-
- /**
- * Cancel a specific request.
- * @param request request to cancel
- */
- void cancelSingleRequest(AMRMClient.ContainerRequest request);
-
- /**
- * Remove a container request.
- * @param priority1 priority to remove at
- * @param priority2 second priority to target
- * @param count number to remove
- */
- int cancelContainerRequests(Priority priority1, Priority priority2, int count);
-
- /**
- * Blacklist resources.
- * @param blacklistAdditions resources to add to the blacklist
- * @param blacklistRemovals resources to remove from the blacklist
- */
- void updateBlacklist(List<String> blacklistAdditions,
- List<String> blacklistRemovals);
-
- /**
- * Execute an entire list of operations.
- * @param operations ops
- */
- void execute(List<AbstractRMOperation> operations);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/UpdateBlacklistOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/UpdateBlacklistOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/UpdateBlacklistOperation.java
deleted file mode 100644
index 90e2e5d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/UpdateBlacklistOperation.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.operations;
-
-import java.util.List;
-
-/**
- * Update blacklisted resources for the application.
- */
-public class UpdateBlacklistOperation extends AbstractRMOperation {
- private final List<String> blacklistAdditions;
- private final List<String> blacklistRemovals;
-
- public UpdateBlacklistOperation(List<String> blacklistAdditions,
- List<String> blacklistRemovals) {
- this.blacklistAdditions = blacklistAdditions;
- this.blacklistRemovals = blacklistRemovals;
- }
-
- @Override
- public void execute(RMOperationHandlerActions handler) {
- handler.updateBlacklist(blacklistAdditions, blacklistRemovals);
- }
-
- @Override
- public String toString() {
- return "blacklist additions: " + blacklistAdditions
- + ", blacklist removals: " + blacklistRemovals;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
deleted file mode 100644
index 2df9472..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
+++ /dev/null
@@ -1,307 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.rpc;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.BlockingService;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.ProtocolProxy;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RpcEngine;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.SecretManager;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.slider.api.SliderClusterProtocol;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.slider.common.tools.Duration;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.SliderException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.security.PrivilegedExceptionAction;
-
-public class RpcBinder {
- protected static final Logger log =
- LoggerFactory.getLogger(RpcBinder.class);
-
- /**
- * Create a protobuf server bonded to the specific socket address
- * @param addr address to listen to; 0.0.0.0 as hostname acceptable
- * @param conf config
- * @param secretManager token secret handler
- * @param numHandlers threads to service requests
- * @param blockingService service to handle
- * @param portRangeConfig range of ports
- * @return the IPC server itself
- * @throws IOException
- */
- public static Server createProtobufServer(InetSocketAddress addr,
- Configuration conf,
- SecretManager<? extends TokenIdentifier> secretManager,
- int numHandlers,
- BlockingService blockingService,
- String portRangeConfig) throws
- IOException {
- Class<SliderClusterProtocolPB> sliderClusterAPIClass = registerSliderAPI(
- conf);
- RPC.Server server = new RPC.Builder(conf).setProtocol(sliderClusterAPIClass)
- .setInstance(blockingService)
- .setBindAddress(addr.getAddress()
- .getCanonicalHostName())
- .setPort(addr.getPort())
- .setNumHandlers(numHandlers)
- .setVerbose(false)
- .setSecretManager(secretManager)
- .setPortRangeConfig(
- portRangeConfig)
- .build();
- log.debug(
- "Adding protocol " + sliderClusterAPIClass.getCanonicalName() + " to the server");
- server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, sliderClusterAPIClass,
- blockingService);
- return server;
- }
-
- /**
- * Add the protobuf engine to the configuration. Harmless and inexpensive
- * if repeated.
- * @param conf configuration to patch
- * @return the protocol class
- */
- public static Class<SliderClusterProtocolPB> registerSliderAPI(
- Configuration conf) {
- Class<SliderClusterProtocolPB> sliderClusterAPIClass =
- SliderClusterProtocolPB.class;
- RPC.setProtocolEngine(conf, sliderClusterAPIClass, ProtobufRpcEngine.class);
-
- //quick sanity check here
- assert verifyBondedToProtobuf(conf, sliderClusterAPIClass);
-
- return sliderClusterAPIClass;
- }
-
- /**
- * Verify that the conf is set up for protobuf transport of Slider RPC
- * @param conf configuration
- * @param sliderClusterAPIClass class for the API
- * @return true if the RPC engine is protocol buffers
- */
- public static boolean verifyBondedToProtobuf(Configuration conf,
- Class<SliderClusterProtocolPB> sliderClusterAPIClass) {
- return conf.getClass("rpc.engine." + sliderClusterAPIClass.getName(),
- RpcEngine.class) .equals(ProtobufRpcEngine.class);
- }
-
-
- /**
- * Connect to a server. May include setting up retry policies
- * @param addr
- * @param currentUser
- * @param conf
- * @param rpcTimeout
- * @return
- * @throws IOException
- */
- public static SliderClusterProtocol connectToServer(InetSocketAddress addr,
- UserGroupInformation currentUser,
- Configuration conf,
- int rpcTimeout) throws IOException {
- Class<SliderClusterProtocolPB> sliderClusterAPIClass =
- registerSliderAPI(conf);
-
- final RetryPolicy retryPolicy = RetryPolicies.TRY_ONCE_THEN_FAIL;
- log.debug("Connecting to Slider AM at {}", addr);
- ProtocolProxy<SliderClusterProtocolPB> protoProxy =
- RPC.getProtocolProxy(sliderClusterAPIClass,
- 1,
- addr,
- currentUser,
- conf,
- NetUtils.getDefaultSocketFactory(conf),
- rpcTimeout,
- retryPolicy);
- SliderClusterProtocolPB endpoint = protoProxy.getProxy();
- return new SliderClusterProtocolProxy(endpoint, addr);
- }
-
-
- /**
- * This loops for a limited period trying to get the Proxy -
- * by doing so it handles AM failover
- * @param conf configuration to patch and use
- * @param rmClient client of the resource manager
- * @param application application to work with
- * @param connectTimeout timeout for the whole proxy operation to timeout
- * (milliseconds). Use 0 to indicate "do not attempt to wait" -fail fast.
- * @param rpcTimeout timeout for RPCs to block during communications
- * @return the proxy
- * @throws IOException IO problems
- * @throws YarnException Slider-generated exceptions related to the binding
- * failing. This can include the application finishing or timeouts
- * @throws InterruptedException if a sleep operation waiting for
- * the cluster to respond is interrupted.
- */
- @SuppressWarnings("NestedAssignment")
- public static SliderClusterProtocol getProxy(final Configuration conf,
- final ApplicationClientProtocol rmClient,
- ApplicationReport application,
- final int connectTimeout,
- final int rpcTimeout)
- throws IOException, YarnException, InterruptedException {
- ApplicationId appId;
- appId = application.getApplicationId();
- Duration timeout = new Duration(connectTimeout);
- timeout.start();
- Exception exception = null;
- YarnApplicationState state = null;
- try {
- while (application != null &&
- (state = application.getYarnApplicationState()).equals(
- YarnApplicationState.RUNNING)) {
-
- try {
- return getProxy(conf, application, rpcTimeout);
- } catch (IOException e) {
- if (connectTimeout <= 0 || timeout.getLimitExceeded()) {
- throw e;
- }
- exception = e;
- } catch (YarnException e) {
- if (connectTimeout <= 0 || timeout.getLimitExceeded()) {
- throw e;
- }
- exception = e;
- }
- //at this point: app failed to work
- log.debug("Could not connect to {}. Waiting for getting the latest AM address...",
- appId);
- Thread.sleep(1000);
- //or get the app report
- application =
- rmClient.getApplicationReport(
- GetApplicationReportRequest.newInstance(appId)).getApplicationReport();
- }
- //get here if the app is no longer running. Raise a specific
- //exception but init it with the previous failure
- throw new BadClusterStateException(
- exception,
- ErrorStrings.E_FINISHED_APPLICATION, appId, state );
- } finally {
- timeout.close();
- }
- }
-
- /**
- * Get a proxy from the application report
- * @param conf config to use
- * @param application app report
- * @param rpcTimeout timeout in RPC operations
- * @return the proxy
- * @throws IOException
- * @throws SliderException
- * @throws InterruptedException
- */
- public static SliderClusterProtocol getProxy(final Configuration conf,
- final ApplicationReport application,
- final int rpcTimeout)
- throws IOException, SliderException, InterruptedException {
-
- String host = application.getHost();
- int port = application.getRpcPort();
- org.apache.hadoop.yarn.api.records.Token clientToAMToken =
- application.getClientToAMToken();
- return createProxy(conf, host, port, clientToAMToken, rpcTimeout);
- }
-
- /**
- *
- * @param conf config to use
- * @param host hosname
- * @param port port
- * @param clientToAMToken auth token: only used in a secure cluster.
- * converted via {@link ConverterUtils#convertFromYarn(org.apache.hadoop.yarn.api.records.Token, InetSocketAddress)}
- * @param rpcTimeout timeout in RPC operations
- * @return the proxy
- * @throws SliderException
- * @throws IOException
- * @throws InterruptedException
- */
- public static SliderClusterProtocol createProxy(final Configuration conf,
- String host,
- int port,
- org.apache.hadoop.yarn.api.records.Token clientToAMToken,
- final int rpcTimeout) throws
- SliderException,
- IOException,
- InterruptedException {
- String address = host + ":" + port;
- if (SliderUtils.isUnset(host) || 0 == port) {
- throw new SliderException(SliderExitCodes.EXIT_CONNECTIVITY_PROBLEM,
- "Slider instance "
- + " isn't providing a valid address for the" +
- " Slider RPC protocol: " + address);
- }
-
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
- final UserGroupInformation newUgi = UserGroupInformation.createRemoteUser(
- currentUser.getUserName());
- final InetSocketAddress serviceAddr =
- NetUtils.createSocketAddrForHost(host, port);
- SliderClusterProtocol realProxy;
-
- log.debug("Connecting to {}", serviceAddr);
- if (UserGroupInformation.isSecurityEnabled()) {
- Preconditions.checkArgument(clientToAMToken != null,
- "Null clientToAMToken");
- Token<ClientToAMTokenIdentifier> token =
- ConverterUtils.convertFromYarn(clientToAMToken, serviceAddr);
- newUgi.addToken(token);
- realProxy =
- newUgi.doAs(new PrivilegedExceptionAction<SliderClusterProtocol>() {
- @Override
- public SliderClusterProtocol run() throws IOException {
- return connectToServer(serviceAddr, newUgi, conf, rpcTimeout);
- }
- });
- } else {
- realProxy = connectToServer(serviceAddr, newUgi, conf, rpcTimeout);
- }
- return realProxy;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java
deleted file mode 100644
index 37c0a70..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.rpc;
-
-import org.apache.hadoop.security.authorize.PolicyProvider;
-import org.apache.hadoop.security.authorize.Service;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-
-/**
- * {@link PolicyProvider} for Slider protocols.
- */
-
-public class SliderAMPolicyProvider extends PolicyProvider {
-
- private static final Service[] services =
- new Service[] {
- new Service(SliderXmlConfKeys.KEY_PROTOCOL_ACL, SliderClusterProtocolPB.class)
- };
-
- @SuppressWarnings("ReturnOfCollectionOrArrayField")
- @Override
- public Service[] getServices() {
- return services;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPB.java
deleted file mode 100644
index 7d237de..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPB.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.rpc;
-
-import org.apache.slider.api.SliderClusterProtocol;
-import org.apache.slider.api.proto.SliderClusterAPI;
-
-public interface SliderClusterProtocolPB extends SliderClusterAPI.SliderClusterProtocolPB.BlockingInterface{
-
- long versionID = SliderClusterProtocol.versionID;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
deleted file mode 100644
index 526ab7c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.rpc;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.slider.api.SliderClusterProtocol;
-import org.apache.slider.api.proto.Messages;
-
-import java.io.IOException;
-
-/**
- * Server-side Relay from Protobuf to internal RPC.
- *
- */
-public class SliderClusterProtocolPBImpl implements SliderClusterProtocolPB {
-
- private SliderClusterProtocol real;
-
- public SliderClusterProtocolPBImpl(SliderClusterProtocol real) {
- this.real = real;
- }
-
- private ServiceException wrap(Exception e) {
- if (e instanceof ServiceException) {
- return (ServiceException) e;
- }
- return new ServiceException(e);
- }
-
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return SliderClusterProtocol.versionID;
- }
-
- @Override
- public Messages.StopClusterResponseProto stopCluster(RpcController controller,
- Messages.StopClusterRequestProto request) throws ServiceException {
- try {
- return real.stopCluster(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.UpgradeContainersResponseProto upgradeContainers(RpcController controller,
- Messages.UpgradeContainersRequestProto request) throws ServiceException {
- try {
- return real.upgradeContainers(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.FlexComponentsResponseProto flexComponents(
- RpcController controller, Messages.FlexComponentsRequestProto request)
- throws ServiceException {
- try {
- return real.flexComponents(request);
- } catch (IOException e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.GetJSONClusterStatusResponseProto getJSONClusterStatus(
- RpcController controller,
- Messages.GetJSONClusterStatusRequestProto request) throws ServiceException {
- try {
- return real.getJSONClusterStatus(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(
- RpcController controller,
- Messages.ListNodeUUIDsByRoleRequestProto request) throws ServiceException {
- try {
- return real.listNodeUUIDsByRole(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.GetNodeResponseProto getNode(RpcController controller,
- Messages.GetNodeRequestProto request) throws ServiceException {
- try {
- return real.getNode(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.GetClusterNodesResponseProto getClusterNodes(RpcController controller,
- Messages.GetClusterNodesRequestProto request) throws ServiceException {
- try {
- return real.getClusterNodes(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.EchoResponseProto echo(RpcController controller,
- Messages.EchoRequestProto request) throws ServiceException {
- try {
- return real.echo(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.KillContainerResponseProto killContainer(RpcController controller,
- Messages.KillContainerRequestProto request) throws ServiceException {
- try {
- return real.killContainer(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.AMSuicideResponseProto amSuicide(RpcController controller,
- Messages.AMSuicideRequestProto request) throws ServiceException {
- try {
- return real.amSuicide(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.ApplicationLivenessInformationProto getLivenessInformation(
- RpcController controller,
- Messages.GetApplicationLivenessRequestProto request) throws ServiceException {
- try {
- return real.getLivenessInformation(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.GetLiveContainersResponseProto getLiveContainers(RpcController controller,
- Messages.GetLiveContainersRequestProto request) throws ServiceException {
- try {
- return real.getLiveContainers(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.ContainerInformationProto getLiveContainer(RpcController controller,
- Messages.GetLiveContainerRequestProto request) throws ServiceException {
- try {
- return real.getLiveContainer(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.GetLiveComponentsResponseProto getLiveComponents(RpcController controller,
- Messages.GetLiveComponentsRequestProto request) throws ServiceException {
- try {
- return real.getLiveComponents(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.ComponentInformationProto getLiveComponent(RpcController controller,
- Messages.GetLiveComponentRequestProto request) throws ServiceException {
- try {
- return real.getLiveComponent(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.GetLiveNodesResponseProto getLiveNodes(RpcController controller,
- Messages.GetLiveNodesRequestProto request) throws ServiceException {
- try {
- return real.getLiveNodes(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-
- @Override
- public Messages.NodeInformationProto getLiveNode(RpcController controller,
- Messages.GetLiveNodeRequestProto request) throws ServiceException {
- try {
- return real.getLiveNode(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java
deleted file mode 100644
index 7b0586e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.UsageException;
-import org.apache.slider.utils.SliderTestBase;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-
-/**
- * Test the argument parsing/validation logic.
- */
-public class TestClientBadArgs extends SliderTestBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestClientBadArgs.class);
-
- //@Test
- public void testNoAction() throws Throwable {
- launchExpectingException(SliderClient.class,
- createTestConfig(),
- "Usage: slider COMMAND",
- EMPTY_LIST);
-
- }
-
- //@Test
- public void testUnknownAction() throws Throwable {
- launchExpectingException(SliderClient.class,
- createTestConfig(),
- "not-a-known-action",
- Arrays.asList("not-a-known-action"));
- }
-
- //@Test
- public void testActionWithoutOptions() throws Throwable {
- launchExpectingException(SliderClient.class,
- createTestConfig(),
- "Usage: slider build <application>",
- Arrays.asList(SliderActions.ACTION_BUILD));
- }
-
- //@Test
- public void testActionWithoutEnoughArgs() throws Throwable {
- launchExpectingException(SliderClient.class,
- createTestConfig(),
- ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS,
- Arrays.asList(SliderActions.ACTION_START));
- }
-
- //@Test
- public void testActionWithTooManyArgs() throws Throwable {
- launchExpectingException(SliderClient.class,
- createTestConfig(),
- ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
- Arrays.asList(SliderActions.ACTION_HELP,
- "hello, world"));
- }
-
- //@Test
- public void testBadImageArg() throws Throwable {
- launchExpectingException(SliderClient.class,
- createTestConfig(),
- "Unknown option: --image",
- Arrays.asList(SliderActions.ACTION_HELP,
- Arguments.ARG_IMAGE));
- }
-
- //@Test
- public void testRegistryUsage() throws Throwable {
- Throwable exception = launchExpectingException(SliderClient.class,
- createTestConfig(),
- "org.apache.slider.core.exceptions.UsageException: Argument --name " +
- "missing",
- Arrays.asList(SliderActions.ACTION_REGISTRY));
- assertTrue(exception instanceof UsageException);
- LOG.info(exception.toString());
- }
-
- //@Test
- public void testRegistryExportBadUsage1() throws Throwable {
- Throwable exception = launchExpectingException(SliderClient.class,
- createTestConfig(),
- "Expected a value after parameter --getexp",
- Arrays.asList(SliderActions.ACTION_REGISTRY,
- Arguments.ARG_NAME,
- "cl1",
- Arguments.ARG_GETEXP));
- assertTrue(exception instanceof BadCommandArgumentsException);
- LOG.info(exception.toString());
- }
-
- //@Test
- public void testRegistryExportBadUsage2() throws Throwable {
- Throwable exception = launchExpectingException(SliderClient.class,
- createTestConfig(),
- "Expected a value after parameter --getexp",
- Arrays.asList(SliderActions.ACTION_REGISTRY,
- Arguments.ARG_NAME,
- "cl1",
- Arguments.ARG_LISTEXP,
- Arguments.ARG_GETEXP));
- assertTrue(exception instanceof BadCommandArgumentsException);
- LOG.info(exception.toString());
- }
-
- //@Test
- public void testRegistryExportBadUsage3() throws Throwable {
- Throwable exception = launchExpectingException(SliderClient.class,
- createTestConfig(),
- "Usage: registry",
- Arrays.asList(SliderActions.ACTION_REGISTRY,
- Arguments.ARG_NAME,
- "cl1",
- Arguments.ARG_LISTEXP,
- Arguments.ARG_GETEXP,
- "export1"));
- assertTrue(exception instanceof UsageException);
- LOG.info(exception.toString());
- }
-
- //@Test
- public void testUpgradeUsage() throws Throwable {
- Throwable exception = launchExpectingException(SliderClient.class,
- createTestConfig(),
- "org.apache.slider.core.exceptions.BadCommandArgumentsException: Not " +
- "enough arguments for action: upgrade Expected minimum 1 but got 0",
- Arrays.asList(SliderActions.ACTION_UPGRADE));
- assertTrue(exception instanceof BadCommandArgumentsException);
- LOG.info(exception.toString());
- }
-
- public Configuration createTestConfig() {
- Configuration configuration = new Configuration();
- configuration.set(YarnConfiguration.RM_ADDRESS, "127.0.0.1:8032");
- return configuration;
- }
-
- @Ignore
- //@Test
- public void testUpgradeWithTemplateResourcesAndContainersOption() throws
- Throwable {
- //TODO test upgrade args
- String appName = "test_hbase";
- Throwable exception = launchExpectingException(SliderClient.class,
- createTestConfig(),
- "BadCommandArgumentsException: Option --containers cannot be "
- + "specified with --appdef",
- Arrays.asList(SliderActions.ACTION_UPGRADE,
- appName,
- Arguments.ARG_APPDEF,
- "/tmp/app.json",
- Arguments.ARG_CONTAINERS,
- "container_1"
- ));
- assertTrue(exception instanceof BadCommandArgumentsException);
- LOG.info(exception.toString());
- }
-
- @Ignore
- //@Test
- public void testUpgradeWithTemplateResourcesAndComponentsOption() throws
- Throwable {
- //TODO test upgrade args
- String appName = "test_hbase";
- Throwable exception = launchExpectingException(SliderClient.class,
- createTestConfig(),
- "BadCommandArgumentsException: Option --components cannot be "
- + "specified with --appdef",
- Arrays.asList(SliderActions.ACTION_UPGRADE,
- appName,
- Arguments.ARG_APPDEF,
- "/tmp/app.json",
- Arguments.ARG_COMPONENTS,
- "HBASE_MASTER"
- ));
- assertTrue(exception instanceof BadCommandArgumentsException);
- LOG.info(exception.toString());
- }
-
- //@Test
- public void testNodesMissingFile() throws Throwable {
- Throwable exception = launchExpectingException(SliderClient.class,
- createTestConfig(),
- "after parameter --out",
- Arrays.asList(SliderActions.ACTION_NODES, Arguments.ARG_OUTPUT));
- assertTrue(exception instanceof BadCommandArgumentsException);
- }
-
- //@Test
- public void testFlexWithNoComponents() throws Throwable {
- Throwable exception = launchExpectingException(SliderClient.class,
- new Configuration(),
- "Usage: slider flex <application>",
- Arrays.asList(
- SliderActions.ACTION_FLEX,
- "flex1",
- Arguments.ARG_DEFINE,
- YarnConfiguration.RM_ADDRESS + "=127.0.0.1:8032"
- ));
- assertTrue(exception instanceof UsageException);
- LOG.info(exception.toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBasicArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBasicArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBasicArgs.java
deleted file mode 100644
index b29c581..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBasicArgs.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client;
-
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.main.ServiceLauncher;
-import org.apache.slider.utils.SliderTestBase;
-import org.junit.Test;
-
-import java.net.UnknownHostException;
-import java.util.Arrays;
-
-/**
- * Test bad argument handling.
- */
-public class TestClientBasicArgs extends SliderTestBase {
-
- /**
- * Help should print out help string and then succeed.
- * @throws Throwable
- */
- //@Test
- public void testHelp() throws Throwable {
- ServiceLauncher launcher = launch(SliderClient.class,
- SliderUtils.createConfiguration(),
- Arrays.asList(ClientArgs.ACTION_HELP));
- assertEquals(0, launcher.getServiceExitCode());
- }
-
- //@Test
- public void testNoArgs() throws Throwable {
- launchExpectingException(SliderClient.class,
- SliderUtils.createConfiguration(),
- "Usage: slider COMMAND",
- EMPTY_LIST);
- }
-
- //@Test
- public void testListUnknownRM() throws Throwable {
- try {
- YarnConfiguration conf = SliderUtils.createConfiguration();
- conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
- 1000);
- conf.setLong(YarnConfiguration
- .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1000);
- ServiceLauncher launcher = launch(SliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_LIST,
- "cluster",
- Arguments.ARG_MANAGER,
- "badhost:8888"));
- fail("expected an exception, got a launcher with exit code " +
- launcher.getServiceExitCode());
- } catch (UnknownHostException expected) {
- //expected
- }
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestCommonArgParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestCommonArgParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestCommonArgParsing.java
deleted file mode 100644
index 72960fa..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestCommonArgParsing.java
+++ /dev/null
@@ -1,522 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.RoleKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionBuildArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionCreateArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionDestroyArgs;
-import org.apache.slider.common.params.ActionExistsArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
-import org.apache.slider.common.params.ActionFreezeArgs;
-import org.apache.slider.common.params.ActionListArgs;
-import org.apache.slider.common.params.ActionStatusArgs;
-import org.apache.slider.common.params.ActionThawArgs;
-import org.apache.slider.common.params.ActionUpdateArgs;
-import org.apache.hadoop.yarn.service.client.params.ArgOps;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.SliderException;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test handling of common arguments, specifically how things get split up.
- */
-public class TestCommonArgParsing implements SliderActions, Arguments {
-
-
- public static final String CLUSTERNAME = "clustername";
-
- //@Test
- public void testCreateActionArgs() throws Throwable {
- ClientArgs clientArgs = createClientArgs(Arrays.asList(ACTION_CREATE,
- "cluster1"));
- assertEquals("cluster1", clientArgs.getClusterName());
- }
-
- //@Test
- public void testCreateFailsNoClustername() throws Throwable {
- assertParseFails(Arrays.asList(ACTION_CREATE));
- }
-
- //@Test
- public void testCreateFailsTwoClusternames() throws Throwable {
- assertParseFails(Arrays.asList(
- ACTION_CREATE,
- "c1",
- "c2"
- ));
- }
-
- //@Test
- public void testHelp() throws Throwable {
- ClientArgs clientArgs = createClientArgs(Arrays.asList(ACTION_HELP));
- assertNull(clientArgs.getClusterName());
- }
-
- //@Test
- public void testSliderBasePath() throws Throwable {
- ClientArgs clientArgs = createClientArgs(Arrays.asList(ACTION_LIST,
- ARG_BASE_PATH, "/projects/slider/clusters"));
- assertEquals(new Path("/projects/slider/clusters"),
- clientArgs.getBasePath());
- }
-
- //@Test
- public void testNoSliderBasePath() throws Throwable {
- ClientArgs clientArgs = createClientArgs(Arrays.asList(ACTION_LIST));
- assertNull(clientArgs.getBasePath());
- }
-
- //@Test
- public void testListNoClusternames() throws Throwable {
- ClientArgs clientArgs = createClientArgs(Arrays.asList(ACTION_LIST));
- assertNull(clientArgs.getClusterName());
- }
-
- //@Test
- public void testListNoClusternamesDefinition() throws Throwable {
- ClientArgs clientArgs = createClientArgs(Arrays.asList(
- ACTION_LIST,
- ARG_DEFINE,
- "fs.default.FS=file://localhost"
- ));
- assertNull(clientArgs.getClusterName());
- }
-
- //@Test
- public void testList1Clustername() throws Throwable {
- ClientArgs ca = createClientArgs(Arrays.asList(ACTION_LIST, "cluster1"));
- assertEquals("cluster1", ca.getClusterName());
- assertTrue(ca.getCoreAction() instanceof ActionListArgs);
- }
-
- //@Test
- public void testListFailsTwoClusternames() throws Throwable {
- assertParseFails(Arrays.asList(
- ACTION_LIST,
- "c1",
- "c2"
- ));
- }
-
- //@Test
- public void testDefinitions() throws Throwable {
- ClientArgs ca = createClientArgs(Arrays.asList(
- ACTION_CREATE,
- CLUSTERNAME,
- "-D", "yarn.resourcemanager.principal=yarn/server@LOCAL",
- "-D", "dfs.datanode.kerberos.principal=hdfs/server@LOCAL"
- ));
- Configuration conf = new Configuration(false);
- ca.applyDefinitions(conf);
- assertEquals(CLUSTERNAME, ca.getClusterName());
- assertNull(conf.get(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH));
- SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
- SliderUtils.verifyPrincipalSet(
- conf,
- SliderXmlConfKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY);
-
- }
-
- //@Test
- public void testDefinitionsSettingBaseSliderDir() throws Throwable {
- ClientArgs ca = createClientArgs(Arrays.asList(
- ACTION_CREATE,
- CLUSTERNAME,
- "--basepath", "/projects/slider/clusters",
- "-D", "yarn.resourcemanager.principal=yarn/server@LOCAL",
- "-D", "dfs.datanode.kerberos.principal=hdfs/server@LOCAL"
- ));
- Configuration conf = new Configuration(false);
- ca.applyDefinitions(conf);
- assertEquals(CLUSTERNAME, ca.getClusterName());
- assertEquals("/projects/slider/clusters", conf.get(SliderXmlConfKeys
- .KEY_SLIDER_BASE_PATH));
- SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
- SliderUtils.verifyPrincipalSet(conf, SliderXmlConfKeys
- .DFS_DATANODE_KERBEROS_PRINCIPAL_KEY);
-
- }
-
- /**
- * Test a start command.
- * @throws Throwable
- */
- //@Test
- public void testComplexThaw() throws Throwable {
- ClientArgs ca = createClientArgs(Arrays.asList(
- ACTION_START,
- "--manager", "rhel:8032",
- "--filesystem", "hdfs://rhel:9090",
- "-S", "java.security.krb5.realm=LOCAL",
- "-S", "java.security.krb5.kdc=rhel",
- "-D", "yarn.resourcemanager.principal=yarn/rhel@LOCAL",
- "-D", "namenode.resourcemanager.principal=hdfs/rhel@LOCAL",
- "cl1"
- ));
- assertEquals("cl1", ca.getClusterName());
- assertTrue(ca.getCoreAction() instanceof ActionThawArgs);
- }
-
- /**
- * Test a force kill command where the app comes at the end of the line.
- * @throws Throwable
- *
- */
- //@Test
- public void testStatusSplit() throws Throwable {
-
- String appId = "application_1381252124398_0013";
- ClientArgs ca = createClientArgs(Arrays.asList(
- ACTION_STATUS,
- "--manager", "rhel:8032",
- "--filesystem", "hdfs://rhel:9090",
- "-S", "java.security.krb5.realm=LOCAL",
- "-S", "java.security.krb5.kdc=rhel",
- "-D", "yarn.resourcemanager.principal=yarn/rhel@LOCAL",
- "-D", "namenode.resourcemanager.principal=hdfs/rhel@LOCAL",
- appId
- ));
- assertEquals(appId, ca.getClusterName());
- }
-
- //@Test
- public void testFreezeFailsNoArg() throws Throwable {
- assertParseFails(Arrays.asList(
- ACTION_STOP
- ));
- }
-
- //@Test
- public void testFreezeWorks1Arg() throws Throwable {
- ClientArgs ca = createClientArgs(Arrays.asList(
- ACTION_STOP,
- CLUSTERNAME
- ));
- assertEquals(CLUSTERNAME, ca.getClusterName());
- assertTrue(ca.getCoreAction() instanceof ActionFreezeArgs);
- }
-
- //@Test
- public void testFreezeFails2Arg() throws Throwable {
- assertParseFails(Arrays.asList(
- ACTION_STOP, "cluster", "cluster2"
- ));
- }
-
- //@Test
- public void testFreezeForceWaitAndMessage() throws Throwable {
- ClientArgs ca = createClientArgs(Arrays.asList(
- ACTION_STOP, CLUSTERNAME,
- ARG_FORCE,
- ARG_WAIT, "0",
- ARG_MESSAGE, "explanation"
- ));
- assertEquals(CLUSTERNAME, ca.getClusterName());
- assertTrue(ca.getCoreAction() instanceof ActionFreezeArgs);
- ActionFreezeArgs freezeArgs = (ActionFreezeArgs) ca.getCoreAction();
- assertEquals("explanation", freezeArgs.message);
- assertTrue(freezeArgs.force);
- }
-
- //@Test
- public void testGetStatusWorks1Arg() throws Throwable {
- ClientArgs ca = createClientArgs(Arrays.asList(
- ACTION_STATUS,
- CLUSTERNAME
- ));
- assertEquals(CLUSTERNAME, ca.getClusterName());
- assertTrue(ca.getCoreAction() instanceof ActionStatusArgs);
- }
-
- //@Test
- public void testExistsWorks1Arg() throws Throwable {
- ClientArgs ca = createClientArgs(Arrays.asList(
- ACTION_EXISTS,
- CLUSTERNAME,
- ARG_LIVE
- ));
- assertEquals(CLUSTERNAME, ca.getClusterName());
- assertTrue(ca.getCoreAction() instanceof ActionExistsArgs);
- assertTrue(ca.getActionExistsArgs().live);
- }
-
- //@Test
- public void testDestroy1Arg() throws Throwable {
- ClientArgs ca = createClientArgs(Arrays.asList(
- ACTION_DESTROY,
- CLUSTERNAME
- ));
- assertEquals(CLUSTERNAME, ca.getClusterName());
- assertTrue(ca.getCoreAction() instanceof ActionDestroyArgs);
- }
-
- /**
- * Assert that a pass fails with a BadCommandArgumentsException.
- * @param argsList
- */
-
- private void assertParseFails(List argsList) throws SliderException {
- try {
- ClientArgs clientArgs = createClientArgs(argsList);
- Assert.fail("exected an exception, got " + clientArgs);
- } catch (BadCommandArgumentsException ignored) {
- //expected
- }
- }
-
- /**
- * Build and parse client args, after adding the base args list.
- * @param argsList
- */
- public ClientArgs createClientArgs(List<String> argsList)
- throws SliderException {
- ClientArgs serviceArgs = new ClientArgs(argsList);
- serviceArgs.parse();
- return serviceArgs;
- }
-
- public ActionCreateArgs createAction(List<String> argsList)
- throws SliderException {
- ClientArgs ca = createClientArgs(argsList);
- assertEquals(ACTION_CREATE, ca.getAction());
- ActionCreateArgs args = ca.getActionCreateArgs();
- assertNotNull(args);
- return args;
- }
-
- //@Test
- public void testSingleRoleArg() throws Throwable {
- ActionCreateArgs createArgs = createAction(Arrays.asList(
- ACTION_CREATE, "cluster1",
- ARG_COMPONENT, "master", "5"
- ));
- List<String> tuples = createArgs.getComponentTuples();
- assertEquals(2, tuples.size());
- Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples);
- assertEquals("5", roleMap.get("master"));
- }
-
- //@Test
- public void testNoRoleArg() throws Throwable {
- ActionCreateArgs createArgs = createAction(Arrays.asList(
- ACTION_CREATE, "cluster1"
- ));
- List<String> tuples = createArgs.getComponentTuples();
- Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples);
- assertNull(roleMap.get("master"));
- }
-
-
- //@Test
- public void testMultiRoleArgBuild() throws Throwable {
- ClientArgs ca = createClientArgs(Arrays.asList(
- ACTION_BUILD, "cluster1",
- ARG_COMPONENT, "master", "1",
- ARG_COMPONENT, "worker", "2"
- ));
- assertEquals(ACTION_BUILD, ca.getAction());
- assertTrue(ca.getCoreAction() instanceof ActionBuildArgs);
- assertTrue(ca.getBuildingActionArgs() instanceof ActionBuildArgs);
- AbstractClusterBuildingActionArgs args = ca.getActionBuildArgs();
- List<String> tuples = args.getComponentTuples();
- assertEquals(4, tuples.size());
- Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples);
- assertEquals("1", roleMap.get("master"));
- assertEquals("2", roleMap.get("worker"));
- }
-
- //@Test
- public void testArgUpdate() throws Throwable {
- ClientArgs ca = createClientArgs(Arrays.asList(
- ACTION_UPDATE, "cluster1",
- ARG_APPDEF, "app.json"
- ));
- assertEquals(ACTION_UPDATE, ca.getAction());
- assertTrue(ca.getCoreAction() instanceof ActionUpdateArgs);
- assertTrue(ca.getActionUpdateArgs() instanceof ActionUpdateArgs);
- AbstractClusterBuildingActionArgs args = ca.getActionUpdateArgs();
- assertNotNull(args.appDef);
- }
-
- //@Test
- public void testFlexArgs() throws Throwable {
- ClientArgs ca = createClientArgs(Arrays.asList(
- ACTION_FLEX, "cluster1",
- ARG_COMPONENT, "master", "1",
- ARG_COMPONENT, "worker", "2"
- ));
- assertTrue(ca.getCoreAction() instanceof ActionFlexArgs);
- List<String> tuples = ca.getActionFlexArgs().getComponentTuples();
- assertEquals(4, tuples.size());
- Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples);
- assertEquals("1", roleMap.get("master"));
- assertEquals("2", roleMap.get("worker"));
- }
-
- //@Test
- public void testDuplicateRole() throws Throwable {
- ActionCreateArgs createArgs = createAction(Arrays.asList(
- ACTION_CREATE, "cluster1",
- ARG_COMPONENT, "master", "1",
- ARG_COMPONENT, "master", "2"
- ));
- List<String> tuples = createArgs.getComponentTuples();
- assertEquals(4, tuples.size());
- try {
- Map<String, String> roleMap = ArgOps.convertTupleListToMap(
- "roles",
- tuples);
- Assert.fail("got a role map $roleMap not a failure");
- } catch (BadCommandArgumentsException expected) {
- assertTrue(expected.getMessage().contains(ErrorStrings
- .ERROR_DUPLICATE_ENTRY));
- }
- }
-
- //@Test
- public void testOddRoleCount() throws Throwable {
- ActionCreateArgs createArgs = createAction(Arrays.asList(
- ACTION_CREATE, "cluster1",
- ARG_COMPONENT, "master", "1",
- ARG_COMPONENT, "master", "2"
- ));
- List<String> tuples = createArgs.getComponentTuples();
- tuples.add("loggers");
- assertEquals(5, tuples.size());
- try {
- Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles",
- tuples);
- Assert.fail("got a role map " + roleMap + " not a failure");
- } catch (BadCommandArgumentsException expected) {
- assertTrue(expected.getMessage().contains(ErrorStrings
- .ERROR_PARSE_FAILURE));
- }
- }
-
- /**
- * Create some role-opt client args, so that multiple tests can use it.
- * @return the args
- */
- public ActionCreateArgs createRoleOptClientArgs() throws SliderException {
- ActionCreateArgs createArgs = createAction(Arrays.asList(
- ACTION_CREATE, "cluster1",
- ARG_COMPONENT, "master", "1",
- ARG_COMP_OPT, "master", "cheese", "swiss",
- ARG_COMP_OPT, "master", "env.CHEESE", "cheddar",
- ARG_COMP_OPT, "master", ResourceKeys.YARN_CORES, "3",
-
- ARG_COMPONENT, "worker", "2",
- ARG_COMP_OPT, "worker", ResourceKeys.YARN_CORES, "2",
- ARG_COMP_OPT, "worker", RoleKeys.JVM_HEAP, "65536",
- ARG_COMP_OPT, "worker", "env.CHEESE", "stilton"
- ));
- return createArgs;
- }
-
- //@Test
- public void testRoleOptionParse() throws Throwable {
- ActionCreateArgs createArgs = createRoleOptClientArgs();
- Map<String, Map<String, String>> tripleMaps = createArgs.getCompOptionMap();
- Map<String, String> workerOpts = tripleMaps.get("worker");
- assertEquals(3, workerOpts.size());
- assertEquals("2", workerOpts.get(ResourceKeys.YARN_CORES));
- assertEquals("65536", workerOpts.get(RoleKeys.JVM_HEAP));
-
- Map<String, String> masterOpts = tripleMaps.get("master");
- assertEquals(3, masterOpts.size());
- assertEquals("3", masterOpts.get(ResourceKeys.YARN_CORES));
-
- }
-
- //@Test
- public void testRoleOptionsMerge() throws Throwable {
- ActionCreateArgs createArgs = createRoleOptClientArgs();
-
- Map<String, Map<String, String>> roleOpts = createArgs.getCompOptionMap();
-
- Map<String, Map<String, String>> clusterRoleMap = createEnvMap();
- SliderUtils.applyCommandLineRoleOptsToRoleMap(clusterRoleMap, roleOpts);
-
- Map<String, String> masterOpts = clusterRoleMap.get("master");
- assertEquals("swiss", masterOpts.get("cheese"));
-
- Map<String, String> workerOpts = clusterRoleMap.get("worker");
- assertEquals("stilton", workerOpts.get("env.CHEESE"));
- }
-
- //@Test
- public void testEnvVariableApply() throws Throwable {
- ActionCreateArgs createArgs = createRoleOptClientArgs();
-
-
- Map<String, Map<String, String>> roleOpts = createArgs.getCompOptionMap();
-
- Map<String, Map<String, String>> clusterRoleMap = createEnvMap();
- SliderUtils.applyCommandLineRoleOptsToRoleMap(clusterRoleMap, roleOpts);
-
- Map<String, String> workerOpts = clusterRoleMap.get("worker");
- assertEquals("stilton", workerOpts.get("env.CHEESE"));
-
- Map<String, String> envmap = SliderUtils.buildEnvMap(workerOpts);
- assertEquals("stilton", envmap.get("CHEESE"));
-
- }
-
- /**
- * Static compiler complaining about matching LinkedHashMap with Map,
- * so some explicit creation here.
- * @return a map of maps
- */
- public Map<String, Map<String, String>> createEnvMap() {
-
- Map<String, String> cheese = new HashMap<>();
- cheese.put("cheese", "french");
- Map<String, String> envCheese = new HashMap<>();
- envCheese.put("env.CHEESE", "french");
- Map<String, Map<String, String>> envMap = new HashMap<>();
- envMap.put("master", cheese);
- envMap.put("worker", envCheese);
- return envMap;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java
deleted file mode 100644
index 11f8c38..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java
+++ /dev/null
@@ -1,414 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.log4j.AppenderSkeleton;
-import org.apache.log4j.Logger;
-import org.apache.log4j.spi.LoggingEvent;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.main.ServiceLauncher;
-import org.apache.slider.utils.SliderTestBase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.UUID;
-
-/**
- * Test a keytab installation.
- */
-public class TestKeytabCommandOptions extends SliderTestBase {
-
- private static SliderFileSystem testFileSystem;
- private File testFolderDir;
-
- @Before
- public void setupFilesystem() throws IOException {
- org.apache.hadoop.fs.FileSystem fileSystem = new RawLocalFileSystem();
- YarnConfiguration configuration = SliderUtils.createConfiguration();
- fileSystem.setConf(configuration);
- testFileSystem = new SliderFileSystem(fileSystem, configuration);
- testFolderDir = new File(testFileSystem
- .buildKeytabInstallationDirPath("").toUri().getPath());
- FileUtils.deleteDirectory(testFolderDir);
- }
-
- @After
- public void cleanup() throws IOException {
- if (testFolderDir != null && testFolderDir.exists()) {
- FileUtils.deleteDirectory(testFolderDir);
- }
- }
-
- //@Test
- public void testInstallKeytab() throws Throwable {
- // create a mock keytab file
- File localKeytab =
- FileUtil.createLocalTempFile(getTempLocation(), "test", true);
- String contents = UUID.randomUUID().toString();
- FileUtils.write(localKeytab, contents);
- YarnConfiguration conf = SliderUtils.createConfiguration();
- ServiceLauncher launcher = launch(TestSliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_KEYTAB,
- ClientArgs.ARG_KEYTABINSTALL,
- ClientArgs.ARG_KEYTAB,
- localKeytab.getAbsolutePath(),
- Arguments.ARG_FOLDER,
- "testFolder"));
- Path installedPath = new Path(testFileSystem
- .buildKeytabInstallationDirPath("testFolder"), localKeytab.getName());
- File installedKeytab = new File(installedPath.toUri().getPath());
- assertTrue(installedKeytab.exists());
- assertEquals(FileUtils.readFileToString(installedKeytab),
- FileUtils.readFileToString(localKeytab));
- }
-
- //@Test
- public void testInstallThenDeleteKeytab() throws Throwable {
- // create a mock keytab file
- File localKeytab =
- FileUtil.createLocalTempFile(getTempLocation(), "test", true);
- String contents = UUID.randomUUID().toString();
- FileUtils.write(localKeytab, contents);
- YarnConfiguration conf = SliderUtils.createConfiguration();
- ServiceLauncher launcher = launch(TestSliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_KEYTAB,
- ClientArgs.ARG_KEYTABINSTALL,
- ClientArgs.ARG_KEYTAB,
- localKeytab.getAbsolutePath(),
- Arguments.ARG_FOLDER,
- "testFolder"));
- Path installedPath = new Path(testFileSystem
- .buildKeytabInstallationDirPath("testFolder"), localKeytab.getName());
- File installedKeytab = new File(installedPath.toUri().getPath());
- assertTrue(installedKeytab.exists());
- assertEquals(FileUtils.readFileToString(installedKeytab),
- FileUtils.readFileToString(localKeytab));
-
- launcher = launch(TestSliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_KEYTAB,
- ClientArgs.ARG_KEYTABDELETE,
- ClientArgs.ARG_KEYTAB,
- localKeytab.getName(),
- Arguments.ARG_FOLDER,
- "testFolder"));
-
- assertFalse(installedKeytab.exists());
-
- }
-
- //@Test
- public void testInstallThenListKeytab() throws Throwable {
- // create a mock keytab file
- File localKeytab =
- FileUtil.createLocalTempFile(getTempLocation(), "test", true);
- String contents = UUID.randomUUID().toString();
- FileUtils.write(localKeytab, contents);
- YarnConfiguration conf = SliderUtils.createConfiguration();
- ServiceLauncher launcher = launch(TestSliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_KEYTAB,
- ClientArgs.ARG_KEYTABINSTALL,
- ClientArgs.ARG_KEYTAB,
- localKeytab.getAbsolutePath(),
- Arguments.ARG_FOLDER,
- "testFolder"));
- Path installedPath = new Path(testFileSystem
- .buildKeytabInstallationDirPath("testFolder"), localKeytab.getName());
- File installedKeytab = new File(installedPath.toUri().getPath());
- assertTrue(installedKeytab.exists());
- assertEquals(FileUtils.readFileToString(installedKeytab),
- FileUtils.readFileToString(localKeytab));
-
- // install an additional copy into another folder to test listing
- launcher = launch(TestSliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_KEYTAB,
- ClientArgs.ARG_KEYTABINSTALL,
- ClientArgs.ARG_KEYTAB,
- localKeytab.getAbsolutePath(),
- Arguments.ARG_FOLDER,
- "testFolder2"));
-
- TestAppender testAppender = new TestAppender();
-
- Logger.getLogger(SliderClient.class).addAppender(testAppender);
-
- try {
- launcher = launch(TestSliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_KEYTAB,
- ClientArgs.ARG_KEYTABLIST)
- );
- assertEquals(3, testAppender.events.size());
- String msg = (String) testAppender.events.get(1).getMessage();
- assertTrue(msg.contains("/.slider/keytabs/testFolder"));
- assertTrue(msg.endsWith(installedKeytab.getName()));
- msg = (String) testAppender.events.get(2).getMessage();
- assertTrue(msg.contains("/.slider/keytabs/testFolder"));
- assertTrue(msg.endsWith(installedKeytab.getName()));
- } finally {
- Logger.getLogger(SliderClient.class).removeAppender(testAppender);
- }
-
- // now listing while specifying the folder name
- testAppender = new TestAppender();
-
- Logger.getLogger(SliderClient.class).addAppender(testAppender);
-
- try {
- launcher = launch(TestSliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_KEYTAB,
- ClientArgs.ARG_KEYTABLIST,
- Arguments.ARG_FOLDER,
- "testFolder"));
- assertEquals(2, testAppender.events.size());
- String msg = (String) testAppender.events.get(1).getMessage();
- assertTrue(msg.contains("/.slider/keytabs/testFolder/" +
- installedKeytab.getName()));
- } finally {
- Logger.getLogger(SliderClient.class).removeAppender(testAppender);
- }
- }
-
- //@Test
- public void testDeleteNonExistentKeytab() throws Throwable {
- // create a mock keytab file
- YarnConfiguration conf = SliderUtils.createConfiguration();
- try {
- ServiceLauncher launcher = launch(TestSliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_KEYTAB,
- ClientArgs.ARG_KEYTABDELETE,
- ClientArgs.ARG_KEYTAB,
- "HeyIDontExist.keytab",
- Arguments.ARG_FOLDER,
- "testFolder"));
- fail("expected BadCommandArgumentsException from launch");
- } catch (BadCommandArgumentsException e) {
- // expected
- }
- }
-
- //@Test
- public void testInstallKeytabWithNoFolder() throws Throwable {
- // create a mock keytab file
- File localKeytab =
- FileUtil.createLocalTempFile(getTempLocation(), "test", true);
- String contents = UUID.randomUUID().toString();
- FileUtils.write(localKeytab, contents);
- YarnConfiguration conf = SliderUtils.createConfiguration();
- try {
- ServiceLauncher launcher = launch(TestSliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_KEYTAB,
- ClientArgs.ARG_KEYTABINSTALL,
- ClientArgs.ARG_KEYTAB,
- localKeytab.getAbsolutePath()));
- fail("expected BadCommandArgumentsException from launch");
- } catch (BadCommandArgumentsException e) {
- // expected
- }
- }
-
- //@Test
- public void testInstallKeytabWithNoKeytab() throws Throwable {
- // create a mock keytab file
- File localKeytab =
- FileUtil.createLocalTempFile(getTempLocation(), "test", true);
- String contents = UUID.randomUUID().toString();
- FileUtils.write(localKeytab, contents);
- YarnConfiguration conf = SliderUtils.createConfiguration();
- try {
- ServiceLauncher launcher = launch(TestSliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_KEYTAB,
- ClientArgs.ARG_KEYTABINSTALL,
- ClientArgs.ARG_FOLDER,
- "testFolder"));
- fail("expected BadCommandArgumentsException from launch");
- } catch (BadCommandArgumentsException e) {
- // expected
- }
- }
-
- //@Test
- public void testInstallKeytabAllowingOverwrite() throws Throwable {
- // create a mock keytab file
- File localKeytab =
- FileUtil.createLocalTempFile(getTempLocation(), "test", true);
- String contents = UUID.randomUUID().toString();
- FileUtils.write(localKeytab, contents);
- YarnConfiguration conf = SliderUtils.createConfiguration();
- ServiceLauncher launcher = launch(TestSliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_KEYTAB,
- ClientArgs.ARG_KEYTABINSTALL,
- ClientArgs.ARG_KEYTAB,
- localKeytab.getAbsolutePath(),
- Arguments.ARG_FOLDER,
- "testFolder"));
- Path installedPath = new Path(testFileSystem
- .buildKeytabInstallationDirPath("testFolder"), localKeytab.getName());
- File installedKeytab = new File(installedPath.toUri().getPath());
- assertTrue(installedKeytab.exists());
- assertEquals(FileUtils.readFileToString(installedKeytab), FileUtils
- .readFileToString(localKeytab));
- launcher = launch(TestSliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_KEYTAB,
- ClientArgs.ARG_KEYTABINSTALL,
- ClientArgs.ARG_KEYTAB,
- localKeytab.getAbsolutePath(),
- Arguments.ARG_FOLDER,
- "testFolder",
- Arguments.ARG_OVERWRITE)
- );
- assertTrue(installedKeytab.exists());
- assertEquals(FileUtils.readFileToString(installedKeytab),
- FileUtils.readFileToString(localKeytab));
- }
-
- //@Test
- public void testInstallKeytabNotAllowingOverwrite() throws Throwable {
- // create a mock keytab file
- File localKeytab =
- FileUtil.createLocalTempFile(getTempLocation(), "test", true);
- String contents = UUID.randomUUID().toString();
- FileUtils.write(localKeytab, contents);
- YarnConfiguration conf = SliderUtils.createConfiguration();
- ServiceLauncher launcher = launch(TestSliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_KEYTAB,
- ClientArgs.ARG_KEYTABINSTALL,
- ClientArgs.ARG_KEYTAB,
- localKeytab.getAbsolutePath(),
- Arguments.ARG_FOLDER,
- "testFolder"));
- Path installedPath = new Path(testFileSystem
- .buildKeytabInstallationDirPath("testFolder"), localKeytab.getName());
- File installedKeytab = new File(installedPath.toUri().getPath());
- assertTrue(installedKeytab.exists());
- assertEquals(FileUtils.readFileToString(installedKeytab),
- FileUtils.readFileToString(localKeytab));
- try {
- launcher = launch(TestSliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_KEYTAB,
- ClientArgs.ARG_KEYTABINSTALL,
- ClientArgs.ARG_KEYTAB,
- localKeytab.getAbsolutePath(),
- Arguments.ARG_FOLDER,
- "testFolder"));
- fail("expected BadCommandArgumentsException from launch");
- } catch (BadCommandArgumentsException e) {
- // expected
- }
- }
-
- //@Test
- public void testInstallKeytabWithMissingKeytab() throws Throwable {
- // create a mock keytab file
- YarnConfiguration conf = SliderUtils.createConfiguration();
- try {
- ServiceLauncher launcher = launch(TestSliderClient.class,
- conf,
- Arrays.asList(
- ClientArgs.ACTION_KEYTAB,
- ClientArgs.ARG_KEYTABINSTALL,
- ClientArgs.ARG_KEYTAB,
- "HeyIDontExist.keytab",
- Arguments.ARG_FOLDER,
- "testFolder"));
- fail("expected BadCommandArgumentsException from launch");
- } catch (BadCommandArgumentsException e) {
- // expected
- }
- }
-
- private File getTempLocation() {
- return new File(System.getProperty("user.dir") + "/target");
- }
-
- /**
- * Test SliderClient with overridden filesystem.
- */
- public static class TestSliderClient extends SliderClient {
- public TestSliderClient() {
- super();
- }
-
- @Override
- protected void initHadoopBinding() throws IOException, SliderException {
- sliderFileSystem = testFileSystem;
- }
-
- }
-
- /**
- * Appender that captures logging events.
- */
- public static class TestAppender extends AppenderSkeleton {
- private List<LoggingEvent> events = new ArrayList<>();
-
- public void close() {}
-
- public boolean requiresLayout() {
- return false;
- }
-
- @Override
- protected void append(LoggingEvent event) {
- events.add(event);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderClientMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderClientMethods.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderClientMethods.java
deleted file mode 100644
index 66aa7b8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderClientMethods.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client;
-
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.server.appmaster.model.mock.MockApplicationId;
-import org.apache.slider.utils.SliderTestBase;
-import org.easymock.EasyMock;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.api.easymock.PowerMock;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * Test slider client methods.
- */
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(SliderUtils.class)
-public class TestSliderClientMethods extends SliderTestBase {
- protected static final Logger LOG =
- LoggerFactory.getLogger(TestSliderClientMethods.class);
-
- static final String AM_ENV = "LD_LIBRARY_PATH";
- static final String PLACEHOLDER_KEY = "${distro.version}";
- static final String PLACEHOLDER_SYSTEM_KEY = "DISTRO_VERSION";
- static final String PLACEHOLDER_VALUE = "1.0.0";
- static final String AM_ENV_2 = "PATH";
- static final String PLACEHOLDER_KEY_2 = "${native.version}";
- static final String PLACEHOLDER_SYSTEM_KEY_2 = "NATIVE_VERSION";
- static final String PLACEHOLDER_VALUE_2 = "2.0.0";
-
- //@Test
- public void testGeneratePlaceholderKeyValueMap() throws Throwable {
- TestSliderClient testSliderClient = new TestSliderClient();
-
- PowerMock.mockStatic(System.class);
- EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY))
- .andReturn(PLACEHOLDER_VALUE).anyTimes();
- PowerMock.replayAll();
-
- Map<String, String> placeholders = testSliderClient
- .generatePlaceholderKeyValueMap(AM_ENV + "=/usr/lib/" +
- PLACEHOLDER_KEY);
- Assert.assertTrue(placeholders.containsKey(PLACEHOLDER_KEY));
- Assert.assertEquals("Should be equal", PLACEHOLDER_VALUE,
- placeholders.get(PLACEHOLDER_KEY));
-
- PowerMock.verifyAll();
- LOG.info("Placeholders = {}", placeholders);
- }
-
- //@Test
- public void testSetAmLaunchEnv() throws Throwable {
- TestSliderClient testSliderClient = new TestSliderClient();
- YarnConfiguration conf = SliderUtils.createConfiguration();
- conf.set(SliderXmlConfKeys.KEY_AM_LAUNCH_ENV, AM_ENV + "=/usr/lib/"
- + PLACEHOLDER_KEY);
-
- PowerMock.mockStatic(System.class);
- EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY))
- .andReturn(PLACEHOLDER_VALUE);
- PowerMock.replayAll();
-
- Map<String, String> amLaunchEnv = testSliderClient.getAmLaunchEnv(conf);
- Assert.assertNotNull(amLaunchEnv);
- Assert.assertNotNull(amLaunchEnv.get(AM_ENV));
- Assert.assertEquals("Should be equal", amLaunchEnv.get(AM_ENV),
- (Shell.WINDOWS ? "%" + AM_ENV + "%;" : "$" + AM_ENV + ":") +
- "/usr/lib/" + PLACEHOLDER_VALUE);
-
- PowerMock.verifyAll();
- LOG.info("amLaunchEnv = {}", amLaunchEnv);
- }
-
- //@Test
- public void testSetAmLaunchEnvMulti() throws Throwable {
- TestSliderClient testSliderClient = new TestSliderClient();
- YarnConfiguration conf = SliderUtils.createConfiguration();
- conf.set(SliderXmlConfKeys.KEY_AM_LAUNCH_ENV, AM_ENV + "=/usr/lib/"
- + PLACEHOLDER_KEY + "," + AM_ENV_2 + "=/usr/bin/" + PLACEHOLDER_KEY_2);
-
- PowerMock.mockStatic(System.class);
- EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY))
- .andReturn(PLACEHOLDER_VALUE);
- EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY_2))
- .andReturn(PLACEHOLDER_VALUE_2);
- PowerMock.replayAll();
-
- Map<String, String> amLaunchEnv = testSliderClient.getAmLaunchEnv(conf);
- Assert.assertNotNull(amLaunchEnv);
- Assert.assertEquals("Should have 2 envs", amLaunchEnv.size(), 2);
- Assert.assertNotNull(amLaunchEnv.get(AM_ENV));
- Assert.assertEquals("Should be equal", amLaunchEnv.get(AM_ENV),
- (Shell.WINDOWS ? "%" + AM_ENV + "%;" : "$" + AM_ENV + ":") +
- "/usr/lib/" + PLACEHOLDER_VALUE);
- Assert.assertNotNull(amLaunchEnv.get(AM_ENV_2));
- Assert.assertEquals("Should be equal", amLaunchEnv.get(AM_ENV_2),
- (Shell.WINDOWS ? "%" + AM_ENV_2 + "%;" : "$" + AM_ENV_2 + ":") +
- "/usr/bin/" + PLACEHOLDER_VALUE_2);
-
- PowerMock.verifyAll();
- LOG.info("amLaunchEnv = " + amLaunchEnv);
- }
-
- static class TestSliderClient extends SliderClient {
- @Override
- public ApplicationId submitApplication(ApplicationSubmissionContext
- context)
- throws YarnException, IOException {
- return new MockApplicationId(1);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderTokensCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderTokensCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderTokensCommand.java
deleted file mode 100644
index d140521..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderTokensCommand.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client;
-
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.common.params.ActionTokensArgs;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.NotFoundException;
-import org.apache.slider.utils.SliderTestBase;
-import org.junit.Test;
-
-import java.util.Arrays;
-
-/**
- * Test the argument parsing/validation logic.
- */
-public class TestSliderTokensCommand extends SliderTestBase {
-
- private static YarnConfiguration config = createTestConfig();
-
- public static YarnConfiguration createTestConfig() {
- YarnConfiguration configuration = new YarnConfiguration();
- configuration.set(YarnConfiguration.RM_ADDRESS, "127.0.0.1:8032");
- return configuration;
- }
-
- //@Test
- public void testBadSourceArgs() throws Throwable {
- launchExpectingException(SliderClient.class,
- config,
- ActionTokensArgs.DUPLICATE_ARGS,
- Arrays.asList(SliderActions.ACTION_TOKENS,
- Arguments.ARG_SOURCE, "target/tokens.bin",
- Arguments.ARG_OUTPUT, "target/tokens.bin"
- ));
- }
-
- //@Test
- public void testKTNoPrincipal() throws Throwable {
- launchExpectingException(SliderClient.class,
- config,
- ActionTokensArgs.MISSING_KT_PROVIDER,
- Arrays.asList(SliderActions.ACTION_TOKENS,
- Arguments.ARG_KEYTAB, "target/keytab"
- ));
- }
-
- //@Test
- public void testPrincipalNoKT() throws Throwable {
- launchExpectingException(SliderClient.class,
- config,
- ActionTokensArgs.MISSING_KT_PROVIDER,
- Arrays.asList(SliderActions.ACTION_TOKENS,
- Arguments.ARG_PRINCIPAL, "bob@REALM"
- ));
- }
-
- /**
- * A missing keytab is an error.
- * @throws Throwable
- */
- //@Test
- public void testMissingKT() throws Throwable {
- Throwable ex = launchExpectingException(SliderClient.class,
- config,
- TokensOperation.E_NO_KEYTAB,
- Arrays.asList(SliderActions.ACTION_TOKENS,
- Arguments.ARG_PRINCIPAL, "bob@REALM",
- Arguments.ARG_KEYTAB, "target/keytab"
- ));
- if (!(ex instanceof NotFoundException)) {
- throw ex;
- }
- }
-
- //@Test
- public void testMissingSourceFile() throws Throwable {
- Throwable ex = launchExpectingException(SliderClient.class,
- config,
- TokensOperation.E_MISSING_SOURCE_FILE,
- Arrays.asList(SliderActions.ACTION_TOKENS,
- Arguments.ARG_SOURCE, "target/tokens.bin"
- ));
- if (!(ex instanceof NotFoundException)) {
- throw ex;
- }
- }
-
- //@Test
- public void testListHarmlessWhenInsecure() throws Throwable {
- execSliderCommand(0, config, Arrays.asList(SliderActions.ACTION_TOKENS));
- }
-
- //@Test
- public void testCreateFailsWhenInsecure() throws Throwable {
- Throwable ex = launchExpectingException(SliderClient.class,
- config,
- TokensOperation.E_INSECURE,
- Arrays.asList(SliderActions.ACTION_TOKENS,
- Arguments.ARG_OUTPUT, "target/tokens.bin"
- ));
- if (!(ex instanceof BadClusterStateException)) {
- throw ex;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelper.java
deleted file mode 100644
index b452aba..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelper.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.slider.utils.YarnMiniClusterTestBase;
-import org.junit.Test;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.util.Map;
-
-/**
- * Test config helper.
- */
-public class TestConfigHelper extends YarnMiniClusterTestBase {
-
- //@Test
- public void testConfigLoaderIteration() throws Throwable {
-
- String xml = "<?xml version=\"1.0\" encoding=\"UTF-8\" " +
- "standalone=\"no\"?><configuration><property><name>key</name>" +
- "<value>value</value><source>programatically</source></property>" +
- "</configuration>";
- InputStream ins = new ByteArrayInputStream(xml.getBytes("UTF8"));
- Configuration conf = new Configuration(false);
- conf.addResource(ins);
- Configuration conf2 = new Configuration(false);
- for (Map.Entry<String, String> entry : conf) {
- conf2.set(entry.getKey(), entry.getValue(), "src");
- }
-
- }
-
- //@Test
- public void testConfigDeprecation() throws Throwable {
- ConfigHelper.registerDeprecatedConfigItems();
- Configuration conf = new Configuration(false);
- // test deprecated items here
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelperHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelperHDFS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelperHDFS.java
deleted file mode 100644
index 1853c84..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelperHDFS.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.utils.YarnMiniClusterTestBase;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.URI;
-
-/**
- * Test config helper loading configuration from HDFS.
- */
-public class TestConfigHelperHDFS extends YarnMiniClusterTestBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestConfigHelperHDFS.class);
-
- //@Test
- public void testConfigHelperHDFS() throws Throwable {
- YarnConfiguration config = getConfiguration();
- createMiniHDFSCluster("testConfigHelperHDFS", config);
-
- Configuration conf = new Configuration(false);
- conf.set("key", "value");
- URI fsURI = new URI(getFsDefaultName());
- Path root = new Path(fsURI);
- Path confPath = new Path(root, "conf.xml");
- FileSystem dfs = FileSystem.get(fsURI, config);
- ConfigHelper.saveConfig(dfs, confPath, conf);
- //load time
- Configuration loaded = ConfigHelper.loadConfiguration(dfs, confPath);
- LOG.info(ConfigHelper.dumpConfigToString(loaded));
- assertEquals("value", loaded.get("key"));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestExecutionEnvironment.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestExecutionEnvironment.java
deleted file mode 100644
index f08bf31..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestExecutionEnvironment.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import org.apache.slider.utils.SliderTestBase;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Test execution environment.
- */
-public class TestExecutionEnvironment extends SliderTestBase {
- protected static final Logger LOG =
- LoggerFactory.getLogger(TestExecutionEnvironment.class);
-
- //@Test
- public void testClientEnv() throws Throwable {
- SliderUtils.validateSliderClientEnvironment(LOG);
- }
-
- //@Test
- public void testWinutils() throws Throwable {
- SliderUtils.maybeVerifyWinUtilsValid();
- }
-
- //@Test
- public void testServerEnv() throws Throwable {
- SliderUtils.validateSliderServerEnvironment(LOG, true);
- }
-
- //@Test
- public void testServerEnvNoDependencies() throws Throwable {
- SliderUtils.validateSliderServerEnvironment(LOG, false);
- }
-
- //@Test
- public void testopenSSLEnv() throws Throwable {
- SliderUtils.validateOpenSSLEnv(LOG);
- }
-
- //@Test
- public void testValidatePythonEnv() throws Throwable {
- SliderUtils.validatePythonEnv(LOG);
- }
-
- //@Test
- public void testNativeLibs() throws Throwable {
- assertNativeLibrariesPresent();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestPortScan.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestPortScan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestPortScan.java
deleted file mode 100644
index a161779..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestPortScan.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.junit.Test;
-
-import java.net.ServerSocket;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.junit.Assert.*;
-
-/**
- * Test finding a port in a range.
- */
-public class TestPortScan {
-
- //@Test
- public void testScanPorts() throws Throwable {
-
- ServerSocket server = new ServerSocket(0);
-
- try {
- int serverPort = server.getLocalPort();
- assertFalse(SliderUtils.isPortAvailable(serverPort));
- int port = SliderUtils.findFreePort(serverPort, 10);
- assertTrue(port > 0 && serverPort < port);
- } finally {
- server.close();
- }
- }
-
- //@Test
- public void testRequestedPortsLogic() throws Throwable {
- PortScanner portScanner = new PortScanner();
- portScanner.setPortRange("5,6,8-10, 11,14 ,20 - 22");
- List<Integer> ports = portScanner.getRemainingPortsToCheck();
- List<Integer> expectedPorts =
- Arrays.asList(5, 6, 8, 9, 10, 11, 14, 20, 21, 22);
- assertEquals(expectedPorts, ports);
- }
-
- //@Test
- public void testRequestedPortsOutOfOrder() throws Throwable {
- PortScanner portScanner = new PortScanner();
- portScanner.setPortRange("8-10,5,6, 11,20 - 22, 14 ");
- List<Integer> ports = portScanner.getRemainingPortsToCheck();
- List<Integer> expectedPorts =
- Arrays.asList(5, 6, 8, 9, 10, 11, 14, 20, 21, 22);
- assertEquals(expectedPorts, ports);
- }
-
- //@Test
- public void testFindAvailablePortInRange() throws Throwable {
- ServerSocket server = new ServerSocket(0);
- try {
- int serverPort = server.getLocalPort();
-
- PortScanner portScanner = new PortScanner();
- portScanner.setPortRange("" + (serverPort-1) + "-" + (serverPort + 3));
- int port = portScanner.getAvailablePort();
- assertNotEquals(port, serverPort);
- assertTrue(port >= serverPort -1 && port <= serverPort + 3);
- } finally {
- server.close();
- }
- }
-
- //@Test
- public void testFindAvailablePortInList() throws Throwable {
- ServerSocket server = new ServerSocket(0);
- try {
- int serverPort = server.getLocalPort();
-
- PortScanner portScanner = new PortScanner();
- portScanner.setPortRange("" + (serverPort-1) + ", " + (serverPort + 1));
- int port = portScanner.getAvailablePort();
- assertNotEquals(port, serverPort);
- assertTrue(port == serverPort -1 || port == serverPort + 1);
- } finally {
- server.close();
- }
- }
-
- //@Test
- public void testNoAvailablePorts() throws Throwable {
- ServerSocket server1 = new ServerSocket(0);
- ServerSocket server2 = new ServerSocket(0);
- try {
- int serverPort1 = server1.getLocalPort();
- int serverPort2 = server2.getLocalPort();
-
- PortScanner portScanner = new PortScanner();
- portScanner.setPortRange("" + serverPort1+ ", " + serverPort2);
- try {
- portScanner.getAvailablePort();
- fail("expected SliderException");
- } catch (SliderException e) {
- // expected
- }
- } finally {
- server1.close();
- server2.close();
- }
- }
-
- //@Test
- public void testPortRemovedFromRange() throws Throwable {
- ServerSocket server = new ServerSocket(0);
- try {
- int serverPort = server.getLocalPort();
-
- PortScanner portScanner = new PortScanner();
- portScanner.setPortRange("" + (serverPort-1) + "-" + (serverPort + 3));
- int port = portScanner.getAvailablePort();
- assertNotEquals(port, serverPort);
- assertTrue(port >= serverPort -1 && port <= serverPort + 3);
- assertFalse(portScanner.getRemainingPortsToCheck().contains(port));
- } finally {
- server.close();
- }
- }
-
- //@Test(expected = BadConfigException.class)
- public void testBadRange() throws BadConfigException {
- PortScanner portScanner = new PortScanner();
- // note the em dash
- portScanner.setPortRange("2000–2010");
- }
-
- //@Test(expected = BadConfigException.class)
- public void testEndBeforeStart() throws BadConfigException {
- PortScanner portScanner = new PortScanner();
- portScanner.setPortRange("2001-2000");
- }
-
- //@Test(expected = BadConfigException.class)
- public void testEmptyRange() throws BadConfigException {
- PortScanner portScanner = new PortScanner();
- portScanner.setPortRange("");
- }
-
- //@Test(expected = BadConfigException.class)
- public void testBlankRange() throws BadConfigException {
- PortScanner portScanner = new PortScanner();
- portScanner.setPortRange(" ");
- }
-
- //@Test
- public void testExtraComma() throws BadConfigException {
- PortScanner portScanner = new PortScanner();
- portScanner.setPortRange("2000-2001, ");
- List<Integer> ports = portScanner.getRemainingPortsToCheck();
- List<Integer> expectedPorts = Arrays.asList(2000, 2001);
- assertEquals(expectedPorts, ports);
- }
-
- //@Test
- public void testExtraCommas() throws BadConfigException {
- PortScanner portScanner = new PortScanner();
- portScanner.setPortRange("2000-2001,, ,2003,");
- List<Integer> ports = portScanner.getRemainingPortsToCheck();
- List<Integer> expectedPorts = Arrays.asList(2000, 2001, 2003);
- assertEquals(expectedPorts, ports);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderFileSystem.java
deleted file mode 100644
index 755a4c6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderFileSystem.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.common.tools;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.utils.SliderTestBase;
-import org.junit.Test;
-
-/**
- * Test slider file system.
- */
-public class TestSliderFileSystem extends SliderTestBase {
- private static Configuration defaultConfiguration() {
- return new Configuration();
- }
-
- private static Configuration createConfigurationWithKV(String key, String
- value) {
- Configuration conf = defaultConfiguration();
- conf.set(key, value);
- return conf;
- }
-
- //@Test
- public void testSliderBasePathDefaultValue() throws Throwable {
- Configuration configuration = defaultConfiguration();
- FileSystem fileSystem = FileSystem.get(configuration);
-
- SliderFileSystem fs2 = new SliderFileSystem(fileSystem, configuration);
- assertEquals(fs2.getBaseApplicationPath(), new Path(fileSystem
- .getHomeDirectory(), ".slider"));
- }
-
- //@Test
- public void testSliderBasePathCustomValue() throws Throwable {
- Configuration configuration = createConfigurationWithKV(SliderXmlConfKeys
- .KEY_SLIDER_BASE_PATH, "/slider/cluster");
- FileSystem fileSystem = FileSystem.get(configuration);
- SliderFileSystem fs2 = new SliderFileSystem(fileSystem, configuration);
-
- assertEquals(fs2.getBaseApplicationPath(), new Path("/slider/cluster"));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderTestUtils.java
deleted file mode 100644
index e19e33f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderTestUtils.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.slider.utils.SliderTestUtils;
-import org.junit.Test;
-import org.junit.internal.AssumptionViolatedException;
-
-/**
- * Test slider test utils.
- */
-public class TestSliderTestUtils extends SliderTestUtils {
-
- //@Test
- public void testAssumeTrue() throws Throwable {
-
- try {
- assume(true, "true");
- } catch (AssumptionViolatedException e) {
- throw new Exception(e);
- }
- }
-
- //@Test
- public void testAssumeFalse() throws Throwable {
-
- try {
- assume(false, "false");
- fail("expected an exception");
- } catch (AssumptionViolatedException ignored) {
- //expected
- }
- }
-
- //@Test
- public void testAssumeBoolOptionSetInConf() throws Throwable {
- Configuration conf = new Configuration(false);
- conf.set("key", "true");
- try {
- assumeBoolOption(conf, "key", false);
- } catch (AssumptionViolatedException e) {
- throw new Exception(e);
- }
- }
-
- //@Test
- public void testAssumeBoolOptionUnsetInConf() throws Throwable {
- Configuration conf = new Configuration(false);
- try {
- assumeBoolOption(conf, "key", true);
- } catch (AssumptionViolatedException e) {
- throw new Exception(e);
- }
- }
-
-
- //@Test
- public void testAssumeBoolOptionFalseInConf() throws Throwable {
- Configuration conf = new Configuration(false);
- conf.set("key", "false");
- try {
- assumeBoolOption(conf, "key", true);
- fail("expected an exception");
- } catch (AssumptionViolatedException ignored) {
- //expected
- }
- }
-
- //@Test
- public void testAssumeBoolOptionFalseUnsetInConf() throws Throwable {
- Configuration conf = new Configuration(false);
- try {
- assumeBoolOption(conf, "key", false);
- fail("expected an exception");
- } catch (AssumptionViolatedException ignored) {
- //expected
- }
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
deleted file mode 100644
index 86025ee..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import org.apache.hadoop.util.VersionInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.util.Locale;
-import java.util.Properties;
-
-/**
- * Extract the version properties, which will look something like
- * <pre>
- * application.name=${pom.name}
- * application.version=${pom.version}
- * application.build=${buildNumber}
- * application.build.java.version=${java.version}
- * application.build.info=${pom.name}-${pom.version} Built against ${buildNumber} on ${java.version} by ${user.name}
- * </pre>
- *
- * the <code>mvn process-resources</code> target will expand the properties
- * and add the resources to target/classes, which will then look something like
- * <pre>
- * application.name=Slider Core
- * application.version=0.7.1-SNAPSHOT
- * application.build=1dd69
- * application.build.java.version=1.7.0_45
- * application.build.user=stevel
- * application.build.info=Slider Core-0.7.1-SNAPSHOT Built against 1dd69 on 1.7.0_45 by stevel
- * </pre>
- *
- * Note: the values will change and more properties added.
- */
-public class SliderVersionInfo {
- private static final Logger log = LoggerFactory.getLogger(SliderVersionInfo.class);
-
- /**
- * Name of the resource containing the filled-in-at-runtime props
- */
- public static final String VERSION_RESOURCE =
- "org/apache/slider/providers/dynamic/application.properties";
-
- public static final String APP_NAME = "application.name";
- public static final String APP_VERSION = "application.version";
- public static final String APP_BUILD = "application.build";
- public static final String APP_BUILD_JAVA_VERSION = "application.build.java.version";
- public static final String APP_BUILD_USER = "application.build.user";
- public static final String APP_BUILD_INFO = "application.build.info";
- public static final String HADOOP_BUILD_INFO = "hadoop.build.info";
- public static final String HADOOP_DEPLOYED_INFO = "hadoop.deployed.info";
-
-
- public static Properties loadVersionProperties() {
- Properties props = new Properties();
- URL resURL = SliderVersionInfo.class.getClassLoader()
- .getResource(VERSION_RESOURCE);
- assert resURL != null : "Null resource " + VERSION_RESOURCE;
-
- try {
- InputStream inStream = resURL.openStream();
- assert inStream != null : "Null input stream from " + VERSION_RESOURCE;
- props.load(inStream);
- } catch (IOException e) {
- log.warn("IOE loading " + VERSION_RESOURCE, e);
- }
- return props;
- }
-
- /**
- * Load the version info and print it
- * @param logger logger
- */
- public static void loadAndPrintVersionInfo(Logger logger) {
- Properties props = loadVersionProperties();
- logger.info(props.getProperty(APP_BUILD_INFO));
- logger.info("Compiled against Hadoop {}",
- props.getProperty(HADOOP_BUILD_INFO));
- logger.info(getHadoopVersionString());
- }
-
- public static String getHadoopVersionString() {
- return String.format(Locale.ENGLISH,
- "Hadoop runtime version %s with source checksum %s and build date %s",
- VersionInfo.getBranch(),
- VersionInfo.getSrcChecksum(),
- VersionInfo.getDate());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/BuildHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/BuildHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/BuildHelper.java
deleted file mode 100644
index 80f165f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/BuildHelper.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.buildutils;
-
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.slider.common.tools.SliderVersionInfo;
-
-import java.util.Map;
-import java.util.Properties;
-
-/**
- * classes to help with the build
- */
-public class BuildHelper {
- /**
- * Add the cluster build information; this will include Hadoop details too
- * @param dest map to insert this too
- * @param prefix prefix for the build info
- */
- public static void addBuildMetadata(Map<String, Object> dest, String prefix) {
-
- Properties props = SliderVersionInfo.loadVersionProperties();
- dest.put(prefix + "." + SliderVersionInfo.APP_BUILD_INFO,
- props.getProperty(
- SliderVersionInfo.APP_BUILD_INFO));
- dest.put(prefix + "." + SliderVersionInfo.HADOOP_BUILD_INFO,
- props.getProperty(SliderVersionInfo.HADOOP_BUILD_INFO));
-
- dest.put(prefix + "." + SliderVersionInfo.HADOOP_DEPLOYED_INFO,
- VersionInfo.getBranch() + " @" + VersionInfo.getSrcChecksum());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
deleted file mode 100644
index 9714a0f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
+++ /dev/null
@@ -1,344 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.conf;
-
-import com.google.common.base.Preconditions;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Standard map operations.
- *
- * This delegates the standard map interface to the map passed in,
- * so it can be used to add more actions to the map.
- */
-public class MapOperations implements Map<String, String> {
- private static final Logger log =
- LoggerFactory.getLogger(MapOperations.class);
- public static final String DAYS = ".days";
- public static final String HOURS = ".hours";
- public static final String MINUTES = ".minutes";
- public static final String SECONDS = ".seconds";
-
- /**
- * Global options
- */
- public final Map<String, String> options;
-
- public final String name;
-
- public MapOperations() {
- options = new HashMap<String, String>();
- name = "";
- }
-
- /**
- * Create an instance
- * @param name name
- * @param options source of options
- */
- public MapOperations(String name, Map<String, String> options) {
- Preconditions.checkArgument(options != null, "null map");
- this.options = options;
- this.name = name;
- }
-
- /**
- * Create an instance from an iterative map entry
- * @param entry entry to work with
- */
- public MapOperations(Map.Entry<String, Map<String, String>> entry) {
- Preconditions.checkArgument(entry != null, "null entry");
- this.name = entry.getKey();
- this.options = entry.getValue();
- }
-
- /**
- * Get an option value
- *
- * @param key key
- * @param defVal default value
- * @return option in map or the default
- */
- public String getOption(String key, String defVal) {
- String val = options.get(key);
- return val != null ? val : defVal;
- }
-
- /**
- * Get a boolean option
- *
- * @param key option key
- * @param defVal default value
- * @return option true if the option equals "true", or the default value
- * if the option was not defined at all.
- */
- public Boolean getOptionBool(String key, boolean defVal) {
- String val = getOption(key, Boolean.toString(defVal));
- return Boolean.valueOf(val);
- }
-
- /**
- * Get a cluster option or value
- *
- * @param key option key
- * @return the value
- * @throws BadConfigException if the option is missing
- */
-
- public String getMandatoryOption(String key) throws BadConfigException {
- String val = options.get(key);
- if (val == null) {
- if (log.isDebugEnabled()) {
- log.debug("Missing key {} from config containing {}",
- key, this);
- }
- String text = "Missing option " + key;
- if (SliderUtils.isSet(name)) {
- text += " from set " + name;
- }
- throw new BadConfigException(text);
- }
- return val;
- }
-
- /**
- * Get an integer option; use {@link Integer#decode(String)} so as to take hex
- * oct and bin values too.
- *
- * @param option option name
- * @param defVal default value
- * @return parsed value
- * @throws NumberFormatException if the role could not be parsed.
- */
- public int getOptionInt(String option, int defVal) {
- String val = getOption(option, Integer.toString(defVal));
- return Integer.decode(val);
- }
-
- /**
- * Get a long option; use {@link Long#decode(String)} so as to take hex
- * oct and bin values too.
- *
- * @param option option name
- * @param defVal default value
- * @return parsed value
- * @throws NumberFormatException
- */
- public long getOptionLong(String option, long defVal) {
- String val = getOption(option, Long.toString(defVal));
- return Long.decode(val);
- }
-
- /**
- * Get a mandatory integer option; use {@link Integer#decode(String)} so as to take hex
- * oct and bin values too.
- *
- * @param option option name
- * @return parsed value
- * @throws NumberFormatException if the option could not be parsed.
- * @throws BadConfigException if the option could not be found
- */
- public int getMandatoryOptionInt(String option) throws BadConfigException {
- getMandatoryOption(option);
- return getOptionInt(option, 0);
- }
-
- /**
- * Verify that an option is set: that is defined AND non-empty
- * @param key
- * @throws BadConfigException
- */
- public void verifyOptionSet(String key) throws BadConfigException {
- if (SliderUtils.isUnset(getOption(key, null))) {
- throw new BadConfigException("Unset option %s", key);
- }
- }
-
- public void mergeWithoutOverwrite(Map<String, String> that) {
- SliderUtils.mergeMapsIgnoreDuplicateKeys(options, that);
- }
-
- /**
- * Merge a map by prefixed keys
- * @param that the map to merge in
- * @param prefix prefix to match on
- * @param overwrite flag to enable overwrite
- */
- public void mergeMapPrefixedKeys(Map<String, String> that,
- String prefix,
- boolean overwrite) {
- for (Map.Entry<String, String> entry : that.entrySet()) {
- String key = entry.getKey();
- if (key.startsWith(prefix)) {
- if (overwrite || get(key) == null) {
- put(key, entry.getValue());
- }
- }
- }
- }
-
- /**
- * Set a property if it is not already set
- * @param key key
- * @param value value
- */
- public void putIfUnset(String key, String value) {
- if (get(key) == null) {
- put(key, value);
- }
- }
-
- public void set(String key, Object value) {
- assert value != null;
- put(key, value.toString());
- }
-
- public int size() {
- return options.size();
- }
-
- public boolean isEmpty() {
- return options.isEmpty();
- }
-
- public boolean containsValue(Object value) {
- return options.containsValue(value);
- }
-
- public boolean containsKey(Object key) {
- return options.containsKey(key);
- }
-
- public String get(Object key) {
- return options.get(key);
- }
-
- public String put(String key, String value) {
- return options.put(key, value);
- }
-
- public String remove(Object key) {
- return options.remove(key);
- }
-
- public void putAll(Map<? extends String, ? extends String> m) {
- options.putAll(m);
- }
-
- public void clear() {
- options.clear();
- }
-
- public Set<String> keySet() {
- return options.keySet();
- }
-
- public Collection<String> values() {
- return options.values();
- }
-
- public Set<Map.Entry<String, String>> entrySet() {
- return options.entrySet();
- }
-
- @SuppressWarnings("EqualsWhichDoesntCheckParameterClass")
- public boolean equals(Object o) {
- return options.equals(o);
- }
-
- @Override
- public int hashCode() {
- return options.hashCode();
- }
-
- public boolean isSet(String key) {
- return SliderUtils.isSet(get(key));
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append(name).append("=\n");
-
- for (Entry<String, String> entry : options.entrySet()) {
- builder.append(" ")
- .append(entry.getKey())
- .append('=')
- .append(entry.getValue())
- .append('\n');
- }
- return builder.toString();
- }
-
- /**
- * Get the time range of a set of keys
- * @param basekey base key to which suffix gets applied
- * @param defDays
- * @param defHours
- * @param defMins
- * @param defSecs
- * @return the aggregate time range in seconds
- */
- public long getTimeRange(String basekey,
- int defDays,
- int defHours,
- int defMins,
- int defSecs) {
- Preconditions.checkArgument(basekey != null);
- int days = getOptionInt(basekey + DAYS, defDays);
- int hours = getOptionInt(basekey + HOURS, defHours);
-
- int minutes = getOptionInt(basekey + MINUTES, defMins);
- int seconds = getOptionInt(basekey + SECONDS, defSecs);
- // range check
- Preconditions.checkState(days >= 0 && hours >= 0 && minutes >= 0
- && seconds >= 0,
- "Time range for %s has negative time component %s:%s:%s:%s",
- basekey, days, hours, minutes, seconds);
-
- // calculate total time, schedule the reset if expected
- long totalMinutes = (long) days * 24 * 60 + (long) hours * 24 + minutes;
- return totalMinutes * 60 + seconds;
- }
-
- /**
- * Get all entries with a specific prefix
- * @param prefix prefix
- * @return a prefixed map, possibly empty
- */
- public Map<String, String> prefixedWith(String prefix) {
-
- Map<String, String> prefixed = new HashMap<>(size());
- for (Entry<String, String> entry: entrySet()) {
- if (entry.getKey().startsWith(prefix)) {
- prefixed.put(entry.getKey(), entry.getValue());
- }
- }
- return prefixed;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/BadClusterStateException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/BadClusterStateException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/BadClusterStateException.java
deleted file mode 100644
index e73ce57..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/BadClusterStateException.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-
-/**
- * The system is in a bad state
- */
-public class BadClusterStateException extends SliderException {
- public BadClusterStateException(String message,
- Object... args) {
- super(EXIT_BAD_STATE, message, args);
- }
-
- public BadClusterStateException(Throwable throwable,
- String message, Object... args) {
- super(EXIT_BAD_STATE, throwable, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/BadCommandArgumentsException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/BadCommandArgumentsException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/BadCommandArgumentsException.java
deleted file mode 100644
index 0d5d686..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/BadCommandArgumentsException.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-public class BadCommandArgumentsException extends SliderException {
- public BadCommandArgumentsException(String s, Object... args) {
- super(EXIT_COMMAND_ARGUMENT_ERROR, s, args);
- }
-
- public BadCommandArgumentsException(Throwable throwable, String message,
- Object... args) {
- super(EXIT_COMMAND_ARGUMENT_ERROR, throwable, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/BadConfigException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/BadConfigException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/BadConfigException.java
deleted file mode 100644
index 65a8ea8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/BadConfigException.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-/**
- * An exception to raise on a bad configuration
- */
-public class BadConfigException extends SliderException {
-
- public BadConfigException(String s) {
- super(EXIT_BAD_CONFIGURATION, s);
- }
-
- public BadConfigException(String message, Object... args) {
- super(EXIT_BAD_CONFIGURATION, message, args);
- }
-
- public BadConfigException(
- Throwable throwable,
- String message, Object... args) {
- super(EXIT_BAD_CONFIGURATION, throwable, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java
deleted file mode 100644
index 8b04969..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-public interface ErrorStrings {
- String E_UNSTABLE_CLUSTER = "Unstable Application Instance :";
- String E_CLUSTER_RUNNING = "Application Instance running";
- String E_ALREADY_EXISTS = "already exists";
- String PRINTF_E_INSTANCE_ALREADY_EXISTS = "Application Instance \"%s\" already exists and is defined in %s";
- String PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS = "Application Instance dir already exists: %s";
- String E_MISSING_PATH = "Missing path ";
- String E_INCOMPLETE_CLUSTER_SPEC =
- "Cluster specification is marked as incomplete: ";
- String E_UNKNOWN_INSTANCE = "Unknown application instance ";
- String E_DESTROY_CREATE_RACE_CONDITION =
- "created while it was being destroyed";
- String E_UNKNOWN_ROLE = "Unknown role ";
- /**
- * ERROR Strings
- */
- String ERROR_NO_ACTION = "No action specified";
- String ERROR_UNKNOWN_ACTION = "Unknown command: ";
- String ERROR_NOT_ENOUGH_ARGUMENTS =
- "Not enough arguments for action: ";
- String ERROR_PARSE_FAILURE =
- "Failed to parse ";
- /**
- * All the remaining values after argument processing
- */
- String ERROR_TOO_MANY_ARGUMENTS =
- "Too many arguments";
- String ERROR_DUPLICATE_ENTRY = "Duplicate entry for ";
- String E_APPLICATION_NOT_RUNNING = "Application not running";
- String E_FINISHED_APPLICATION = E_APPLICATION_NOT_RUNNING + ": %s state=%s ";
- String E_NO_IMAGE_OR_HOME_DIR_SPECIFIED =
- "Neither an image path nor binary home directory were specified";
- String E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED =
- "Both application image path and home dir have been provided";
- String E_CONFIGURATION_DIRECTORY_NOT_FOUND =
- "Configuration directory \"%s\" not found";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
deleted file mode 100644
index efec676..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import org.apache.hadoop.fs.InvalidRequestException;
-import org.apache.hadoop.fs.PathAccessDeniedException;
-import org.apache.hadoop.fs.PathIOException;
-import org.apache.hadoop.yarn.webapp.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletResponse;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
-/**
- * static methods to convert exceptions into different types, including
- * extraction of details and finer-grained conversions.
- */
-public class ExceptionConverter {
- private static final Logger
- log = LoggerFactory.getLogger(ExceptionConverter.class);
-
- /**
- * Uprate error codes 400 and up into faults;
- * 404 is converted to a {@link FileNotFoundException},
- * 401 to {@link ForbiddenException}
- * FileNotFoundException for an unknown resource
- * PathAccessDeniedException for access denied
- * PathIOException for anything else
- * @param verb HTTP Verb used
- * @param targetURL URL being targeted
- * @param exception original exception
- * @return a new exception, the original one nested as a cause
- */
- public static IOException convertJerseyException(String verb,
- String targetURL,
- UniformInterfaceException exception) {
-
- IOException ioe = null;
- ClientResponse response = exception.getResponse();
- if (response != null) {
- int status = response.getStatus();
- String body = "";
- try {
- if (response.hasEntity()) {
- body = response.getEntity(String.class);
- log.error("{} {} returned status {} and body\n{}",
- verb, targetURL, status, body);
- } else {
- log.error("{} {} returned status {} and empty body",
- verb, targetURL, status);
- }
- } catch (Exception e) {
- log.warn("Failed to extract body from client response", e);
- }
-
- if (status == HttpServletResponse.SC_UNAUTHORIZED
- || status == HttpServletResponse.SC_FORBIDDEN) {
- ioe = new PathAccessDeniedException(targetURL);
- } else if (status == HttpServletResponse.SC_BAD_REQUEST
- || status == HttpServletResponse.SC_NOT_ACCEPTABLE
- || status == HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE) {
- // bad request
- ioe = new InvalidRequestException(
- String.format("Bad %s request: status code %d against %s",
- verb, status, targetURL));
- } else if (status > 400 && status < 500) {
- ioe = new FileNotFoundException(targetURL);
- }
- if (ioe == null) {
- ioe = new PathIOException(targetURL,
- verb + " " + targetURL
- + " failed with status code : " + status
- + ":" + exception);
- }
- } else {
- ioe = new PathIOException(targetURL,
- verb + " " + targetURL + " failed: " + exception);
- }
- ioe.initCause(exception);
- return ioe;
- }
-
- /**
- * Handle a client-side Jersey exception.
- * <p>
- * If there's an inner IOException, return that.
- * <p>
- * Otherwise: create a new wrapper IOE including verb and target details
- * @param verb HTTP Verb used
- * @param targetURL URL being targeted
- * @param exception original exception
- * @return an exception to throw
- */
- public static IOException convertJerseyException(String verb,
- String targetURL,
- ClientHandlerException exception) {
- if (exception.getCause() instanceof IOException) {
- return (IOException)exception.getCause();
- } else {
- IOException ioe = new IOException(
- verb + " " + targetURL + " failed: " + exception);
- ioe.initCause(exception);
- return ioe;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/NoSuchNodeException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/NoSuchNodeException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/NoSuchNodeException.java
deleted file mode 100644
index ad2f1a4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/NoSuchNodeException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-import java.io.IOException;
-
-/**
- * Exception raised when a node cannot be found in the structure
- * that is being examined.
- */
-public class NoSuchNodeException extends IOException {
-
- public NoSuchNodeException(String uuid) {
- super(uuid);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/NotFoundException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/NotFoundException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/NotFoundException.java
deleted file mode 100644
index 40cb94d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/NotFoundException.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-
-/**
- * Whatever was being resolved: it was not found
- */
-public class NotFoundException extends SliderException {
- public NotFoundException(String message,
- Object... args) {
- super(EXIT_NOT_FOUND, message, args);
- }
-
- public NotFoundException(Throwable throwable,
- String message, Object... args) {
- super(EXIT_NOT_FOUND, throwable, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/ServiceNotReadyException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/ServiceNotReadyException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/ServiceNotReadyException.java
deleted file mode 100644
index 435bc1a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/ServiceNotReadyException.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-import java.io.IOException;
-
-/**
- * This is an exception raised when the service does not consider itself
- * live (yet)
- */
-public class ServiceNotReadyException extends IOException {
-
- public static final String E_NOT_READY =
- "Service not ready for access: please retry";
-
- public ServiceNotReadyException(String message) {
- super(message);
- }
-
- public ServiceNotReadyException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public ServiceNotReadyException(Throwable cause) {
- super(cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
deleted file mode 100644
index 1430c5a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.slider.core.main.ServiceLaunchException;
-
-public class SliderException extends ServiceLaunchException implements
- SliderExitCodes {
- public SliderException() {
- super(EXIT_EXCEPTION_THROWN, "SliderException");
- }
-
- public SliderException(int code, String message) {
- super(code, message);
- }
-
- public SliderException(String s) {
- super(EXIT_EXCEPTION_THROWN, s);
- }
-
- public SliderException(String s, Throwable throwable) {
- super(EXIT_EXCEPTION_THROWN, s, throwable);
- }
-
- /**
- * Format the exception as you create it
- * @param code exit code
- * @param message exception message -sprintf formatted
- * @param args arguments for the formatting
- */
- public SliderException(int code, String message, Object... args) {
- super(code, String.format(message, args));
- }
-
- /**
- * Format the exception, include a throwable.
- * The throwable comes before the message so that it is out of the varargs
- * @param code exit code
- * @param throwable thrown
- * @param message message
- * @param args arguments
- */
- public SliderException(int code,
- Throwable throwable,
- String message,
- Object... args) {
- super(code, String.format(message, args), throwable);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderInternalStateException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderInternalStateException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderInternalStateException.java
deleted file mode 100644
index deddbbc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderInternalStateException.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-public class SliderInternalStateException extends SliderException {
- public SliderInternalStateException(String s) {
- super(EXIT_INTERNAL_ERROR, s);
- }
-
- public SliderInternalStateException(String s, Throwable throwable) {
- super(EXIT_INTERNAL_ERROR, throwable, s);
- }
-
- public SliderInternalStateException(String message,
- Object... args) {
- super(EXIT_INTERNAL_ERROR, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java
deleted file mode 100644
index bb9f430..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-
-/**
- * An Exception to be thrown for an explicit "shut down the cluster" operation
- * raised by the application state or other parts of the AM
- */
-public class TriggerClusterTeardownException extends SliderException {
-
- private final FinalApplicationStatus finalApplicationStatus;
-
- public TriggerClusterTeardownException(int code,
- FinalApplicationStatus finalApplicationStatus, String message,
- Object... args) {
- super(code, message, args);
- this.finalApplicationStatus = finalApplicationStatus;
- }
-
- public FinalApplicationStatus getFinalApplicationStatus() {
- return finalApplicationStatus;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/UnknownApplicationInstanceException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/UnknownApplicationInstanceException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/UnknownApplicationInstanceException.java
deleted file mode 100644
index a1f8ae9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/UnknownApplicationInstanceException.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-public class UnknownApplicationInstanceException extends SliderException {
- public UnknownApplicationInstanceException(String s) {
- super(EXIT_UNKNOWN_INSTANCE, s);
- }
-
- public UnknownApplicationInstanceException(String s, Throwable throwable) {
- super(EXIT_UNKNOWN_INSTANCE, throwable, s);
- }
-
- public UnknownApplicationInstanceException(String message,
- Object... args) {
- super(EXIT_UNKNOWN_INSTANCE, message, args);
- }
-
- /**
- * Create an instance with the standard exception name
- * @param name name
- * @return an instance to throw
- */
- public static UnknownApplicationInstanceException unknownInstance(String name) {
- return new UnknownApplicationInstanceException(ErrorStrings.E_UNKNOWN_INSTANCE
- + ": " + name);
- }
- public static UnknownApplicationInstanceException unknownInstance(String name,
- Throwable throwable) {
- UnknownApplicationInstanceException exception =
- unknownInstance(name);
- exception.initCause(throwable);
- return exception;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java
deleted file mode 100644
index 8684294..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-/**
- * Used to raise a usage exception ... this has the exit code
- * {@link #EXIT_USAGE}
- */
-public class UsageException extends SliderException {
- public UsageException(String s, Object... args) {
- super(EXIT_USAGE, s, args);
- }
-
- public UsageException(Throwable throwable, String message,
- Object... args) {
- super(EXIT_USAGE, throwable, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/WaitTimeoutException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/WaitTimeoutException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/WaitTimeoutException.java
deleted file mode 100644
index 5ad3fdc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/WaitTimeoutException.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-import java.io.IOException;
-
-/**
- * Called when some spinning operation timed out
- */
-public class WaitTimeoutException extends IOException {
- public WaitTimeoutException(String message) {
- super(message);
- }
-
- public WaitTimeoutException(String message, Throwable cause) {
- super(message, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
deleted file mode 100644
index a3e1bf2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
-import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.slider.common.tools.CoreFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import static org.apache.hadoop.yarn.service.provider.docker.DockerKeys.DEFAULT_DOCKER_NETWORK;
-
-/**
- * Launcher of applications: base class
- */
-public class AbstractLauncher {
- private static final Logger log =
- LoggerFactory.getLogger(AbstractLauncher.class);
- public static final String CLASSPATH = "CLASSPATH";
- /**
- * Filesystem to use for the launch
- */
- protected final CoreFileSystem coreFileSystem;
- /**
- * Env vars; set up at final launch stage
- */
- protected final Map<String, String> envVars = new HashMap<>();
- protected final ContainerLaunchContext containerLaunchContext =
- Records.newRecord(ContainerLaunchContext.class);
- protected final List<String> commands = new ArrayList<>(20);
- protected final Map<String, LocalResource> localResources = new HashMap<>();
- protected final Map<String, String> mountPaths = new HashMap<>();
- private final Map<String, ByteBuffer> serviceData = new HashMap<>();
- // security
- protected final Credentials credentials;
- protected boolean yarnDockerMode = false;
- protected String dockerImage;
- protected String dockerNetwork = DEFAULT_DOCKER_NETWORK;
- protected String dockerHostname;
- protected String runPrivilegedContainer;
-
-
- /**
- * Create instance.
- * @param coreFileSystem filesystem
- * @param credentials initial set of credentials -null is permitted
- */
- public AbstractLauncher(
- CoreFileSystem coreFileSystem,
- Credentials credentials) {
- this.coreFileSystem = coreFileSystem;
- this.credentials = credentials != null ? credentials: new Credentials();
- }
-
- public void setYarnDockerMode(boolean yarnDockerMode){
- this.yarnDockerMode = yarnDockerMode;
- }
-
- /**
- * Get the env vars to work on
- * @return env vars
- */
- public Map<String, String> getEnv() {
- return envVars;
- }
-
- /**
- * Get the launch commands.
- * @return the live list of commands
- */
- public List<String> getCommands() {
- return commands;
- }
-
- public void addLocalResource(String subPath, LocalResource resource) {
- localResources.put(subPath, resource);
- }
-
- public void addLocalResource(String subPath, LocalResource resource, String mountPath) {
- localResources.put(subPath, resource);
- mountPaths.put(subPath, mountPath);
- }
-
- /**
- * Accessor to the credentials
- * @return the credentials associated with this launcher
- */
- public Credentials getCredentials() {
- return credentials;
- }
-
-
- public void addCommand(String cmd) {
- commands.add(cmd);
- }
-
- /**
- * Complete the launch context (copy in env vars, etc).
- * @return the container to launch
- */
- public ContainerLaunchContext completeContainerLaunch() throws IOException {
-
- String cmdStr = SliderUtils.join(commands, " ", false);
- log.debug("Completed setting up container command {}", cmdStr);
- containerLaunchContext.setCommands(commands);
-
- //env variables
- if (log.isDebugEnabled()) {
- log.debug("Environment variables");
- for (Map.Entry<String, String> envPair : envVars.entrySet()) {
- log.debug(" \"{}\"=\"{}\"", envPair.getKey(), envPair.getValue());
- }
- }
- containerLaunchContext.setEnvironment(envVars);
-
- //service data
- if (log.isDebugEnabled()) {
- log.debug("Service Data size");
- for (Map.Entry<String, ByteBuffer> entry : serviceData.entrySet()) {
- log.debug("\"{}\"=> {} bytes of data", entry.getKey(),
- entry.getValue().array().length);
- }
- }
- containerLaunchContext.setServiceData(serviceData);
-
- // resources
- dumpLocalResources();
- containerLaunchContext.setLocalResources(localResources);
-
- //tokens
- log.debug("{} tokens", credentials.numberOfTokens());
- containerLaunchContext.setTokens(CredentialUtils.marshallCredentials(
- credentials));
-
- if(yarnDockerMode){
- Map<String, String> env = containerLaunchContext.getEnvironment();
- env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker");
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage);
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", dockerNetwork);
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME",
- dockerHostname);
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer);
- StringBuilder sb = new StringBuilder();
- for (Entry<String,String> mount : mountPaths.entrySet()) {
- if (sb.length() > 0) {
- sb.append(",");
- }
- sb.append(mount.getKey());
- sb.append(":");
- sb.append(mount.getValue());
- }
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS", sb.toString());
- log.info("yarn docker env var has been set {}", containerLaunchContext.getEnvironment().toString());
- }
-
- return containerLaunchContext;
- }
-
- public void setRetryContext(int maxRetries, int retryInterval) {
- ContainerRetryContext retryContext = ContainerRetryContext
- .newInstance(ContainerRetryPolicy.RETRY_ON_ALL_ERRORS, null, maxRetries,
- retryInterval);
- containerLaunchContext.setContainerRetryContext(retryContext);
- }
-
- /**
- * Dump local resources at debug level
- */
- private void dumpLocalResources() {
- if (log.isDebugEnabled()) {
- log.debug("{} resources: ", localResources.size());
- for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) {
-
- String key = entry.getKey();
- LocalResource val = entry.getValue();
- log.debug(key + "=" + SliderUtils.stringify(val.getResource()));
- }
- }
- }
-
- /**
- * This is critical for an insecure cluster -it passes
- * down the username to YARN, and so gives the code running
- * in containers the rights it needs to work with
- * data.
- * @throws IOException problems working with current user
- */
- protected void propagateUsernameInInsecureCluster() throws IOException {
- //insecure cluster: propagate user name via env variable
- String userName = UserGroupInformation.getCurrentUser().getUserName();
- envVars.put(SliderKeys.HADOOP_USER_NAME, userName);
- }
-
- /**
- * Utility method to set up the classpath
- * @param classpath classpath to use
- */
- public void setClasspath(ClasspathConstructor classpath) {
- setEnv(CLASSPATH, classpath.buildClasspath());
- }
-
- /**
- * Set an environment variable in the launch context
- * @param var variable name
- * @param value value (must be non null)
- */
- public void setEnv(String var, String value) {
- Preconditions.checkArgument(var != null, "null variable name");
- Preconditions.checkArgument(value != null, "null value");
- envVars.put(var, value);
- }
-
-
- public void putEnv(Map<String, String> map) {
- envVars.putAll(map);
- }
-
-
- public void setDockerImage(String dockerImage) {
- this.dockerImage = dockerImage;
- }
-
- public void setDockerNetwork(String dockerNetwork) {
- this.dockerNetwork = dockerNetwork;
- }
-
- public void setDockerHostname(String dockerHostname) {
- this.dockerHostname = dockerHostname;
- }
-
- public void setRunPrivilegedContainer(boolean runPrivilegedContainer) {
- if (runPrivilegedContainer) {
- this.runPrivilegedContainer = Boolean.toString(true);
- } else {
- this.runPrivilegedContainer = Boolean.toString(false);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
deleted file mode 100644
index 6eb4058..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.common.tools.SliderUtils;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * build a classpath -allows for entries to be injected in front of
- * YARN classpath as well as behind, adds appropriate separators,
- * extraction of local classpath, etc.
- */
-public class ClasspathConstructor {
-
- public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR;
- private final List<String> pathElements = new ArrayList<>();
-
- public ClasspathConstructor() {
- }
-
-
- /**
- * Get the list of JARs from the YARN settings
- * @param config configuration
- */
- public List<String> yarnApplicationClasspath(Configuration config) {
- String[] cp = config.getTrimmedStrings(
- YarnConfiguration.YARN_APPLICATION_CLASSPATH,
- YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH);
- return cp != null ? Arrays.asList(cp) : new ArrayList<String>(0);
-
- }
-
-
- @Override
- public String toString() {
- return buildClasspath();
- }
-
- public String buildClasspath() {
- return SliderUtils.join(pathElements,
- CLASS_PATH_SEPARATOR,
- false);
- }
-
- /**
- * Get a copy of the path list
- * @return the JARs
- */
- public List<String> getPathElements() {
- return Collections.unmodifiableList(pathElements);
- }
-
- /**
- * Append an entry
- * @param path path
- */
- public void append(String path) {
- pathElements.add(path);
- }
-
- /**
- * Insert a path at the front of the list. This places it ahead of
- * the standard YARN artifacts
- * @param path path to the JAR. Absolute or relative -on the target
- * system
- */
- public void insert(String path) {
- pathElements.add(0, path);
- }
-
- public void appendAll(Collection<String> paths) {
- pathElements.addAll(paths);
- }
-
- public void insertAll(Collection<String> paths) {
- pathElements.addAll(0, paths);
- }
-
-
- public void addLibDir(String pathToLibDir) {
- append(buildLibDir(pathToLibDir));
- }
-
- public void insertLibDir(String pathToLibDir) {
- insert(buildLibDir(pathToLibDir));
- }
-
- public void addClassDirectory(String pathToDir) {
- append(appendDirectoryTerminator(pathToDir));
- }
-
- public void insertClassDirectory(String pathToDir) {
- insert(buildLibDir(appendDirectoryTerminator(pathToDir)));
- }
-
-
- public void addRemoteClasspathEnvVar() {
- append(ApplicationConstants.Environment.CLASSPATH.$$());
- }
-
-
- public void insertRemoteClasspathEnvVar() {
- append(ApplicationConstants.Environment.CLASSPATH.$$());
- }
-
-
- /**
- * Build a lib dir path
- * @param pathToLibDir path to the directory; may or may not end with a
- * trailing space
- * @return a path to a lib dir that is compatible with the java classpath
- */
- public String buildLibDir(String pathToLibDir) {
- String dir = appendDirectoryTerminator(pathToLibDir);
- dir += "*";
- return dir;
- }
-
- private String appendDirectoryTerminator(String pathToLibDir) {
- String dir = pathToLibDir.trim();
- if (!dir.endsWith("/")) {
- dir += "/";
- }
- return dir;
- }
-
- /**
- * Split a classpath. This uses the local path separator so MUST NOT
- * be used to work with remote classpaths
- * @param localpath local path
- * @return a splite
- */
- public Collection<String> splitClasspath(String localpath) {
- String separator = System.getProperty("path.separator");
- return StringUtils.getStringCollection(localpath, separator);
- }
-
- /**
- * Get the local JVM classpath split up
- * @return the list of entries on the JVM classpath env var
- */
- public Collection<String> localJVMClasspath() {
- return splitClasspath(System.getProperty("java.class.path"));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
deleted file mode 100644
index 5ab0532..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.slider.common.tools.SliderUtils;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Build a single command line to include in the container commands;
- * Special support for JVM command buildup.
- */
-public class CommandLineBuilder {
- protected final List<String> argumentList = new ArrayList<>(20);
-
- /**
- * Add an entry to the command list
- * @param args arguments -these will be converted strings
- */
- public void add(Object... args) {
- for (Object arg : args) {
- argumentList.add(arg.toString());
- }
- }
-
- /**
- * Get the number of arguments
- * @return an integer >= 0
- */
- public int size() {
- return argumentList.size();
- }
-
- /**
- * Append the output and error files to the tail of the command
- * @param stdout out
- * @param stderr error. Set this to null to append into stdout
- */
- public void addOutAndErrFiles(String stdout, String stderr) {
- Preconditions.checkNotNull(stdout, "Null output file");
- Preconditions.checkState(!stdout.isEmpty(), "output filename invalid");
- // write out the path output
- argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
- stdout);
- if (stderr != null) {
- argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
- stderr);
- } else {
- argumentList.add("2>&1");
- }
- }
-
- /**
- * This just returns the command line
- * @see #build()
- * @return the command line
- */
- @Override
- public String toString() {
- return build();
- }
-
- /**
- * Build the command line
- * @return the command line
- */
- public String build() {
- return SliderUtils.join(argumentList, " ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
deleted file mode 100644
index 7e164e4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.slider.common.tools.CoreFileSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.InetSocketAddress;
-
-/**
- * Code to ease launching of any container
- */
-public class ContainerLauncher extends AbstractLauncher {
- private static final Logger log =
- LoggerFactory.getLogger(ContainerLauncher.class);
-
- public ContainerLauncher(Configuration conf,
- CoreFileSystem coreFileSystem,
- Container container,
- Credentials credentials) {
- super(coreFileSystem, credentials);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
deleted file mode 100644
index 395ff22..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
+++ /dev/null
@@ -1,380 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.providers.PlacementPolicy;
-import org.apache.slider.server.appmaster.model.mock.MockAppState;
-import org.apache.slider.server.appmaster.model.mock.MockFactory;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.AppState.NodeUpdatedOutcome;
-import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.slider.api.ResourceKeys.COMPONENT_PLACEMENT_POLICY;
-import static org.apache.slider.server.appmaster.model.mock.MockFactory.AAROLE_2;
-
-/**
- * Test Anti-affine placement.
- */
-public class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
- implements MockRoles {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestMockAppStateAAPlacement.class);
-
- private static final int NODES = 3;
-
- /**
- * The YARN engine has a cluster with very few nodes (3) and lots of
- * containers, so if AA placement isn't working, there will be affine
- * placements surfacing.
- * @return
- */
- @Override
- public MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(NODES, 8);
- }
-
- /**
- * This is the simplest AA allocation: no labels, so allocate anywhere.
- * @throws Throwable
- */
- //@Test
- public void testAllocateAANoLabel() throws Throwable {
- RoleStatus aaRole = getAaRole();
-
- assertTrue(cloneNodemap().size() > 0);
-
- // want multiple instances, so there will be iterations
- aaRole.setDesired(2);
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- AMRMClient.ContainerRequest request = getSingleRequest(ops);
- assertFalse(request.getRelaxLocality());
- assertEquals(request.getNodes().size(), engine.getCluster()
- .getClusterSize());
- assertNull(request.getRacks());
- assertNotNull(request.getCapability());
-
- Container allocated = engine.allocateContainer(request);
-
- // notify the container ane expect
- List<ContainerAssignment> assignments = new ArrayList<>();
- List<AbstractRMOperation> operations = new ArrayList<>();
- appState.onContainersAllocated(Arrays.asList(allocated), assignments,
- operations);
-
- String host = allocated.getNodeId().getHost();
- NodeInstance hostInstance = cloneNodemap().get(host);
- assertEquals(1, hostInstance.get(aaRole.getKey()).getStarting());
- assertFalse(hostInstance.canHost(aaRole.getKey(), ""));
- assertFalse(hostInstance.canHost(aaRole.getKey(), null));
-
- // assignment
- assertEquals(1, assignments.size());
-
- // verify the release matches the allocation
- assertEquals(2, operations.size());
- assertNotNull(getCancel(operations, 0).getCapability().equals(allocated
- .getResource()));
-
- // we also expect a new allocation request to have been issued
-
- ContainerRequest req2 = getRequest(operations, 1);
- assertEquals(req2.getNodes().size(), engine.getCluster()
- .getClusterSize() - 1);
-
- assertFalse(req2.getNodes().contains(host));
- assertFalse(request.getRelaxLocality());
-
- // verify the pending couner is down
- assertEquals(0L, aaRole.getAAPending());
- Container allocated2 = engine.allocateContainer(req2);
-
- // placement must be on a different host
- assertNotEquals(allocated2.getNodeId(), allocated.getNodeId());
-
- ContainerAssignment assigned = assignments.get(0);
- Container container = assigned.container;
- RoleInstance ri = roleInstance(assigned);
- //tell the app it arrived
- appState.containerStartSubmitted(container, ri);
- assertNotNull(appState.onNodeManagerContainerStarted(container.getId()));
- ops = appState.reviewRequestAndReleaseNodes();
- assertEquals(0, ops.size());
- assertAllContainersAA();
-
- // identify those hosts with an aa role on
- Map<Integer, String> naming = appState.buildNamingMap();
- assertEquals(3, naming.size());
-
- String name = aaRole.getName();
- assertEquals(name, naming.get(aaRole.getKey()));
- Map<String, NodeInformation> info =
- appState.getRoleHistory().getNodeInformationSnapshot(naming);
- assertTrue(SliderUtils.isNotEmpty(info));
-
- NodeInformation nodeInformation = info.get(host);
- assertNotNull(nodeInformation);
- assertTrue(SliderUtils.isNotEmpty(nodeInformation.entries));
- assertNotNull(nodeInformation.entries.get(name));
- assertEquals(1, nodeInformation.entries.get(name).live);
- }
-
- //@Test
- public void testAllocateFlexUp() throws Throwable {
- RoleStatus aaRole = getAaRole();
-
- // want multiple instances, so there will be iterations
- aaRole.setDesired(2);
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- getSingleRequest(ops);
- assertEquals(1, aaRole.getRequested());
- assertEquals(1, aaRole.getAAPending());
- assertEquals(aaRole.getActualAndRequested() + aaRole
- .getAAPending(), aaRole.getDesired());
-
- // now trigger that flex up
- aaRole.setDesired(3);
-
- // expect: no new reqests, pending count ++
- List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes();
- assertTrue(ops2.isEmpty());
- assertEquals(aaRole.getRunning() + aaRole.getAAPending() +
- aaRole.getOutstandingAARequestCount(), aaRole.getDesired());
-
- // 1 outstanding
- assertEquals(0, aaRole.getRunning());
- assertTrue(aaRole.isAARequestOutstanding());
- // and one AA
- assertEquals(2, aaRole.getAAPending());
- assertAllContainersAA();
-
- // next iter
- assertEquals(1, submitOperations(ops, EMPTY_ID_LIST, ops2).size());
- assertEquals(2, ops2.size());
- assertEquals(1, aaRole.getAAPending());
- assertAllContainersAA();
-
- assertEquals(0, appState.reviewRequestAndReleaseNodes().size());
- // now trigger the next execution cycle
- List<AbstractRMOperation> ops3 = new ArrayList<>();
- assertEquals(1, submitOperations(ops2, EMPTY_ID_LIST, ops3).size());
- assertEquals(2, ops3.size());
- assertEquals(0, aaRole.getAAPending());
- assertAllContainersAA();
-
- }
-
- //@Test
- public void testAllocateFlexDownDecrementsPending() throws Throwable {
- RoleStatus aaRole = getAaRole();
-
- // want multiple instances, so there will be iterations
- aaRole.setDesired(2);
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- getSingleRequest(ops);
- assertEquals(1, aaRole.getAAPending());
- assertTrue(aaRole.isAARequestOutstanding());
-
- // flex down so that the next request should be cancelled
- aaRole.setDesired(1);
-
- // expect: no new requests, pending count --
- List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes();
- assertTrue(ops2.isEmpty());
- assertTrue(aaRole.isAARequestOutstanding());
- assertEquals(0, aaRole.getAAPending());
- assertAllContainersAA();
-
- // next iter
- submitOperations(ops, EMPTY_ID_LIST, ops2).size();
- assertEquals(1, ops2.size());
- assertAllContainersAA();
- }
-
- /**
- * Here flex down while there is only one outstanding request.
- * The outstanding flex should be cancelled
- * @throws Throwable
- */
- //@Test
- public void testAllocateFlexDownForcesCancel() throws Throwable {
- RoleStatus aaRole = getAaRole();
-
- // want multiple instances, so there will be iterations
- aaRole.setDesired(1);
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- getSingleRequest(ops);
- assertEquals(1, aaRole.getRequested());
- assertEquals(0, aaRole.getAAPending());
- assertTrue(aaRole.isAARequestOutstanding());
-
- // flex down so that the next request should be cancelled
- aaRole.setDesired(0);
- // expect: no new requests, pending count --
- List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes();
- assertEquals(0, aaRole.getRequested());
- assertEquals(0, aaRole.getAAPending());
- assertFalse(aaRole.isAARequestOutstanding());
- assertEquals(1, ops2.size());
- getSingleCancel(ops2);
-
- // next iter
- submitOperations(ops, EMPTY_ID_LIST, ops2).size();
- getSingleRelease(ops2);
- }
-
- void assertAllContainersAA() {
- assertAllContainersAA(getAaRole().getKey());
- }
-
- /**
- *
- * @throws Throwable
- */
- //@Test
- public void testAskForTooMany() throws Throwable {
- RoleStatus aaRole = getAaRole();
-
- describe("Ask for 1 more than the no of available nodes;" +
- " expect the final request to be unsatisfied until the cluster " +
- "changes size");
- //more than expected
- aaRole.setDesired(NODES + 1);
- List<AbstractRMOperation > operations = appState
- .reviewRequestAndReleaseNodes();
- assertTrue(aaRole.isAARequestOutstanding());
- assertEquals(NODES, aaRole.getAAPending());
- for (int i = 0; i < NODES; i++) {
- String iter = "Iteration " + i + " role = " + aaRole;
- LOG.info(iter);
- List<AbstractRMOperation > operationsOut = new ArrayList<>();
- assertEquals(1, submitOperations(operations, EMPTY_ID_LIST,
- operationsOut).size());
- operations = operationsOut;
- if (i + 1 < NODES) {
- assertEquals(2, operations.size());
- } else {
- assertEquals(1, operations.size());
- }
- assertAllContainersAA();
- }
- // expect an outstanding AA request to be unsatisfied
- assertTrue(aaRole.getRunning() < aaRole.getDesired());
- assertEquals(0, aaRole.getRequested());
- assertFalse(aaRole.isAARequestOutstanding());
- List<Container> allocatedContainers = engine.execute(operations,
- EMPTY_ID_LIST);
- assertEquals(0, allocatedContainers.size());
- // in a review now, no more requests can be generated, as there is no
- // space for AA placements, even though there is cluster capacity
- assertEquals(0, appState.reviewRequestAndReleaseNodes().size());
-
- // now do a node update (this doesn't touch the YARN engine; the node
- // isn't really there)
- NodeUpdatedOutcome outcome = addNewNode();
- assertEquals(cloneNodemap().size(), NODES + 1);
- assertTrue(outcome.clusterChanged);
- // no active calls to empty
- assertTrue(outcome.operations.isEmpty());
- assertEquals(1, appState.reviewRequestAndReleaseNodes().size());
- }
-
- protected AppState.NodeUpdatedOutcome addNewNode() {
- return updateNodes(MockFactory.INSTANCE.newNodeReport("4", NodeState
- .RUNNING, "gpu"));
- }
-
- //@Test
- public void testClusterSizeChangesDuringRequestSequence() throws Throwable {
- RoleStatus aaRole = getAaRole();
- describe("Change the cluster size where the cluster size changes during " +
- "a test sequence.");
- aaRole.setDesired(NODES + 1);
- appState.reviewRequestAndReleaseNodes();
- assertTrue(aaRole.isAARequestOutstanding());
- assertEquals(NODES, aaRole.getAAPending());
- NodeUpdatedOutcome outcome = addNewNode();
- assertTrue(outcome.clusterChanged);
- // one call to cancel
- assertEquals(1, outcome.operations.size());
- // and on a review, one more to rebuild
- assertEquals(1, appState.reviewRequestAndReleaseNodes().size());
- }
-
- //@Test
- public void testBindingInfoMustHaveNodeMap() throws Throwable {
- AppStateBindingInfo bindingInfo = buildBindingInfo();
- bindingInfo.nodeReports = null;
- try {
- MockAppState state = new MockAppState(bindingInfo);
- fail("Expected an exception, got " + state);
- } catch (IllegalArgumentException expected) {
- }
- }
-
- //@Test
- public void testAMRestart() throws Throwable {
- int desiredAA = 3;
- getAaRole().setDesired(desiredAA);
- List<RoleInstance> instances = createAndStartNodes();
- List<Container> containers = new ArrayList<>();
- for (RoleInstance instance : instances) {
- containers.add(instance.container);
- }
-
- // now destroy the app state
- AppStateBindingInfo bindingInfo = buildBindingInfo();
- bindingInfo.application = factory.newApplication(0, 0, desiredAA).name(
- getValidTestName());
- bindingInfo.application.getComponent(ROLE2)
- .getConfiguration().setProperty(COMPONENT_PLACEMENT_POLICY,
- Integer.toString(PlacementPolicy.ANTI_AFFINITY_REQUIRED));
- bindingInfo.liveContainers = containers;
- appState = new MockAppState(bindingInfo);
-
- RoleStatus aaRole = lookupRole(AAROLE_2.name);
- RoleStatus gpuRole = lookupRole(MockFactory.AAROLE_1_GPU.name);
- appState.reviewRequestAndReleaseNodes();
- assertTrue(aaRole.isAntiAffinePlacement());
- assertTrue(aaRole.isAARequestOutstanding());
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java
deleted file mode 100644
index 41ff0fa..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java
+++ /dev/null
@@ -1,387 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.server.appmaster.actions.ResetFailureWindow;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockAM;
-import org.apache.slider.server.appmaster.model.mock.MockAppState;
-import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
-import org.apache.slider.server.appmaster.state.ContainerOutcome;
-import org.apache.slider.server.appmaster.state.NodeEntry;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.RoleHistory;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * Test that if you have >1 role, the right roles are chosen for release.
- */
-public class TestMockAppStateContainerFailure extends BaseMockAppStateTest
- implements MockRoles {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestMockAppStateContainerFailure.class);
-
- private MockRMOperationHandler operationHandler = new
- MockRMOperationHandler();
- private MockAM mockAM = new MockAM();
-
- @Override
- public String getTestName() {
- return "TestMockAppStateContainerFailure";
- }
-
- /**
- * Small cluster with multiple containers per node,
- * to guarantee many container allocations on each node.
- * @return
- */
- @Override
- public MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(4, 8000);
- }
-
- @Override
- public Application buildApplication() {
- Application application = super.buildApplication();
- application.getConfiguration().setProperty(
- ResourceKeys.CONTAINER_FAILURE_THRESHOLD, "10");
- return application;
- }
-
- //@Test
- public void testShortLivedFail() throws Throwable {
-
- getRole0Status().setDesired(1);
- List<RoleInstance> instances = createAndStartNodes();
- assertEquals(1, instances.size());
-
- RoleInstance instance = instances.get(0);
- long created = instance.createTime;
- long started = instance.startTime;
- assertTrue(created > 0);
- assertTrue(started >= created);
- List<ContainerId> ids = extractContainerIds(instances, ROLE0);
-
- ContainerId cid = ids.get(0);
- assertTrue(appState.isShortLived(instance));
- AppState.NodeCompletionResult result = appState.onCompletedContainer(
- containerStatus(cid, 1));
- assertNotNull(result.roleInstance);
- assertTrue(result.containerFailed);
- RoleStatus status = getRole0Status();
- assertEquals(1, status.getFailed());
-// assertEquals(1, status.getStartFailed());
-
- //view the world
- appState.getRoleHistory().dump();
- List<NodeInstance> queue = appState.getRoleHistory().cloneRecentNodeList(
- getRole0Status().getKey());
- assertEquals(0, queue.size());
-
- }
-
- //@Test
- public void testLongLivedFail() throws Throwable {
-
- getRole0Status().setDesired(1);
- List<RoleInstance> instances = createAndStartNodes();
- assertEquals(1, instances.size());
-
- RoleInstance instance = instances.get(0);
- instance.startTime = System.currentTimeMillis() - 60 * 60 * 1000;
- assertFalse(appState.isShortLived(instance));
- List<ContainerId> ids = extractContainerIds(instances, ROLE0);
-
- ContainerId cid = ids.get(0);
- AppState.NodeCompletionResult result = appState.onCompletedContainer(
- containerStatus(cid, 1));
- assertNotNull(result.roleInstance);
- assertTrue(result.containerFailed);
- RoleStatus status = getRole0Status();
- assertEquals(1, status.getFailed());
-// assertEquals(0, status.getStartFailed());
-
- //view the world
- appState.getRoleHistory().dump();
- List<NodeInstance> queue = appState.getRoleHistory().cloneRecentNodeList(
- getRole0Status().getKey());
- assertEquals(1, queue.size());
-
- }
-
- //@Test
- public void testNodeStartFailure() throws Throwable {
-
- getRole0Status().setDesired(1);
- List<RoleInstance> instances = createAndSubmitNodes();
- assertEquals(1, instances.size());
-
- RoleInstance instance = instances.get(0);
-
- List<ContainerId> ids = extractContainerIds(instances, ROLE0);
-
- ContainerId cid = ids.get(0);
- appState.onNodeManagerContainerStartFailed(cid, new SliderException(
- "oops"));
- RoleStatus status = getRole0Status();
- assertEquals(1, status.getFailed());
-// assertEquals(1, status.getStartFailed());
-
-
- RoleHistory history = appState.getRoleHistory();
- history.dump();
- List<NodeInstance> queue = history.cloneRecentNodeList(getRole0Status()
- .getKey());
- assertEquals(0, queue.size());
-
- NodeInstance ni = history.getOrCreateNodeInstance(instance.container);
- NodeEntry re = ni.get(getRole0Status().getKey());
- assertEquals(1, re.getFailed());
- assertEquals(1, re.getStartFailed());
- }
-
- //@Test
- public void testRecurrentStartupFailure() throws Throwable {
-
- getRole0Status().setDesired(1);
- try {
- for (int i = 0; i< 100; i++) {
- List<RoleInstance> instances = createAndSubmitNodes();
- assertEquals(1, instances.size());
-
- List<ContainerId> ids = extractContainerIds(instances, ROLE0);
-
- ContainerId cid = ids.get(0);
- LOG.info("{} instance {} {}", i, instances.get(0), cid);
- assertNotNull(cid);
- appState.onNodeManagerContainerStartFailed(cid,
- new SliderException("failure #" + i));
- AppState.NodeCompletionResult result = appState.onCompletedContainer(
- containerStatus(cid));
- assertTrue(result.containerFailed);
- }
- fail("Cluster did not fail from too many startup failures");
- } catch (TriggerClusterTeardownException teardown) {
- LOG.info("Exception {} : {}", teardown.getExitCode(), teardown);
- }
- }
-
- //@Test
- public void testRecurrentStartupFailureWithUnlimitedFailures() throws
- Throwable {
- // Update instance definition to allow containers to fail any number of
- // times
- AppStateBindingInfo bindingInfo = buildBindingInfo();
- bindingInfo.application.getComponent(ROLE0).getConfiguration().setProperty(
- ResourceKeys.CONTAINER_FAILURE_THRESHOLD, "0");
- appState = new MockAppState(bindingInfo);
-
- getRole0Status().setDesired(1);
- try {
- for (int i = 0; i < 100; i++) {
- List<RoleInstance> instances = createAndSubmitNodes();
- assertEquals(1, instances.size());
-
- List<ContainerId> ids = extractContainerIds(instances, ROLE0);
-
- ContainerId cid = ids.get(0);
- LOG.info("{} instance {} {}", i, instances.get(0), cid);
- assertNotNull(cid);
- appState.onNodeManagerContainerStartFailed(cid,
- new SliderException("failure #" + i));
- AppState.NodeCompletionResult result = appState.onCompletedContainer(
- containerStatus(cid));
- assertTrue(result.containerFailed);
- }
- } catch (TriggerClusterTeardownException teardown) {
- LOG.info("Exception {} : {}", teardown.getExitCode(), teardown);
- fail("Cluster failed despite " + ResourceKeys
- .CONTAINER_FAILURE_THRESHOLD + " = 0");
- }
- }
-
- //@Test
- public void testRoleStatusFailureWindow() throws Throwable {
-
- ResetFailureWindow resetter = new ResetFailureWindow(operationHandler);
-
- // initial reset
- resetter.execute(mockAM, null, appState);
-
- getRole0Status().setDesired(1);
- for (int i = 0; i < 100; i++) {
- resetter.execute(mockAM, null, appState);
- List<RoleInstance> instances = createAndSubmitNodes();
- assertEquals(1, instances.size());
-
- List<ContainerId> ids = extractContainerIds(instances, ROLE0);
-
- ContainerId cid = ids.get(0);
- LOG.info("{} instance {} {}", i, instances.get(0), cid);
- assertNotNull(cid);
- appState.onNodeManagerContainerStartFailed(
- cid,
- new SliderException("failure #" + i));
- AppState.NodeCompletionResult result = appState.onCompletedContainer(
- containerStatus(cid));
- assertTrue(result.containerFailed);
- }
- }
-
- //@Test
- public void testRoleStatusFailed() throws Throwable {
- RoleStatus status = getRole0Status();
- // limits exceeded
- appState.incFailedContainers(status, ContainerOutcome.Failed);
- assertEquals(1, status.getFailed());
- assertEquals(1L, status.getFailedRecently());
- assertEquals(0L, status.getLimitsExceeded());
- assertEquals(0L, status.getPreempted());
- assertEquals(0L, status.getDiskFailed());
-
- ResetFailureWindow resetter = new ResetFailureWindow(operationHandler);
- resetter.execute(mockAM, null, appState);
- assertEquals(1, status.getFailed());
- assertEquals(0L, status.getFailedRecently());
- }
-
- //@Test
- public void testRoleStatusFailedLimitsExceeded() throws Throwable {
- RoleStatus status = getRole0Status();
- // limits exceeded
- appState.incFailedContainers(status, ContainerOutcome
- .Failed_limits_exceeded);
- assertEquals(1, status.getFailed());
- assertEquals(1L, status.getFailedRecently());
- assertEquals(1L, status.getLimitsExceeded());
- assertEquals(0L, status.getPreempted());
- assertEquals(0L, status.getDiskFailed());
-
- ResetFailureWindow resetter = new ResetFailureWindow(operationHandler);
- resetter.execute(mockAM, null, appState);
- assertEquals(1, status.getFailed());
- assertEquals(0L, status.getFailedRecently());
- assertEquals(1L, status.getLimitsExceeded());
- }
-
-
- //@Test
- public void testRoleStatusFailedPrempted() throws Throwable {
- RoleStatus status = getRole0Status();
- // limits exceeded
- appState.incFailedContainers(status, ContainerOutcome.Preempted);
- assertEquals(0, status.getFailed());
- assertEquals(1L, status.getPreempted());
- assertEquals(0L, status.getFailedRecently());
- assertEquals(0L, status.getDiskFailed());
-
- ResetFailureWindow resetter = new ResetFailureWindow(operationHandler);
- resetter.execute(mockAM, null, appState);
- assertEquals(1L, status.getPreempted());
- }
-
-
- //@Test
- public void testRoleStatusFailedNode() throws Throwable {
- RoleStatus status = getRole0Status();
- // limits exceeded
- appState.incFailedContainers(status, ContainerOutcome.Disk_failure);
- assertEquals(1, status.getFailed());
- assertEquals(0L, status.getFailedRecently());
- assertEquals(0L, status.getLimitsExceeded());
- assertEquals(0L, status.getPreempted());
- assertEquals(1L, status.getDiskFailed());
- }
-
- //@Test
- public void testNodeEntryCompleted() throws Throwable {
- NodeEntry nodeEntry = new NodeEntry(1);
- nodeEntry.containerCompleted(true, ContainerOutcome.Completed);
- assertEquals(0, nodeEntry.getFailed());
- assertEquals(0, nodeEntry.getFailedRecently());
- assertEquals(0, nodeEntry.getStartFailed());
- assertEquals(0, nodeEntry.getPreempted());
- assertEquals(0, nodeEntry.getActive());
- assertTrue(nodeEntry.isAvailable());
- }
-
- //@Test
- public void testNodeEntryFailed() throws Throwable {
- NodeEntry nodeEntry = new NodeEntry(1);
- nodeEntry.containerCompleted(false, ContainerOutcome.Failed);
- assertEquals(1, nodeEntry.getFailed());
- assertEquals(1, nodeEntry.getFailedRecently());
- assertEquals(0, nodeEntry.getStartFailed());
- assertEquals(0, nodeEntry.getPreempted());
- assertEquals(0, nodeEntry.getActive());
- assertTrue(nodeEntry.isAvailable());
- nodeEntry.resetFailedRecently();
- assertEquals(1, nodeEntry.getFailed());
- assertEquals(0, nodeEntry.getFailedRecently());
- }
-
- //@Test
- public void testNodeEntryLimitsExceeded() throws Throwable {
- NodeEntry nodeEntry = new NodeEntry(1);
- nodeEntry.containerCompleted(false, ContainerOutcome
- .Failed_limits_exceeded);
- assertEquals(0, nodeEntry.getFailed());
- assertEquals(0, nodeEntry.getFailedRecently());
- assertEquals(0, nodeEntry.getStartFailed());
- assertEquals(0, nodeEntry.getPreempted());
- }
-
- //@Test
- public void testNodeEntryPreempted() throws Throwable {
- NodeEntry nodeEntry = new NodeEntry(1);
- nodeEntry.containerCompleted(false, ContainerOutcome.Preempted);
- assertEquals(0, nodeEntry.getFailed());
- assertEquals(0, nodeEntry.getFailedRecently());
- assertEquals(0, nodeEntry.getStartFailed());
- assertEquals(1, nodeEntry.getPreempted());
- }
-
- //@Test
- public void testNodeEntryNodeFailure() throws Throwable {
- NodeEntry nodeEntry = new NodeEntry(1);
- nodeEntry.containerCompleted(false, ContainerOutcome.Disk_failure);
- assertEquals(1, nodeEntry.getFailed());
- assertEquals(1, nodeEntry.getFailedRecently());
- assertEquals(0, nodeEntry.getStartFailed());
- assertEquals(0, nodeEntry.getPreempted());
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java
deleted file mode 100644
index cbef2be..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.slider.api.types.ApplicationLivenessInformation;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.apache.slider.server.servicemonitor.ProbeStatus;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Test for postponing container requests until dependencies are ready.
- */
-public class TestMockAppStateDependencies extends BaseMockAppStateTest
- implements MockRoles {
-
- private org.apache.slider.server.servicemonitor.Probe successProbe =
- new org.apache.slider.server.servicemonitor.Probe("success", null) {
- @Override
- public ProbeStatus ping(ComponentInstance roleInstance) {
- ProbeStatus status = new ProbeStatus();
- status.succeed(this);
- return status;
- }
- };
-
- private org.apache.slider.server.servicemonitor.Probe failureProbe =
- new org.apache.slider.server.servicemonitor.Probe("failure", null) {
- @Override
- public ProbeStatus ping(ComponentInstance roleInstance) {
- ProbeStatus status = new ProbeStatus();
- status.fail(this, new Exception());
- return status;
- }
- };
-
- @Override
- public String getTestName() {
- return "TestMockAppStateDependencies";
- }
-
- //@Test
- public void testDependencies() throws Throwable {
- RoleStatus role0Status = getRole0Status();
- RoleStatus role1Status = getRole1Status();
-
- // set desired instances for role0 to 1
- role0Status.setDesired(1);
- // set probe for role0 to use a ping that will always succeed
- role0Status.getProviderRole().probe = successProbe;
-
- // set desired instances for role1 to 1
- role1Status.setDesired(1);
- // set role0 as a dependency of role1
- role1Status.getProviderRole().component.setDependencies(Collections
- .singletonList(ROLE0));
-
- // role0 has no dependencies, so its dependencies are ready
- assertTrue(appState.areDependenciesReady(role0Status));
- // role1 dependency (role0) is not ready yet
- assertFalse(appState.areDependenciesReady(role1Status));
- // start the single requested instance for role0
- review(ROLE0, 2);
-
- // role0 is still not ready because a ping has not been issued
- assertFalse(appState.areDependenciesReady(role1Status));
- // issue pings
- appState.monitorComponentInstances();
- // now role0 is ready
- assertTrue(appState.areDependenciesReady(role1Status));
- // increase the desired containers for role0
- role0Status.setDesired(2);
- // role0 is no longer ready
- assertFalse(appState.areDependenciesReady(role1Status));
- // start a second instance for role0
- review(ROLE0, 2);
-
- // role0 is not ready because ping has not been issued for the new instance
- assertFalse(appState.areDependenciesReady(role1Status));
- // issue pings
- appState.monitorComponentInstances();
- // role0 is ready
- assertTrue(appState.areDependenciesReady(role1Status));
-
- // set probe for role0 to use a ping that will always fail
- role0Status.getProviderRole().probe = failureProbe;
- // issue pings
- appState.monitorComponentInstances();
- // role0 is not ready (failure probe works)
- assertFalse(appState.areDependenciesReady(role1Status));
- // set probe for role0 to use a ping that will always succeed
- role0Status.getProviderRole().probe = successProbe;
- // issue pings
- appState.monitorComponentInstances();
- // role0 is ready
- assertTrue(appState.areDependenciesReady(role1Status));
-
- // now role1 instances can be started
- review(ROLE1, 1);
- }
-
- public void review(String expectedRole, int outstanding) throws Exception {
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
-
- // expect one request in the list
- assertEquals(1, ops.size());
- // and in a liveness check, expected outstanding
- ApplicationLivenessInformation liveness =
- appState.getApplicationLivenessInformation();
- assertEquals(outstanding, liveness.requestsOutstanding);
- assertFalse(liveness.allRequestsSatisfied);
-
- // record container allocated and verify it has the expected role
- List<Container> allocations = engine.execute(ops);
- List<ContainerAssignment> assignments = new ArrayList<>();
- List<AbstractRMOperation> releases = new ArrayList<>();
- appState.onContainersAllocated(allocations, assignments, releases);
- assertEquals(1, assignments.size());
- ContainerAssignment assigned = assignments.get(0);
- Container target = assigned.container;
- RoleInstance ri = roleInstance(assigned);
- assertEquals(expectedRole, ri.role);
-
- // one fewer request outstanding
- liveness = appState.getApplicationLivenessInformation();
- assertEquals(outstanding - 1, liveness.requestsOutstanding);
-
- // record container start submitted
- appState.containerStartSubmitted(target, ri);
-
- // additional review results in no additional requests
- ops = appState.reviewRequestAndReleaseNodes();
- assertTrue(ops.isEmpty());
-
- // record container start
- appState.innerOnNodeManagerContainerStarted(target.getId());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java
deleted file mode 100644
index 76f3f37..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.providers.PlacementPolicy;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.NodeEntry;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.RoleHistory;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Test that if you have >1 role, the right roles are chosen for release.
- */
-public class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
- implements MockRoles {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestMockAppStateDynamicHistory.class);
-
- /**
- * Small cluster with multiple containers per node,
- * to guarantee many container allocations on each node.
- * @return
- */
- @Override
- public MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(8, 1);
- }
-
- // TODO does not support adding new components dynamically
- public void testDynamicRoleHistory() throws Throwable {
-
- String dynamic = "dynamicRole";
- long desired = 1;
- int placementPolicy = PlacementPolicy.DEFAULT;
- // snapshot and patch existing spec
- Application application = appState.getClusterStatus();
- Component component = new Component().name(dynamic).numberOfContainers(
- desired);
- component.getConfiguration().setProperty(ResourceKeys
- .COMPONENT_PLACEMENT_POLICY, "" + placementPolicy);
- application.getComponents().add(component);
-
- appState.updateComponents(
- Collections.singletonMap(dynamic, desired));
-
- // now look at the role map
- assertNotNull(appState.getRoleMap().get(dynamic));
- ProviderRole mappedRole = appState.getRoleMap().get(dynamic);
- int rolePriority = mappedRole.id;
-
- Map<Integer, ProviderRole> priorityMap = appState.getRolePriorityMap();
- assertEquals(priorityMap.size(), 4);
- ProviderRole dynamicProviderRole = priorityMap.get(rolePriority);
- assertNotNull(dynamicProviderRole);
- assertEquals(dynamicProviderRole.id, rolePriority);
-
- assertNotNull(appState.getRoleStatusMap().get(rolePriority));
- RoleStatus dynamicRoleStatus =
- appState.getRoleStatusMap().get(rolePriority);
- assertEquals(dynamicRoleStatus.getDesired(), desired);
-
-
- // before allocating the nodes, fill up the capacity of some of the
- // hosts
- engine.getAllocator().nextIndex();
-
- int targetNode = 2;
- assertEquals(targetNode, engine.getAllocator().nextIndex());
- String targetHostname = engine.getCluster().nodeAt(targetNode)
- .getHostname();
-
- // clock is set to a small value
- appState.setTime(100000);
-
- // allocate the nodes
- List<AbstractRMOperation> actions = appState.reviewRequestAndReleaseNodes();
- assertEquals(1, actions.size());
- ContainerRequestOperation action0 = (ContainerRequestOperation)actions
- .get(0);
-
- ContainerRequest request = action0.getRequest();
- assertTrue(SliderUtils.isEmpty(request.getNodes()));
-
- List<ContainerId> released = new ArrayList<>();
- List<RoleInstance> allocations = submitOperations(actions, released);
- processSubmissionOperations(allocations, new ArrayList<>(), released);
- assertEquals(1, allocations.size());
- RoleInstance ri = allocations.get(0);
-
- assertEquals(ri.role, dynamic);
- assertEquals(ri.roleId, rolePriority);
- assertEquals(ri.host, targetHostname);
-
- // now look at the role history
-
- RoleHistory roleHistory = appState.getRoleHistory();
- List<NodeInstance> activeNodes = roleHistory.listActiveNodes(
- rolePriority);
- assertEquals(activeNodes.size(), 1);
- NodeInstance activeNode = activeNodes.get(0);
- assertNotNull(activeNode.get(rolePriority));
- NodeEntry entry8 = activeNode.get(rolePriority);
- assertEquals(entry8.getActive(), 1);
-
- assertEquals(activeNode.hostname, targetHostname);
-
- NodeInstance activeNodeInstance =
- roleHistory.getOrCreateNodeInstance(ri.container);
-
- assertEquals(activeNode, activeNodeInstance);
- NodeEntry entry = activeNodeInstance.get(rolePriority);
- assertNotNull(entry);
- assertTrue(entry.getActive() > 0);
- assertTrue(entry.getLive() > 0);
-
-
- // now trigger a termination event on that role
-
- // increment time for a long-lived failure event
- appState.incTime(100000);
-
- LOG.debug("Triggering failure");
- ContainerId cid = ri.getContainerId();
- AppState.NodeCompletionResult result = appState.onCompletedContainer(
- containerStatus(cid, 1));
- assertEquals(result.roleInstance, ri);
- assertTrue(result.containerFailed);
-
- roleHistory.dump();
- // values should have changed
- assertEquals(1, entry.getFailed());
- assertEquals(0, entry.getStartFailed());
- assertEquals(0, entry.getActive());
- assertEquals(0, entry.getLive());
-
-
- List<NodeInstance> nodesForRoleId =
- roleHistory.getRecentNodesForRoleId(rolePriority);
- assertNotNull(nodesForRoleId);
-
- // make sure new nodes will default to a different host in the engine
- assertTrue(targetNode < engine.getAllocator().nextIndex());
-
- actions = appState.reviewRequestAndReleaseNodes();
- assertEquals(1, actions.size());
- ContainerRequestOperation action1 = (ContainerRequestOperation) actions
- .get(0);
- ContainerRequest request1 = action1.getRequest();
- assertTrue(SliderUtils.isNotEmpty(request1.getNodes()));
- }
-
- //@Test(expected = BadConfigException.class)
- public void testRoleHistoryRoleAdditions() throws Throwable {
- MockRoleHistory roleHistory = new MockRoleHistory(new ArrayList<>());
- roleHistory.addNewRole(new RoleStatus(new ProviderRole("one", 1)));
- roleHistory.addNewRole(new RoleStatus(new ProviderRole("two", 1)));
- roleHistory.dump();
- }
-
- //@Test(expected = BadConfigException.class)
- public void testRoleHistoryRoleStartupConflict() throws Throwable {
- MockRoleHistory roleHistory = new MockRoleHistory(Arrays.asList(
- new ProviderRole("one", 1), new ProviderRole("two", 1)
- ));
- roleHistory.dump();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.java
deleted file mode 100644
index 5669d2d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.providers.PlacementPolicy;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.apache.slider.server.appmaster.state.AppState.NodeCompletionResult;
-import org.apache.slider.server.appmaster.state.ContainerPriority;
-import org.apache.slider.server.appmaster.state.RoleHistoryUtils;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.slider.server.appmaster.model.mock.MockFactory.NODE_FAILURE_THRESHOLD;
-
-/**
- * Test that if you have >1 role, the right roles are chosen for release.
- */
-public class TestMockAppStateDynamicRoles extends BaseMockAppStateTest
- implements MockRoles {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestMockAppStateDynamicRoles.class);
- private static final String ROLE4 = "4";
- private static final String ROLE5 = "5";
-
- @Override
- public String getTestName() {
- return "TestMockAppStateDynamicRoles";
- }
-
- /**
- * Small cluster with multiple containers per node,
- * to guarantee many container allocations on each node.
- * @return
- */
- @Override
- public MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(8, 2);
- }
-
- @Override
- public Application buildApplication() {
- Application application = super.buildApplication();
-
- Component component = new Component().name(ROLE4).numberOfContainers(1L);
- component.getConfiguration().setProperty(ResourceKeys
- .NODE_FAILURE_THRESHOLD, Integer.toString(3));
- application.getComponents().add(component);
-
- component = new Component().name(ROLE5).numberOfContainers(1L);
- component.getConfiguration().setProperty(ResourceKeys
- .COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy.STRICT));
- application.getComponents().add(component);
-
- return application;
- }
-
- //@Test
- public void testAllocateReleaseRealloc() throws Throwable {
-
- createAndStartNodes();
- appState.reviewRequestAndReleaseNodes();
- appState.getRoleHistory().dump();
- }
-
- /**
- * Find all allocations for a specific role.
- * @param role role Id/priority
- * @param actions source list
- * @return found list
- */
- List<ContainerRequestOperation> findAllocationsForRole(int role,
- List<AbstractRMOperation> actions) {
- List<ContainerRequestOperation> ops = new ArrayList<>();
- for (AbstractRMOperation op : actions) {
- if (op instanceof ContainerRequestOperation && role ==
- ContainerPriority.extractRole(((ContainerRequestOperation) op)
- .getRequest().getPriority())) {
- ops.add((ContainerRequestOperation) op);
- }
- }
- return ops;
- }
-
- //@Test
- public void testStrictPlacementInitialRequest() throws Throwable {
- LOG.info("Initial engine state = {}", engine);
- List<AbstractRMOperation> actions = appState.reviewRequestAndReleaseNodes();
- assertEquals(2, actions.size());
-
- // neither have locality at this point
- assertRelaxLocalityFlag(appState.lookupRoleStatus(ROLE4).getKey(), null,
- true, actions);
- assertRelaxLocalityFlag(appState.lookupRoleStatus(ROLE5).getKey(), null,
- true, actions);
- }
-
- //@Test
- public void testPolicyPropagation() throws Throwable {
- assertEquals(0, (appState.lookupRoleStatus(ROLE4).getPlacementPolicy() &
- PlacementPolicy.STRICT));
- assertNotEquals(0, (appState.lookupRoleStatus(ROLE5).getPlacementPolicy() &
- PlacementPolicy.STRICT));
-
- }
-
- //@Test
- public void testNodeFailureThresholdPropagation() throws Throwable {
- assertEquals(3, appState.lookupRoleStatus(ROLE4).getNodeFailureThreshold());
- assertEquals(NODE_FAILURE_THRESHOLD, appState.lookupRoleStatus(ROLE5)
- .getNodeFailureThreshold());
- }
-
- //@Test
- public void testLaxPlacementSecondRequestRole4() throws Throwable {
- LOG.info("Initial engine state = {}", engine);
- RoleStatus role4 = appState.lookupRoleStatus(ROLE4);
- RoleStatus role5 = appState.lookupRoleStatus(ROLE5);
- role4.setDesired(1);
- role5.setDesired(0);
-
- List<RoleInstance> instances = createStartAndStopNodes(new ArrayList<>());
- assertEquals(1, instances.size());
-
- int id = appState.lookupRoleStatus(ROLE4).getKey();
- RoleInstance instanceA = null;
- for (RoleInstance instance : instances) {
- if (instance.roleId == id) {
- instanceA = instance;
- }
- }
- assertNotNull(instanceA);
- String hostname = RoleHistoryUtils.hostnameOf(instanceA.container);
-
- LOG.info("Allocated engine state = {}", engine);
- assertEquals(1, engine.containerCount());
-
- assertEquals(1, role4.getRunning());
- // shrinking cluster
-
- role4.setDesired(0);
- appState.lookupRoleStatus(ROLE4).setDesired(0);
- List<NodeCompletionResult> completionResults = new ArrayList<>();
- createStartAndStopNodes(completionResults);
- assertEquals(0, engine.containerCount());
- assertEquals(1, completionResults.size());
-
- // expanding: expect hostnames now
- role4.setDesired(1);
- List<AbstractRMOperation> actions = appState.reviewRequestAndReleaseNodes();
- assertEquals(1, actions.size());
-
- ContainerRequestOperation cro = (ContainerRequestOperation) actions.get(0);
- List<String> nodes = cro.getRequest().getNodes();
- assertEquals(1, nodes.size());
- assertEquals(hostname, nodes.get(0));
- }
-
- //@Test
- public void testStrictPlacementSecondRequestRole5() throws Throwable {
- LOG.info("Initial engine state = {}", engine);
- RoleStatus role4 = appState.lookupRoleStatus(ROLE4);
- RoleStatus role5 = appState.lookupRoleStatus(ROLE5);
- role4.setDesired(0);
- role5.setDesired(1);
-
- List<RoleInstance> instances = createStartAndStopNodes(new ArrayList<>());
- assertEquals(1, instances.size());
-
- int id = appState.lookupRoleStatus(ROLE5).getKey();
- RoleInstance instanceA = null;
- for (RoleInstance instance : instances) {
- if (instance.roleId == id) {
- instanceA = instance;
- }
- }
- assertNotNull(instanceA);
- String hostname = RoleHistoryUtils.hostnameOf(instanceA.container);
-
- LOG.info("Allocated engine state = {}", engine);
- assertEquals(1, engine.containerCount());
-
- assertEquals(1, role5.getRunning());
-
- // shrinking cluster
- role5.setDesired(0);
- List<NodeCompletionResult> completionResults = new ArrayList<>();
- createStartAndStopNodes(completionResults);
- assertEquals(0, engine.containerCount());
- assertEquals(1, completionResults.size());
- assertEquals(0, role5.getRunning());
-
- role5.setDesired(1);
- List<AbstractRMOperation> actions = appState.reviewRequestAndReleaseNodes();
- assertEquals(1, actions.size());
- assertRelaxLocalityFlag(id, "", false, actions);
- ContainerRequestOperation cro = (ContainerRequestOperation) actions.get(0);
- List<String> nodes = cro.getRequest().getNodes();
- assertEquals(1, nodes.size());
- assertEquals(hostname, nodes.get(0));
- }
-
- public void assertRelaxLocalityFlag(
- int role,
- String expectedHost,
- boolean expectedRelaxFlag,
- List<AbstractRMOperation> actions) {
- List<ContainerRequestOperation> requests = findAllocationsForRole(
- role, actions);
- assertEquals(1, requests.size());
- ContainerRequestOperation req = requests.get(0);
- assertEquals(expectedRelaxFlag, req.getRequest().getRelaxLocality());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java
deleted file mode 100644
index 3304da1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.core.exceptions.SliderInternalStateException;
-import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockAppState;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
-import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
-import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
-import org.apache.slider.server.appmaster.state.RoleHistory;
-import org.apache.slider.server.avro.LoadedRoleHistory;
-import org.apache.slider.server.avro.RoleHistoryWriter;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-
-/**
- * Test that if you have more than one role, the right roles are chosen for
- * release.
- */
-public class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
- implements MockRoles {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestMockAppStateFlexDynamicRoles.class);
-
- @Override
- public String getTestName() {
- return "TestMockAppStateFlexDynamicRoles";
- }
-
- /**
- * Small cluster with multiple containers per node,
- * to guarantee many container allocations on each node.
- * @return
- */
- @Override
- public MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(4, 4);
- }
-
- @Override
- public AppStateBindingInfo buildBindingInfo() throws IOException {
- AppStateBindingInfo bindingInfo = super.buildBindingInfo();
- bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector();
- return bindingInfo;
- }
-
- @Override
- public Application buildApplication() {
- Application application = super.buildApplication();
- Component component = new Component().name("dynamic-6")
- .numberOfContainers(1L);
- application.getComponents().add(component);
-
- return application;
- }
-
- @Before
- public void init()
- throws TriggerClusterTeardownException, SliderInternalStateException {
- createAndStartNodes();
- }
-
- // TODO does not support adding new components dynamically
- public void testDynamicFlexAddRole() throws Throwable {
- Application application = appState.getClusterStatus();
- Component component = new Component().name("dynamicAdd7")
- .numberOfContainers(1L);
- application.getComponents().add(component);
- appState.updateComponents(Collections.singletonMap(component.getName(),
- component.getNumberOfContainers()));
- createAndStartNodes();
- appState.lookupRoleStatus("dynamicAdd7");
- }
-
- //@Test
- public void testDynamicFlexDropRole() throws Throwable {
- appState.updateComponents(Collections.singletonMap("dynamic-6", 0L));
- //status is retained for future
- appState.lookupRoleStatus("dynamic-6");
- }
-
-
- //@Test
- public void testHistorySaveFlexLoad() throws Throwable {
- Application application = appState.getClusterStatus();
- RoleHistory roleHistory = appState.getRoleHistory();
- Path history = roleHistory.saveHistory(0x0001);
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- Component component = new Component().name("HistorySaveFlexLoad")
- .numberOfContainers(1L);
- application.getComponents().add(component);
-
- appState.updateComponents(Collections.singletonMap(component.getName(),
- component.getNumberOfContainers()));
- createAndStartNodes();
- LoadedRoleHistory loadedRoleHistory =
- historyWriter.read(fs, history);
- assertEquals(0, appState.getRoleHistory().rebuild(loadedRoleHistory));
- }
-
- //@Test
- public void testHistoryFlexSaveResetLoad() throws Throwable {
- Application application = appState.getClusterStatus();
- Component component = new Component().name("HistoryFlexSaveLoad")
- .numberOfContainers(1L);
- application.getComponents().add(component);
-
- appState.updateComponents(Collections.singletonMap(component.getName(),
- component.getNumberOfContainers()));
- createAndStartNodes();
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- RoleHistory roleHistory = appState.getRoleHistory();
- Path history = roleHistory.saveHistory(0x0002);
- //now reset the app state
- File historyWorkDir2 = new File("target/history" + getTestName() +
- "-0002");
- Path historyPath2 = new Path(historyWorkDir2.toURI());
- appState = new MockAppState();
- AppStateBindingInfo binding2 = buildBindingInfo();
- binding2.application = factory.newApplication(0, 0, 0)
- .name(getValidTestName());
- binding2.historyPath = historyPath2;
- appState.buildInstance(binding2);
- // on this read there won't be the right number of roles
- LoadedRoleHistory loadedRoleHistory = historyWriter.read(fs, history);
- assertEquals(0, appState.getRoleHistory().rebuild(loadedRoleHistory));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.java
deleted file mode 100644
index dbad599..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.types.ApplicationLivenessInformation;
-import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Test app state flexing.
- */
-public class TestMockAppStateFlexing extends BaseMockAppStateTest implements
- MockRoles {
- private static final Logger LOG =
- LoggerFactory.getLogger(BaseMockAppStateTest.class);
-
- @Override
- public String getTestName() {
- return "TestMockAppStateFlexing";
- }
-
- //@Test
- public void testFlexDuringLaunchPhase() throws Throwable {
-
- // ask for one instance of role0
- getRole0Status().setDesired(1);
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
-
- // at this point there's now one request in the list
- assertEquals(1, ops.size());
- // and in a liveness check, one outstanding
- ApplicationLivenessInformation liveness =
- appState.getApplicationLivenessInformation();
- assertEquals(1, liveness.requestsOutstanding);
- assertFalse(liveness.allRequestsSatisfied);
-
- List<Container> allocations = engine.execute(ops);
- List<ContainerAssignment> assignments = new ArrayList<>();
- List<AbstractRMOperation> releases = new ArrayList<>();
- appState.onContainersAllocated(allocations, assignments, releases);
- assertEquals(1, assignments.size());
- ContainerAssignment assigned = assignments.get(0);
- Container target = assigned.container;
- RoleInstance ri = roleInstance(assigned);
-
- ops = appState.reviewRequestAndReleaseNodes();
- assertTrue(ops.isEmpty());
-
- liveness = appState.getApplicationLivenessInformation();
- assertEquals(0, liveness.requestsOutstanding);
- assertTrue(liveness.allRequestsSatisfied);
-
- //now this is the start point.
- appState.containerStartSubmitted(target, ri);
-
- ops = appState.reviewRequestAndReleaseNodes();
- assertTrue(ops.isEmpty());
-
- appState.innerOnNodeManagerContainerStarted(target.getId());
- }
-
- //@Test
- public void testFlexBeforeAllocationPhase() throws Throwable {
- getRole0Status().setDesired(1);
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- assertFalse(ops.isEmpty());
-
- // second scan will find the first run outstanding, so not re-issue
- // any more container requests
- List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes();
- assertTrue(ops2.isEmpty());
-
- // and in a liveness check, one outstanding
- ApplicationLivenessInformation liveness = appState
- .getApplicationLivenessInformation();
- assertEquals(1, liveness.requestsOutstanding);
- assertFalse(liveness.allRequestsSatisfied);
-
- appState.refreshClusterStatus();
- Application application = appState.getClusterStatus();
- // TODO cluster status returns liveness info
-// assertEquals(1, cd.liveness.requestsOutstanding);
-
- }
-
-
- //@Test
- public void testFlexDownTwice() throws Throwable {
- int r0 = 6;
- int r1 = 0;
- int r2 = 0;
- getRole0Status().setDesired(r0);
- getRole1Status().setDesired(r1);
- getRole2Status().setDesired(r2);
- List<RoleInstance> instances = createAndStartNodes();
-
- int clusterSize = r0 + r1 + r2;
- assertEquals(instances.size(), clusterSize);
- LOG.info("shrinking cluster");
- r0 = 4;
- getRole0Status().setDesired(r0);
- List<AppState.NodeCompletionResult> completionResults = new ArrayList<>();
- instances = createStartAndStopNodes(completionResults);
- assertEquals(0, instances.size());
- // assert two nodes were released
- assertEquals(2, completionResults.size());
-
- // no-op review
- completionResults = new ArrayList<>();
- instances = createStartAndStopNodes(completionResults);
- assertEquals(0, instances.size());
- // assert two nodes were released
- assertEquals(0, completionResults.size());
-
-
- // now shrink again
- getRole0Status().setDesired(1);
- completionResults = new ArrayList<>();
- instances = createStartAndStopNodes(completionResults);
- assertEquals(0, instances.size());
- // assert two nodes were released
- assertEquals(3, completionResults.size());
-
- }
-
- //@Test
- public void testFlexNegative() throws Throwable {
- int r0 = 6;
- int r1 = 0;
- int r2 = 0;
- getRole0Status().setDesired(r0);
- getRole1Status().setDesired(r1);
- getRole2Status().setDesired(r2);
- List<RoleInstance> instances = createAndStartNodes();
-
- int clusterSize = r0 + r1 + r2;
- assertEquals(instances.size(), clusterSize);
- LOG.info("shrinking cluster");
- getRole0Status().setDesired(-2);
- List<AppState.NodeCompletionResult> completionResults = new ArrayList<>();
- try {
- createStartAndStopNodes(completionResults);
- fail("expected an exception");
- } catch (TriggerClusterTeardownException e) {
- }
-
- }
-
- //@Test
- public void testCancelWithRequestsOutstanding() throws Throwable {
- // flex cluster size before the original set were allocated
-
-
- getRole0Status().setDesired(6);
- // build the ops
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
- // here the data structures exist
-
- // go down
- getRole0Status().setDesired(3);
- List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes();
- assertEquals(3, ops2.size());
- for (AbstractRMOperation op : ops2) {
- assertTrue(op instanceof CancelSingleRequest);
- }
-
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[78/86] [abbrv] hadoop git commit: YARN-7073. Yarn native services
rest API documentation. Contributed by Gour Saha
Posted by ji...@apache.org.
YARN-7073. Yarn native services rest API documentation. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5fec5b7a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5fec5b7a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5fec5b7a
Branch: refs/heads/yarn-native-services
Commit: 5fec5b7a2cec4ae2b002ec7d54f4639f19c15648
Parents: 3819081
Author: Jian He <ji...@apache.org>
Authored: Thu Aug 31 17:17:22 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 25 16:37:24 2017 -0700
----------------------------------------------------------------------
hadoop-project/src/site/site.xml | 1 +
.../hadoop/yarn/service/webapp/ApiServer.java | 47 +-
.../definition/YARN-Services-Examples.md | 245 ++++++++
...RN-Simplified-V1-API-Layer-For-Services.yaml | 128 ++--
.../native-services/NativeServicesAPI.md | 606 +++++++++++++++++++
5 files changed, 973 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fec5b7a/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index e6e9d86..755bc61 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -154,6 +154,7 @@
<menu name="YARN Native Services" inherit="top">
<item name="Introduction" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesIntro.html"/>
+ <item name="Native Services API" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesAPI.html"/>
<item name="Native Services Discovery" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesDiscovery.html"/>
</menu>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fec5b7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index f55e3f1..e8286ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -24,12 +24,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,6 +48,7 @@ import java.io.IOException;
import java.util.Collections;
import java.util.Map;
+import static org.apache.hadoop.yarn.service.api.records.ServiceState.ACCEPTED;
import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
/**
@@ -76,11 +76,11 @@ public class ApiServer {
@GET
@Path(VERSION)
@Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
+ @Produces({ MediaType.APPLICATION_JSON })
public Response getVersion() {
String version = VersionInfo.getBuildVersion();
LOG.info(version);
- return Response.ok(version).build();
+ return Response.ok("{ \"hadoop_version\": \"" + version + "\"}").build();
}
@POST
@@ -94,11 +94,11 @@ public class ApiServer {
ApplicationId applicationId = SERVICE_CLIENT.actionCreate(service);
LOG.info("Successfully created service " + service.getName()
+ " applicationId = " + applicationId);
- serviceStatus.setState(ServiceState.ACCEPTED);
+ serviceStatus.setState(ACCEPTED);
serviceStatus.setUri(
CONTEXT_ROOT + SERVICE_ROOT_PATH + "/" + service
.getName());
- return Response.status(Status.CREATED).entity(serviceStatus).build();
+ return Response.status(Status.ACCEPTED).entity(serviceStatus).build();
} catch (IllegalArgumentException e) {
serviceStatus.setDiagnostics(e.getMessage());
return Response.status(Status.BAD_REQUEST).entity(serviceStatus)
@@ -182,16 +182,16 @@ public class ApiServer {
+ ": Invalid number of containers specified " + component
.getNumberOfContainers()).build();
}
+ ServiceStatus status = new ServiceStatus();
try {
Map<String, Long> original = SERVICE_CLIENT.flexByRestService(appName,
Collections.singletonMap(component.getName(),
component.getNumberOfContainers()));
- return Response.ok().entity(
- "Updating component " + componentName + " size from " + original
- .get(componentName) + " to " + component.getNumberOfContainers())
- .build();
+ status.setDiagnostics(
+ "Updating component (" + componentName + ") size from " + original
+ .get(componentName) + " to " + component.getNumberOfContainers());
+ return Response.ok().entity(status).build();
} catch (YarnException | IOException e) {
- ServiceStatus status = new ServiceStatus();
status.setDiagnostics(e.getMessage());
return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status)
.build();
@@ -244,31 +244,40 @@ public class ApiServer {
}
private Response updateLifetime(String appName, Service updateAppData) {
+ ServiceStatus status = new ServiceStatus();
try {
String newLifeTime =
SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime());
- return Response.ok("Service " + appName + " lifeTime is successfully updated to "
- + updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build();
+ status.setDiagnostics(
+ "Service (" + appName + ")'s lifeTime is updated to " + newLifeTime
+ + ", " + updateAppData.getLifetime()
+ + " seconds remaining");
+ return Response.ok(status).build();
} catch (Exception e) {
String message =
- "Failed to update service (" + appName + ") lifetime ("
- + updateAppData.getLifetime() + ")";
+ "Failed to update service (" + appName + ")'s lifetime to "
+ + updateAppData.getLifetime();
LOG.error(message, e);
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(message + " : " + e.getMessage()).build();
+ status.setDiagnostics(message + ": " + e.getMessage());
+ return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status)
+ .build();
}
}
private Response startService(String appName) {
+ ServiceStatus status = new ServiceStatus();
try {
SERVICE_CLIENT.actionStart(appName);
LOG.info("Successfully started service " + appName);
- return Response.ok("Service " + appName + " is successfully started").build();
+ status.setDiagnostics("Service " + appName + " is successfully started.");
+ status.setState(ServiceState.ACCEPTED);
+ return Response.ok(status).build();
} catch (Exception e) {
String message = "Failed to start service " + appName;
+ status.setDiagnostics(message + ": " + e.getMessage());
LOG.info(message, e);
return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(message + ": " + e.getMessage()).build();
+ .entity(status).build();
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fec5b7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
new file mode 100644
index 0000000..3cd3d48
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
@@ -0,0 +1,245 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+## Examples
+
+### Create a simple single-component service with most attribute values as defaults
+POST URL - http://localhost:9191/ws/v1/services
+
+##### POST Request JSON
+```json
+{
+ "name": "hello-world",
+ "components" :
+ [
+ {
+ "name": "hello",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "nginx:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "./start_nginx.sh",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ }
+ }
+ ]
+}
+```
+
+##### GET Response JSON
+GET URL - http://localhost:9191/ws/v1/services/hello-world
+
+Note, lifetime value of -1 means unlimited lifetime.
+
+```json
+{
+ "name": "hello-world",
+ "id": "application_1503963985568_0002",
+ "lifetime": -1,
+ "components": [
+ {
+ "name": "hello",
+ "dependencies": [],
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ },
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": [],
+ "containers": [
+ {
+ "id": "container_e03_1503963985568_0002_01_000001",
+ "ip": "10.22.8.143",
+ "hostname": "myhost.local",
+ "state": "READY",
+ "launch_time": 1504051512412,
+ "bare_host": "10.22.8.143",
+ "component_name": "hello-0"
+ },
+ {
+ "id": "container_e03_1503963985568_0002_01_000002",
+ "ip": "10.22.8.143",
+ "hostname": "myhost.local",
+ "state": "READY",
+ "launch_time": 1504051536450,
+ "bare_host": "10.22.8.143",
+ "component_name": "hello-1"
+ }
+ ],
+ "launch_command": "./start_nginx.sh",
+ "number_of_containers": 1,
+ "run_privileged_container": false
+ }
+ ],
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": {}
+}
+
+```
+### Update to modify the lifetime of a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+
+Note, irrespective of what the current lifetime value is, this update request will set the lifetime of the service to be 3600 seconds (1 hour) from the time the request is submitted. Hence, if a a service has remaining lifetime of 5 mins (say) and would like to extend it to an hour OR if an application has remaining lifetime of 5 hours (say) and would like to reduce it down to an hour, then for both scenarios you need to submit the same request below.
+
+```json
+{
+ "lifetime": 3600
+}
+```
+### Stop a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+```json
+{
+ "state": "STOPPED"
+}
+```
+
+### Start a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+```json
+{
+ "state": "STARTED"
+}
+```
+
+### Update to flex up/down the no of containers (instances) of a component of a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello
+
+##### PUT Request JSON
+```json
+{
+ "name": "hello",
+ "number_of_containers": 3
+}
+```
+
+### Destroy a service
+DELETE URL - http://localhost:9191/ws/v1/services/hello-world
+
+***
+
+### Create a complicated service - HBase
+POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
+
+##### POST Request JSON
+
+```json
+{
+ "name": "hbase-app-1",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "hbasemaster",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "hbase:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/hdp/current/hbase-master/bin/hbase master start",
+ "resource": {
+ "cpus": 1,
+ "memory": "2048"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_DIR": "<LOG_DIR>"
+ },
+ "files": [
+ {
+ "type": "XML",
+ "dest_file": "/etc/hadoop/conf/core-site.xml",
+ "props": {
+ "fs.defaultFS": "${CLUSTER_FS_URI}"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "/etc/hbase/conf/hbase-site.xml",
+ "props": {
+ "hbase.cluster.distributed": "true",
+ "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
+ "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
+ "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
+ "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
+ "hbase.master.info.port": "16010"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "regionserver",
+ "number_of_containers": 3,
+ "unique_component_support": "true",
+ "artifact": {
+ "id": "hbase:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/hdp/current/hbase-regionserver/bin/hbase regionserver start",
+ "resource": {
+ "cpus": 1,
+ "memory": "2048"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_DIR": "<LOG_DIR>"
+ },
+ "files": [
+ {
+ "type": "XML",
+ "dest_file": "/etc/hadoop/conf/core-site.xml",
+ "props": {
+ "fs.defaultFS": "${CLUSTER_FS_URI}"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "/etc/hbase/conf/hbase-site.xml",
+ "props": {
+ "hbase.cluster.distributed": "true",
+ "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
+ "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
+ "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
+ "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
+ "hbase.master.info.port": "16010",
+ "hbase.regionserver.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}"
+ }
+ }
+ ]
+ }
+ }
+ ],
+ "quicklinks": {
+ "HBase Master Status UI": "http://hbasemaster0.${SERVICE_NAME}.${USER}.${DOMAIN}:16010/master-status",
+ "Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/"
+ }
+}
+```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fec5b7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index 17f8c95..b084be7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -17,18 +17,30 @@
swagger: '2.0'
info:
- title: "[YARN-4793] Simplified API layer for services and beyond"
+ title: "YARN Simplified API layer for services"
description: |
- Bringing a new service on YARN today is not a simple experience. The APIs of existing frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs) or writing a complex spec (for declarative frameworks). In addition to building critical building blocks inside YARN (as part of other efforts at link:https://issues.apache.org/jira/browse/YARN-4692[YARN-4692]), there is a need for simplifying the user facing story for building services. Experience of projects like Apache Slider running real-life services like HBase, Storm, Accumulo, Solr etc, gives us some very good insights on how simplified APIs for services should look like.
+ Bringing a new service on YARN today is not a simple experience. The APIs of
+ existing frameworks are either too low level (native YARN), require writing
+ new code (for frameworks with programmatic APIs) or writing a complex spec
+ (for declarative frameworks). In addition to building critical building blocks
+ inside YARN (as part of other efforts at YARN-4692), there is a need for
+ simplifying the user facing story for building services. Experience of projects
+ like Apache Slider running real-life services like HBase, Storm, Accumulo,
+ Solr etc, gives us some very good insights on how simplified APIs for services
+ should look like.
+ To this end, we should look at a new simple-services API layer backed by REST
+ interfaces. This API can be used to create and manage the lifecycle of YARN
+ services. Services here can range from simple single-component service to
+ complex multi-component assemblies needing orchestration. YARN-4793 tracks
+ this effort.
- To this end, we should look at a new simple-services API layer backed by REST interfaces. This API can be used to create and manage the lifecycle of YARN services. Services here can range from simple single-component service to complex multi-component assemblies needing orchestration.
-
-
- We should also look at making this a unified REST based entry point for other important features like resource-profile management (link:https://issues.apache.org/jira/browse/YARN-3926[YARN-3926]), package-definitions' lifecycle-management and service-discovery (link:https://issues.apache.org/jira/browse/YARN-913[YARN-913]/link:https://issues.apache.org/jira/browse/YARN-4757[YARN-4757]). We also need to flesh out its relation to our present much lower level REST APIs (link:https://issues.apache.org/jira/browse/YARN-1695[YARN-1695]) in YARN for application-submission and management.
-
-
- This document spotlights on this specification. In most of the cases, the application owner will not be forced to make any changes to their application. This is primarily true if the application is packaged with containerization technologies like docker. Irrespective of how complex the application is, there will be hooks provided at appropriate layers to allow pluggable and customizable application behavior.
+ This document spotlights on this specification. In most of the cases, the
+ application owner will not be forced to make any changes to their applications.
+ This is primarily true if the application is packaged with containerization
+ technologies like docker. Irrespective of how complex the application is,
+ there will be hooks provided at appropriate layers to allow pluggable and
+ customizable application behavior.
version: "1.0.0"
license:
@@ -39,7 +51,6 @@ host: host.mycompany.com
# array of all schemes that your API supports
schemes:
- http
- - https
# will be prefixed to all paths
basePath: /ws/v1/
consumes:
@@ -47,9 +58,17 @@ consumes:
produces:
- application/json
paths:
+ /services/version:
+ get:
+ summary: Get current version of the API server.
+ description: Get current version of the API server.
+ responses:
+ 200:
+ description: Successful request
+
/services:
get:
- summary: List of services running in the cluster
+ summary: (TBD) List of services running in the cluster.
description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name.
responses:
200:
@@ -74,15 +93,62 @@ paths:
$ref: '#/definitions/Service'
responses:
202:
- description: Request accepted
+ description: The request to create a service is accepted
+ 400:
+ description: Invalid service definition provided in the request body
+ 500:
+ description: Failed to create a service
default:
description: Unexpected error
schema:
$ref: '#/definitions/ServiceStatus'
/services/{service_name}:
+ put:
+ summary: Update a service or upgrade the binary version of the components of a running service
+ description: Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service.
+ The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts (TBD).
+ parameters:
+ - name: service_name
+ in: path
+ description: Service name
+ required: true
+ type: string
+ - name: Service
+ in: body
+ description: The updated service definition. It can contain the updated lifetime of a service or the desired state (STOPPED/STARTED) of a service to initiate a start/stop operation against the specified service
+ required: true
+ schema:
+ $ref: '#/definitions/Service'
+ responses:
+ 204:
+ description: Update or upgrade was successful
+ 404:
+ description: Service does not exist
+ default:
+ description: Unexpected error
+ schema:
+ $ref: '#/definitions/ServiceStatus'
+ delete:
+ summary: Destroy a service
+ description: Destroy a service and release all resources. This API might have to return JSON data providing location of logs (TBD), etc.
+ parameters:
+ - name: service_name
+ in: path
+ description: Service name
+ required: true
+ type: string
+ responses:
+ 204:
+ description: Destroy was successful
+ 404:
+ description: Service does not exist
+ default:
+ description: Unexpected error
+ schema:
+ $ref: '#/definitions/ServiceStatus'
get:
- summary: Get service details
+ summary: Get details of a service.
description: Return the details (including containers) of a running service
parameters:
- name: service_name
@@ -108,43 +174,36 @@ paths:
description: Unexpected error
schema:
$ref: '#/definitions/ServiceStatus'
+ /services/{service_name}/components/{component_name}:
put:
- summary: Update a service or upgrade the binary version of the components of a running service
- description: Update the runtime properties of a service. As of now, only update of lifetime and number of instances (flexing) of the components of a service is supported. The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts.
+ summary: Flex a component's number of instances.
+ description: Set a component's desired number of instanes
parameters:
- name: service_name
in: path
description: Service name
required: true
type: string
- responses:
- 204:
- description: Update or upgrade was successful
- 404:
- description: Service does not exist
- default:
- description: Unexpected error
- schema:
- $ref: '#/definitions/ServiceStatus'
- delete:
- summary: Destroy service
- description: Destroy a service and release all resources. This API might have to return JSON data providing location of logs, etc. Not finalized yet.
- parameters:
- - name: service_name
+ - name: component_name
in: path
- description: Service name
+ description: Component name
required: true
type: string
+ - name: Component
+ in: body
+ description: The definition of a component which contains the updated number of instances.
+ required: true
+ schema:
+ $ref: '#/definitions/Component'
responses:
- 204:
- description: Destroy was successful
+ 200:
+ description: Flex was successful
404:
description: Service does not exist
default:
description: Unexpected error
schema:
$ref: '#/definitions/ServiceStatus'
-
definitions:
Service:
description: a service resource has the following attributes.
@@ -183,7 +242,7 @@ definitions:
format: int64
description: Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.
placement_policy:
- description: Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.
+ description: (TBD) Advanced scheduling and placement policies. If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.
$ref: '#/definitions/PlacementPolicy'
components:
description: Components of a service.
@@ -420,4 +479,3 @@ definitions:
type: integer
format: int32
description: An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fec5b7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
new file mode 100644
index 0000000..f56139a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
@@ -0,0 +1,606 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# YARN Simplified API layer for services
+
+## Overview
+Bringing a new service on YARN today is not a simple experience. The APIs of
+existing frameworks are either too low level (native YARN), require writing
+new code (for frameworks with programmatic APIs) or writing a complex spec
+(for declarative frameworks). In addition to building critical building blocks
+inside YARN (as part of other efforts at
+[YARN-4692](https://issues.apache.org/jira/browse/YARN-4692)), there is a need for
+simplifying the user facing story for building services. Experience of projects
+like Apache Slider running real-life services like HBase, Storm, Accumulo,
+Solr etc, gives us some very good insights on how simplified APIs for services
+should look like.
+
+To this end, we should look at a new simple-services API layer backed by REST
+interfaces. This API can be used to create and manage the lifecycle of YARN
+services. Services here can range from simple single-component service to
+complex multi-component assemblies needing orchestration.
+[YARN-4793](https://issues.apache.org/jira/browse/YARN-4793) tracks this
+effort.
+
+This document spotlights on this specification. In most of the cases, the
+application owner will not be forced to make any changes to their applications.
+This is primarily true if the application is packaged with containerization
+technologies like docker. Irrespective of how complex the application is,
+there will be hooks provided at appropriate layers to allow pluggable and
+customizable application behavior.
+
+
+### Version information
+Version: 1.0.0
+
+### License information
+License: Apache 2.0
+License URL: http://www.apache.org/licenses/LICENSE-2.0.html
+
+### URI scheme
+Host: host.mycompany.com
+
+BasePath: /ws/v1/
+
+Schemes: HTTP
+
+### Consumes
+
+* application/json
+
+
+### Produces
+
+* application/json
+
+
+## Paths
+### Create a service
+```
+POST /services
+```
+
+#### Description
+
+Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state.
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|BodyParameter|Service|Service request object|true|Service||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|202|The request to create a service is accepted|No Content|
+|400|Invalid service definition provided in the request body|No Content|
+|500|Failed to create a service|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### (TBD) List of services running in the cluster.
+```
+GET /services
+```
+
+#### Description
+
+Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name.
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|An array of services|Service array|
+|default|Unexpected error|ServiceStatus|
+
+
+### Get current version of the API server.
+```
+GET /services/version
+```
+
+#### Description
+
+Get current version of the API server.
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|Successful request|No Content|
+
+
+### Update a service or upgrade the binary version of the components of a running service
+```
+PUT /services/{service_name}
+```
+
+#### Description
+
+Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service. The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts (TBD).
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+|BodyParameter|Service|The updated service definition. It can contain the updated lifetime of a service or the desired state (STOPPED/STARTED) of a service to initiate a start/stop operation against the specified service|true|Service||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|204|Update or upgrade was successful|No Content|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### Destroy a service
+```
+DELETE /services/{service_name}
+```
+
+#### Description
+
+Destroy a service and release all resources. This API might have to return JSON data providing location of logs (TBD), etc.
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|204|Destroy was successful|No Content|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### Get details of a service.
+```
+GET /services/{service_name}
+```
+
+#### Description
+
+Return the details (including containers) of a running service
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|a service object|object|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### Flex a component's number of instances.
+```
+PUT /services/{service_name}/components/{component_name}
+```
+
+#### Description
+
+Set a component's desired number of instanes
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+|PathParameter|component_name|Component name|true|string||
+|BodyParameter|Component|The definition of a component which contains the updated number of instances.|true|Component||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|Flex was successful|No Content|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+## Definitions
+### Artifact
+
+Artifact of a service component. If not specified, component will just run the bare launch command and no artifact will be localized.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|id|Artifact id. Examples are package location uri for tarball based services, image name for docker, name of service, etc.|true|string||
+|type|Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For SERVICE type, the service specified will be read and its components will be added into this service. The original component with artifact type SERVICE will be removed (any properties specified in the original component will be ignored).|false|enum (DOCKER, TARBALL, SERVICE)|DOCKER|
+|uri|Artifact location to support multiple artifact stores (optional).|false|string||
+
+
+### Component
+
+One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|name|Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.|true|string||
+|dependencies|An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of a service should be represented as a DAG.|false|string array||
+|readiness_check|Readiness check for this component.|false|ReadinessCheck||
+|artifact|Artifact of the component (optional). If not specified, the service level global artifact takes effect.|false|Artifact||
+|launch_command|The custom launch command of this component (optional for DOCKER component, required otherwise). When specified at the component level, it overrides the value specified at the global level (if any).|false|string||
+|resource|Resource of this component (optional). If not specified, the service level global resource takes effect.|false|Resource||
+|number_of_containers|Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.|false|integer (int64)||
+|run_privileged_container|Run all containers of this component in privileged mode (YARN-4262).|false|boolean||
+|placement_policy|Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.|false|PlacementPolicy||
+|configuration|Config properties for this component.|false|Configuration||
+|quicklinks|A list of quicklink keys defined at the service level, and to be resolved by this component.|false|string array||
+
+
+### ConfigFile
+
+A config file that needs to be created and made available as a volume in a service component container.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|type|Config file in the standard format like xml, properties, json, yaml, template.|false|enum (XML, PROPERTIES, JSON, YAML, TEMPLATE, ENV, HADOOP_XML)||
+|dest_file|The path that this configuration file should be created as. If it is an absolute path, it will be mounted into the DOCKER container. Absolute paths are only allowed for DOCKER containers. If it is a relative path, only the file name should be provided, and the file will be created in the container local working directory under a folder named conf.|false|string||
+|src_file|This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.|false|string||
+|props|A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.|false|object||
+
+
+### Configuration
+
+Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|properties|A blob of key-value pairs of common service properties.|false|object||
+|env|A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.|false|object||
+|files|Array of list of files that needs to be created and made available as volumes in the service component containers.|false|ConfigFile array||
+
+
+### Container
+
+An instance of a running service container.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|id|Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002.|false|string||
+|launch_time|The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.|false|string (date)||
+|ip|IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.|false|string||
+|hostname|Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.|false|string||
+|bare_host|The bare node or host in which the container is running, e.g. cn008.example.com.|false|string||
+|state|State of the container of a service.|false|ContainerState||
+|component_name|Name of the component that this container instance belongs to.|false|string||
+|resource|Resource used for this container.|false|Resource||
+|artifact|Artifact used for this container.|false|Artifact||
+|privileged_container|Container running in privileged mode or not.|false|boolean||
+
+
+### ContainerState
+
+The current state of the container of a service.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|state|enum of the state of the container|false|enum (INIT, STARTED, READY)||
+
+
+### PlacementPolicy
+
+Placement policy of an instance of a service. This feature is in the works in YARN-6592.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|label|Assigns a service to a named partition of the cluster where the service desires to run (optional). If not specified all services are submitted to a default label of the service owner. One or more labels can be setup for each service owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.|false|string||
+
+
+### ReadinessCheck
+
+A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|type|E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).|true|enum (HTTP, PORT)||
+|props|A blob of key value pairs that will be used to configure the check.|false|object||
+|artifact|Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET|false|Artifact||
+
+
+### Resource
+
+Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|profile|Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.|false|string||
+|cpus|Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).|false|integer (int32)||
+|memory|Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.|false|string||
+
+
+### Service
+
+a service resource has the following attributes.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|name|A unique service name. If Registry DNS is enabled, the max length is 63 characters.|true|string||
+|id|A unique service id.|false|string||
+|artifact|Artifact of single-component service.|false|Artifact||
+|resource|Resource of single-component service or the global default for multi-component services. Mandatory if it is a single-component service and if cpus and memory are not specified at the Service level.|false|Resource||
+|launch_command|The custom launch command of a service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section.|false|string||
+|launch_time|The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.|false|string (date)||
+|number_of_containers|Number of containers for each component in the service. Each component can further override this service-level global default.|false|integer (int64)||
+|number_of_running_containers|In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.|false|integer (int64)||
+|lifetime|Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.|false|integer (int64)||
+|placement_policy|(TBD) Advanced scheduling and placement policies. If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.|false|PlacementPolicy||
+|components|Components of a service.|false|Component array||
+|configuration|Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.|false|Configuration||
+|containers|Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.|false|Container array||
+|state|State of the service. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started service.|false|ServiceState||
+|quicklinks|A blob of key-value pairs of quicklinks to be exported for a service.|false|object||
+|queue|The YARN queue that this service should be submitted to.|false|string||
+
+
+### ServiceState
+
+The current state of a service.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|state|enum of the state of the service|false|enum (ACCEPTED, STARTED, READY, STOPPED, FAILED)||
+
+
+### ServiceStatus
+
+The current status of a submitted service, returned as a response to the GET API.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|diagnostics|Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state.|false|string||
+|state|Service state.|false|ServiceState||
+|code|An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.|false|integer (int32)||
+
+
+
+## Examples
+
+### Create a simple single-component service with most attribute values as defaults
+POST URL - http://localhost:9191/ws/v1/services
+
+##### POST Request JSON
+```json
+{
+ "name": "hello-world",
+ "components" :
+ [
+ {
+ "name": "hello",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "nginx:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "./start_nginx.sh",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ }
+ }
+ ]
+}
+```
+
+##### GET Response JSON
+GET URL - http://localhost:9191/ws/v1/services/hello-world
+
+Note, lifetime value of -1 means unlimited lifetime.
+
+```json
+{
+ "name": "hello-world",
+ "id": "application_1503963985568_0002",
+ "lifetime": -1,
+ "components": [
+ {
+ "name": "hello",
+ "dependencies": [],
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ },
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": [],
+ "containers": [
+ {
+ "id": "container_e03_1503963985568_0002_01_000001",
+ "ip": "10.22.8.143",
+ "hostname": "myhost.local",
+ "state": "READY",
+ "launch_time": 1504051512412,
+ "bare_host": "10.22.8.143",
+ "component_name": "hello-0"
+ },
+ {
+ "id": "container_e03_1503963985568_0002_01_000002",
+ "ip": "10.22.8.143",
+ "hostname": "myhost.local",
+ "state": "READY",
+ "launch_time": 1504051536450,
+ "bare_host": "10.22.8.143",
+ "component_name": "hello-1"
+ }
+ ],
+ "launch_command": "./start_nginx.sh",
+ "number_of_containers": 1,
+ "run_privileged_container": false
+ }
+ ],
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": {}
+}
+
+```
+### Update to modify the lifetime of a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+
+Note, irrespective of what the current lifetime value is, this update request will set the lifetime of the service to be 3600 seconds (1 hour) from the time the request is submitted. Hence, if a a service has remaining lifetime of 5 mins (say) and would like to extend it to an hour OR if an application has remaining lifetime of 5 hours (say) and would like to reduce it down to an hour, then for both scenarios you need to submit the same request below.
+
+```json
+{
+ "lifetime": 3600
+}
+```
+### Stop a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+```json
+{
+ "state": "STOPPED"
+}
+```
+
+### Start a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+```json
+{
+ "state": "STARTED"
+}
+```
+
+### Update to flex up/down the no of containers (instances) of a component of a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello
+
+##### PUT Request JSON
+```json
+{
+ "name": "hello",
+ "number_of_containers": 3
+}
+```
+
+### Destroy a service
+DELETE URL - http://localhost:9191/ws/v1/services/hello-world
+
+***
+
+### Create a complicated service - HBase
+POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
+
+##### POST Request JSON
+
+```json
+{
+ "name": "hbase-app-1",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "hbasemaster",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "hbase:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/hdp/current/hbase-master/bin/hbase master start",
+ "resource": {
+ "cpus": 1,
+ "memory": "2048"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_DIR": "<LOG_DIR>"
+ },
+ "files": [
+ {
+ "type": "XML",
+ "dest_file": "/etc/hadoop/conf/core-site.xml",
+ "props": {
+ "fs.defaultFS": "${CLUSTER_FS_URI}"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "/etc/hbase/conf/hbase-site.xml",
+ "props": {
+ "hbase.cluster.distributed": "true",
+ "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
+ "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
+ "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
+ "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
+ "hbase.master.info.port": "16010"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "regionserver",
+ "number_of_containers": 3,
+ "unique_component_support": "true",
+ "artifact": {
+ "id": "hbase:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/hdp/current/hbase-regionserver/bin/hbase regionserver start",
+ "resource": {
+ "cpus": 1,
+ "memory": "2048"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_DIR": "<LOG_DIR>"
+ },
+ "files": [
+ {
+ "type": "XML",
+ "dest_file": "/etc/hadoop/conf/core-site.xml",
+ "props": {
+ "fs.defaultFS": "${CLUSTER_FS_URI}"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "/etc/hbase/conf/hbase-site.xml",
+ "props": {
+ "hbase.cluster.distributed": "true",
+ "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
+ "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
+ "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
+ "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
+ "hbase.master.info.port": "16010",
+ "hbase.regionserver.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}"
+ }
+ }
+ ]
+ }
+ }
+ ],
+ "quicklinks": {
+ "HBase Master Status UI": "http://hbasemaster0.${SERVICE_NAME}.${USER}.${DOMAIN}:16010/master-status",
+ "Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/"
+ }
+}
+```
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java
deleted file mode 100644
index a667432..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.junit.Test;
-
-
-public class TestWorkflowServiceTerminatingRunnable extends WorkflowServiceTestBase {
-
- //@Test
- public void testNoservice() throws Throwable {
-
- try {
- new ServiceTerminatingRunnable(null, new SimpleRunnable());
- fail("unexpected ");
- } catch (IllegalArgumentException e) {
-
- // expected
- }
- }
-
-
- //@Test
- public void testBasicRun() throws Throwable {
-
- WorkflowCompositeService svc = run(new WorkflowCompositeService());
- ServiceTerminatingRunnable runnable = new ServiceTerminatingRunnable(svc,
- new SimpleRunnable());
-
- // synchronous in-thread execution
- runnable.run();
- assertStopped(svc);
- }
-
- //@Test
- public void testFailureRun() throws Throwable {
-
- WorkflowCompositeService svc = run(new WorkflowCompositeService());
- ServiceTerminatingRunnable runnable =
- new ServiceTerminatingRunnable(svc, new SimpleRunnable(true));
-
- // synchronous in-thread execution
- runnable.run();
- assertStopped(svc);
- assertNotNull(runnable.getException());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
deleted file mode 100644
index f38bd9d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.Service;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.rules.TestName;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Locale;
-import java.util.concurrent.Callable;
-
-/**
- * Test base for workflow service tests.
- */
-public abstract class WorkflowServiceTestBase extends Assert {
- private static final Logger
- log = LoggerFactory.getLogger(WorkflowServiceTestBase.class);
-
- /**
- * Set the timeout for every test
- */
- @Rule
- public Timeout testTimeout = new Timeout(15000);
-
- @Rule
- public TestName name = new TestName();
-
- @Before
- public void nameThread() {
- Thread.currentThread().setName("JUnit");
- }
-
-
- protected void assertInState(Service service, Service.STATE expected) {
- Service.STATE actual = service.getServiceState();
- if (actual != expected) {
- fail("Service " + service.getName() + " in state " + actual
- + " -expected " + expected);
- }
- }
-
- protected void assertStopped(Service service) {
- assertInState(service, Service.STATE.STOPPED);
- }
-
- protected void logState(ServiceParent p) {
- logService(p);
- for (Service s : p.getServices()) {
- logService(s);
- }
- }
-
- protected void logService(Service s) {
- log.info(s.toString());
- Throwable failureCause = s.getFailureCause();
- if (failureCause != null) {
- log.info("Failed in state {} with {}", s.getFailureState(),
- failureCause);
- }
- }
-
- /**
- * Init and start a service
- * @param svc the service
- * @return the service
- */
- protected <S extends Service> S run(S svc) {
- svc.init(new Configuration());
- svc.start();
- return svc;
- }
-
- /**
- * Handler for callable events
- */
- public static class CallableHandler implements Callable<String> {
- public volatile boolean notified = false;
- public final String result;
-
- public CallableHandler(String result) {
- this.result = result;
- }
-
- @Override
- public String call() throws Exception {
- log.info("CallableHandler::call");
- notified = true;
- return result;
- }
- }
-
- /**
- * Assert that a string is in an output list. Fails fast if the output
- * list is empty
- * @param text text to scan for
- * @param output list of output lines.
- */
- public void assertStringInOutput(String text, List<String> output) {
- assertTrue("Empty output list", !output.isEmpty());
- boolean found = false;
- StringBuilder builder = new StringBuilder();
- for (String s : output) {
- builder.append(s.toLowerCase(Locale.ENGLISH)).append('\n');
- if (s.contains(text)) {
- found = true;
- break;
- }
- }
-
- if (!found) {
- String message =
- "Text \"" + text + "\" not found in " + output.size() + " lines\n";
- fail(message + builder.toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java
deleted file mode 100644
index fc51e31..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java
+++ /dev/null
@@ -1,901 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.utils;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.Assert;
-import org.junit.internal.AssumptionViolatedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Arrays;
-import java.util.Properties;
-import java.util.UUID;
-
-/**
- * Utilities used across test cases to make assertions about filesystems
- * -assertions which fail with useful information.
- * This is lifted from Hadoop common Test; that JAR isn't published, so
- * we have to make do.
- */
-public class ContractTestUtils extends Assert {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ContractTestUtils.class);
-
- public static final String IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
-
- // For scale testing, we can repeatedly write small chunk data to generate
- // a large file.
- public static final String IO_CHUNK_BUFFER_SIZE = "io.chunk.buffer.size";
- public static final int DEFAULT_IO_CHUNK_BUFFER_SIZE = 128;
- public static final String IO_CHUNK_MODULUS_SIZE = "io.chunk.modulus.size";
- public static final int DEFAULT_IO_CHUNK_MODULUS_SIZE = 128;
-
- /**
- * Assert that a property in the property set matches the expected value
- * @param props property set
- * @param key property name
- * @param expected expected value. If null, the property must not be in the set
- */
- public static void assertPropertyEquals(Properties props,
- String key,
- String expected) {
- String val = props.getProperty(key);
- if (expected == null) {
- assertNull("Non null property " + key + " = " + val, val);
- } else {
- assertEquals("property " + key + " = " + val,
- expected,
- val);
- }
- }
-
- /**
- *
- * Write a file and read it in, validating the result. Optional flags control
- * whether file overwrite operations should be enabled, and whether the
- * file should be deleted afterwards.
- *
- * If there is a mismatch between what was written and what was expected,
- * a small range of bytes either side of the first error are logged to aid
- * diagnosing what problem occurred -whether it was a previous file
- * or a corrupting of the current file. This assumes that two
- * sequential runs to the same path use datasets with different character
- * moduli.
- *
- * @param fs filesystem
- * @param path path to write to
- * @param len length of data
- * @param overwrite should the create option allow overwrites?
- * @param delete should the file be deleted afterwards? -with a verification
- * that it worked. Deletion is not attempted if an assertion has failed
- * earlier -it is not in a <code>finally{}</code> block.
- * @throws IOException IO problems
- */
- public static void writeAndRead(FileSystem fs,
- Path path,
- byte[] src,
- int len,
- int blocksize,
- boolean overwrite,
- boolean delete) throws IOException {
- fs.mkdirs(path.getParent());
-
- writeDataset(fs, path, src, len, blocksize, overwrite);
-
- byte[] dest = readDataset(fs, path, len);
-
- compareByteArrays(src, dest, len);
-
- if (delete) {
- rejectRootOperation(path);
- boolean deleted = fs.delete(path, false);
- assertTrue("Deleted", deleted);
- assertPathDoesNotExist(fs, "Cleanup failed", path);
- }
- }
-
- /**
- * Write a file.
- * Optional flags control
- * whether file overwrite operations should be enabled
- * @param fs filesystem
- * @param path path to write to
- * @param len length of data
- * @param overwrite should the create option allow overwrites?
- * @throws IOException IO problems
- */
- public static void writeDataset(FileSystem fs,
- Path path,
- byte[] src,
- int len,
- int buffersize,
- boolean overwrite) throws IOException {
- assertTrue(
- "Not enough data in source array to write " + len + " bytes",
- src.length >= len);
- FSDataOutputStream out = fs.create(path,
- overwrite,
- fs.getConf()
- .getInt(IO_FILE_BUFFER_SIZE,
- 4096),
- (short) 1,
- buffersize);
- out.write(src, 0, len);
- out.close();
- assertFileHasLength(fs, path, len);
- }
-
- /**
- * Read the file and convert to a byte dataset.
- * This implements readfully internally, so that it will read
- * in the file without ever having to seek()
- * @param fs filesystem
- * @param path path to read from
- * @param len length of data to read
- * @return the bytes
- * @throws IOException IO problems
- */
- public static byte[] readDataset(FileSystem fs, Path path, int len)
- throws IOException {
- FSDataInputStream in = fs.open(path);
- byte[] dest = new byte[len];
- int offset =0;
- int nread = 0;
- try {
- while (nread < len) {
- int nbytes = in.read(dest, offset + nread, len - nread);
- if (nbytes < 0) {
- throw new EOFException("End of file reached before reading fully.");
- }
- nread += nbytes;
- }
- } finally {
- in.close();
- }
- return dest;
- }
-
- /**
- * Read a file, verify its length and contents match the expected array
- * @param fs filesystem
- * @param path path to file
- * @param original original dataset
- * @throws IOException IO Problems
- */
- public static void verifyFileContents(FileSystem fs,
- Path path,
- byte[] original) throws IOException {
- FileStatus stat = fs.getFileStatus(path);
- String statText = stat.toString();
- assertTrue("not a file " + statText, stat.isFile());
- assertEquals("wrong length " + statText, original.length, stat.getLen());
- byte[] bytes = readDataset(fs, path, original.length);
- compareByteArrays(original,bytes,original.length);
- }
-
- /**
- * Verify that the read at a specific offset in a stream
- * matches that expected
- * @param stm stream
- * @param fileContents original file contents
- * @param seekOff seek offset
- * @param toRead number of bytes to read
- * @throws IOException IO problems
- */
- public static void verifyRead(FSDataInputStream stm, byte[] fileContents,
- int seekOff, int toRead) throws IOException {
- byte[] out = new byte[toRead];
- stm.seek(seekOff);
- stm.readFully(out);
- byte[] expected = Arrays.copyOfRange(fileContents, seekOff,
- seekOff + toRead);
- compareByteArrays(expected, out,toRead);
- }
-
- /**
- * Assert that tthe array original[0..len] and received[] are equal.
- * A failure triggers the logging of the bytes near where the first
- * difference surfaces.
- * @param original source data
- * @param received actual
- * @param len length of bytes to compare
- */
- public static void compareByteArrays(byte[] original,
- byte[] received,
- int len) {
- assertEquals("Number of bytes read != number written",
- len, received.length);
- int errors = 0;
- int first_error_byte = -1;
- for (int i = 0; i < len; i++) {
- if (original[i] != received[i]) {
- if (errors == 0) {
- first_error_byte = i;
- }
- errors++;
- }
- }
-
- if (errors > 0) {
- String message = String.format(" %d errors in file of length %d",
- errors, len);
- LOG.warn(message);
- // the range either side of the first error to print
- // this is a purely arbitrary number, to aid user debugging
- final int overlap = 10;
- for (int i = Math.max(0, first_error_byte - overlap);
- i < Math.min(first_error_byte + overlap, len);
- i++) {
- byte actual = received[i];
- byte expected = original[i];
- String letter = toChar(actual);
- String line = String.format("[%04d] %2x %s\n", i, actual, letter);
- if (expected != actual) {
- line = String.format("[%04d] %2x %s -expected %2x %s\n",
- i,
- actual,
- letter,
- expected,
- toChar(expected));
- }
- LOG.warn(line);
- }
- fail(message);
- }
- }
-
- /**
- * Convert a byte to a character for printing. If the
- * byte value is < 32 -and hence unprintable- the byte is
- * returned as a two digit hex value
- * @param b byte
- * @return the printable character string
- */
- public static String toChar(byte b) {
- if (b >= 0x20) {
- return Character.toString((char) b);
- } else {
- return String.format("%02x", b);
- }
- }
-
- /**
- * Convert a buffer to a string, character by character
- * @param buffer input bytes
- * @return a string conversion
- */
- public static String toChar(byte[] buffer) {
- StringBuilder builder = new StringBuilder(buffer.length);
- for (byte b : buffer) {
- builder.append(toChar(b));
- }
- return builder.toString();
- }
-
- public static byte[] toAsciiByteArray(String s) {
- char[] chars = s.toCharArray();
- int len = chars.length;
- byte[] buffer = new byte[len];
- for (int i = 0; i < len; i++) {
- buffer[i] = (byte) (chars[i] & 0xff);
- }
- return buffer;
- }
-
- /**
- * Cleanup at the end of a test run
- * @param action action triggering the operation (for use in logging)
- * @param fileSystem filesystem to work with. May be null
- * @param cleanupPath path to delete as a string
- */
- public static void cleanup(String action,
- FileSystem fileSystem,
- String cleanupPath) {
- if (fileSystem == null) {
- return;
- }
- Path path = new Path(cleanupPath).makeQualified(fileSystem.getUri(),
- fileSystem.getWorkingDirectory());
- cleanup(action, fileSystem, path);
- }
-
- /**
- * Cleanup at the end of a test run
- * @param action action triggering the operation (for use in logging)
- * @param fileSystem filesystem to work with. May be null
- * @param path path to delete
- */
- public static void cleanup(String action, FileSystem fileSystem, Path path) {
- noteAction(action);
- try {
- rm(fileSystem, path, true, false);
- } catch (Exception e) {
- LOG.error("Error deleting in "+ action + " - " + path + ": " + e, e);
- }
- }
-
- /**
- * Delete a directory. There's a safety check for operations against the
- * root directory -these are intercepted and rejected with an IOException
- * unless the allowRootDelete flag is true
- * @param fileSystem filesystem to work with. May be null
- * @param path path to delete
- * @param recursive flag to enable recursive delete
- * @param allowRootDelete can the root directory be deleted?
- * @throws IOException on any problem.
- */
- public static boolean rm(FileSystem fileSystem,
- Path path,
- boolean recursive,
- boolean allowRootDelete) throws
- IOException {
- if (fileSystem != null) {
- rejectRootOperation(path, allowRootDelete);
- if (fileSystem.exists(path)) {
- return fileSystem.delete(path, recursive);
- }
- }
- return false;
-
- }
-
- /**
- * Block any operation on the root path. This is a safety check
- * @param path path in the filesystem
- * @param allowRootOperation can the root directory be manipulated?
- * @throws IOException if the operation was rejected
- */
- public static void rejectRootOperation(Path path,
- boolean allowRootOperation) throws IOException {
- if (path.isRoot() && !allowRootOperation) {
- throw new IOException("Root directory operation rejected: " + path);
- }
- }
-
- /**
- * Block any operation on the root path. This is a safety check
- * @param path path in the filesystem
- * @throws IOException if the operation was rejected
- */
- public static void rejectRootOperation(Path path) throws IOException {
- rejectRootOperation(path, false);
- }
-
-
- public static void noteAction(String action) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("============== "+ action +" =============");
- }
- }
-
- /**
- * downgrade a failure to a message and a warning, then an
- * exception for the Junit test runner to mark as failed
- * @param message text message
- * @param failure what failed
- * @throws AssumptionViolatedException always
- */
- public static void downgrade(String message, Throwable failure) {
- LOG.warn("Downgrading test " + message, failure);
- AssumptionViolatedException ave =
- new AssumptionViolatedException(failure, null);
- throw ave;
- }
-
- /**
- * report an overridden test as unsupported
- * @param message message to use in the text
- * @throws AssumptionViolatedException always
- */
- public static void unsupported(String message) {
- skip(message);
- }
-
- /**
- * report a test has been skipped for some reason
- * @param message message to use in the text
- * @throws AssumptionViolatedException always
- */
- public static void skip(String message) {
- LOG.info("Skipping: {}", message);
- throw new AssumptionViolatedException(message);
- }
-
- /**
- * Fail with an exception that was received
- * @param text text to use in the exception
- * @param thrown a (possibly null) throwable to init the cause with
- * @throws AssertionError with the text and throwable -always
- */
- public static void fail(String text, Throwable thrown) {
- AssertionError e = new AssertionError(text);
- e.initCause(thrown);
- throw e;
- }
-
- /**
- * Make an assertion about the length of a file
- * @param fs filesystem
- * @param path path of the file
- * @param expected expected length
- * @throws IOException on File IO problems
- */
- public static void assertFileHasLength(FileSystem fs, Path path,
- int expected) throws IOException {
- FileStatus status = fs.getFileStatus(path);
- assertEquals(
- "Wrong file length of file " + path + " status: " + status,
- expected,
- status.getLen());
- }
-
- /**
- * Assert that a path refers to a directory
- * @param fs filesystem
- * @param path path of the directory
- * @throws IOException on File IO problems
- */
- public static void assertIsDirectory(FileSystem fs,
- Path path) throws IOException {
- FileStatus fileStatus = fs.getFileStatus(path);
- assertIsDirectory(fileStatus);
- }
-
- /**
- * Assert that a path refers to a directory
- * @param fileStatus stats to check
- */
- public static void assertIsDirectory(FileStatus fileStatus) {
- assertTrue("Should be a directory -but isn't: " + fileStatus,
- fileStatus.isDirectory());
- }
-
- /**
- * Write the text to a file, returning the converted byte array
- * for use in validating the round trip
- * @param fs filesystem
- * @param path path of file
- * @param text text to write
- * @param overwrite should the operation overwrite any existing file?
- * @return the read bytes
- * @throws IOException on IO problems
- */
- public static byte[] writeTextFile(FileSystem fs,
- Path path,
- String text,
- boolean overwrite) throws IOException {
- byte[] bytes = new byte[0];
- if (text != null) {
- bytes = toAsciiByteArray(text);
- }
- createFile(fs, path, overwrite, bytes);
- return bytes;
- }
-
- /**
- * Create a file
- * @param fs filesystem
- * @param path path to write
- * @param overwrite overwrite flag
- * @param data source dataset. Can be null
- * @throws IOException on any problem
- */
- public static void createFile(FileSystem fs,
- Path path,
- boolean overwrite,
- byte[] data) throws IOException {
- FSDataOutputStream stream = fs.create(path, overwrite);
- if (data != null && data.length > 0) {
- stream.write(data);
- }
- stream.close();
- }
-
- /**
- * Touch a file
- * @param fs filesystem
- * @param path path
- * @throws IOException IO problems
- */
- public static void touch(FileSystem fs,
- Path path) throws IOException {
- createFile(fs, path, true, null);
- }
-
- /**
- * Delete a file/dir and assert that delete() returned true
- * <i>and</i> that the path no longer exists. This variant rejects
- * all operations on root directories
- * @param fs filesystem
- * @param file path to delete
- * @param recursive flag to enable recursive delete
- * @throws IOException IO problems
- */
- public static void assertDeleted(FileSystem fs,
- Path file,
- boolean recursive) throws IOException {
- assertDeleted(fs, file, recursive, false);
- }
-
- /**
- * Delete a file/dir and assert that delete() returned true
- * <i>and</i> that the path no longer exists. This variant rejects
- * all operations on root directories
- * @param fs filesystem
- * @param file path to delete
- * @param recursive flag to enable recursive delete
- * @param allowRootOperations can the root dir be deleted?
- * @throws IOException IO problems
- */
- public static void assertDeleted(FileSystem fs,
- Path file,
- boolean recursive,
- boolean allowRootOperations) throws IOException {
- rejectRootOperation(file, allowRootOperations);
- assertPathExists(fs, "about to be deleted file", file);
- boolean deleted = fs.delete(file, recursive);
- String dir = ls(fs, file.getParent());
- assertTrue("Delete failed on " + file + ": " + dir, deleted);
- assertPathDoesNotExist(fs, "Deleted file", file);
- }
-
- /**
- * Read in "length" bytes, convert to an ascii string
- * @param fs filesystem
- * @param path path to read
- * @param length #of bytes to read.
- * @return the bytes read and converted to a string
- * @throws IOException IO problems
- */
- public static String readBytesToString(FileSystem fs,
- Path path,
- int length) throws IOException {
- FSDataInputStream in = fs.open(path);
- try {
- byte[] buf = new byte[length];
- in.readFully(0, buf);
- return toChar(buf);
- } finally {
- in.close();
- }
- }
-
- /**
- * Take an array of filestats and convert to a string (prefixed w/ a [01] counter
- * @param stats array of stats
- * @param separator separator after every entry
- * @return a stringified set
- */
- public static String fileStatsToString(FileStatus[] stats, String separator) {
- StringBuilder buf = new StringBuilder(stats.length * 128);
- for (int i = 0; i < stats.length; i++) {
- buf.append(String.format("[%02d] %s", i, stats[i])).append(separator);
- }
- return buf.toString();
- }
-
- /**
- * List a directory
- * @param fileSystem FS
- * @param path path
- * @return a directory listing or failure message
- * @throws IOException
- */
- public static String ls(FileSystem fileSystem, Path path) throws IOException {
- if (path == null) {
- //surfaces when someone calls getParent() on something at the top of the path
- return "/";
- }
- FileStatus[] stats;
- String pathtext = "ls " + path;
- try {
- stats = fileSystem.listStatus(path);
- } catch (FileNotFoundException e) {
- return pathtext + " -file not found";
- } catch (IOException e) {
- return pathtext + " -failed: " + e;
- }
- return dumpStats(pathtext, stats);
- }
-
- public static String dumpStats(String pathname, FileStatus[] stats) {
- return pathname + fileStatsToString(stats, "\n");
- }
-
- /**
- * Assert that a file exists and whose {@link FileStatus} entry
- * declares that this is a file and not a symlink or directory.
- * @param fileSystem filesystem to resolve path against
- * @param filename name of the file
- * @throws IOException IO problems during file operations
- */
- public static void assertIsFile(FileSystem fileSystem, Path filename) throws
- IOException {
- assertPathExists(fileSystem, "Expected file", filename);
- FileStatus status = fileSystem.getFileStatus(filename);
- assertIsFile(filename, status);
- }
-
- /**
- * Assert that a file exists and whose {@link FileStatus} entry
- * declares that this is a file and not a symlink or directory.
- * @param filename name of the file
- * @param status file status
- */
- public static void assertIsFile(Path filename, FileStatus status) {
- String fileInfo = filename + " " + status;
- assertFalse("File claims to be a directory " + fileInfo,
- status.isDirectory());
- assertFalse("File claims to be a symlink " + fileInfo,
- status.isSymlink());
- }
-
- /**
- * Create a dataset for use in the tests; all data is in the range
- * base to (base+modulo-1) inclusive
- * @param len length of data
- * @param base base of the data
- * @param modulo the modulo
- * @return the newly generated dataset
- */
- public static byte[] dataset(int len, int base, int modulo) {
- byte[] dataset = new byte[len];
- for (int i = 0; i < len; i++) {
- dataset[i] = (byte) (base + (i % modulo));
- }
- return dataset;
- }
-
- /**
- * Assert that a path exists -but make no assertions as to the
- * type of that entry
- *
- * @param fileSystem filesystem to examine
- * @param message message to include in the assertion failure message
- * @param path path in the filesystem
- * @throws FileNotFoundException raised if the path is missing
- * @throws IOException IO problems
- */
- public static void assertPathExists(FileSystem fileSystem, String message,
- Path path) throws IOException {
- if (!fileSystem.exists(path)) {
- //failure, report it
- String listing = ls(fileSystem, path.getParent());
- throw new FileNotFoundException(message + ": not found " + path
- + " in \"" + path.getParent() + "\" :\n" + listing);
- }
- }
-
- /**
- * Assert that a path does not exist
- *
- * @param fileSystem filesystem to examine
- * @param message message to include in the assertion failure message
- * @param path path in the filesystem
- * @throws IOException IO problems
- */
- public static void assertPathDoesNotExist(FileSystem fileSystem,
- String message,
- Path path) throws IOException {
- try {
- FileStatus status = fileSystem.getFileStatus(path);
- fail(message + ": unexpectedly found " + path + " as " + status);
- } catch (FileNotFoundException expected) {
- //this is expected
-
- }
- }
-
- /**
- * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry
- * @param fs filesystem
- * @param dir directory to scan
- * @param subdir full path to look for
- * @throws IOException IO probles
- */
- public static void assertListStatusFinds(FileSystem fs,
- Path dir,
- Path subdir) throws IOException {
- FileStatus[] stats = fs.listStatus(dir);
- boolean found = false;
- StringBuilder builder = new StringBuilder();
- for (FileStatus stat : stats) {
- builder.append(stat.toString()).append('\n');
- if (stat.getPath().equals(subdir)) {
- found = true;
- }
- }
- assertTrue("Path " + subdir
- + " not found in directory " + dir + ":" + builder,
- found);
- }
-
- /**
- * Test for the host being an OSX machine
- * @return true if the JVM thinks that is running on OSX
- */
- public static boolean isOSX() {
- return System.getProperty("os.name").contains("OS X");
- }
-
- /**
- * compare content of file operations using a double byte array
- * @param concat concatenated files
- * @param bytes bytes
- */
- public static void validateFileContent(byte[] concat, byte[][] bytes) {
- int idx = 0;
- boolean mismatch = false;
-
- for (byte[] bb : bytes) {
- for (byte b : bb) {
- if (b != concat[idx++]) {
- mismatch = true;
- break;
- }
- }
- if (mismatch)
- break;
- }
- assertFalse("File content of file is not as expected at offset " + idx,
- mismatch);
- }
-
- /**
- * Receives test data from the given input file and checks the size of the
- * data as well as the pattern inside the received data.
- *
- * @param fs FileSystem
- * @param path Input file to be checked
- * @param expectedSize the expected size of the data to be read from the
- * input file in bytes
- * @param bufferLen Pattern length
- * @param modulus Pattern modulus
- * @throws IOException
- * thrown if an error occurs while reading the data
- */
- public static void verifyReceivedData(FileSystem fs, Path path,
- final long expectedSize,
- final int bufferLen,
- final int modulus) throws IOException {
- final byte[] testBuffer = new byte[bufferLen];
-
- long totalBytesRead = 0;
- int nextExpectedNumber = 0;
- final InputStream inputStream = fs.open(path);
- try {
- while (true) {
- final int bytesRead = inputStream.read(testBuffer);
- if (bytesRead < 0) {
- break;
- }
-
- totalBytesRead += bytesRead;
-
- for (int i = 0; i < bytesRead; ++i) {
- if (testBuffer[i] != nextExpectedNumber) {
- throw new IOException("Read number " + testBuffer[i]
- + " but expected " + nextExpectedNumber);
- }
-
- ++nextExpectedNumber;
-
- if (nextExpectedNumber == modulus) {
- nextExpectedNumber = 0;
- }
- }
- }
-
- if (totalBytesRead != expectedSize) {
- throw new IOException("Expected to read " + expectedSize +
- " bytes but only received " + totalBytesRead);
- }
- } finally {
- inputStream.close();
- }
- }
-
- /**
- * Generates test data of the given size according to some specific pattern
- * and writes it to the provided output file.
- *
- * @param fs FileSystem
- * @param path Test file to be generated
- * @param size The size of the test data to be generated in bytes
- * @param bufferLen Pattern length
- * @param modulus Pattern modulus
- * @throws IOException
- * thrown if an error occurs while writing the data
- */
- public static long generateTestFile(FileSystem fs, Path path,
- final long size,
- final int bufferLen,
- final int modulus) throws IOException {
- final byte[] testBuffer = new byte[bufferLen];
- for (int i = 0; i < testBuffer.length; ++i) {
- testBuffer[i] = (byte) (i % modulus);
- }
-
- final OutputStream outputStream = fs.create(path, false);
- long bytesWritten = 0;
- try {
- while (bytesWritten < size) {
- final long diff = size - bytesWritten;
- if (diff < testBuffer.length) {
- outputStream.write(testBuffer, 0, (int) diff);
- bytesWritten += diff;
- } else {
- outputStream.write(testBuffer);
- bytesWritten += testBuffer.length;
- }
- }
-
- return bytesWritten;
- } finally {
- outputStream.close();
- }
- }
-
- /**
- * Creates and reads a file with the given size. The test file is generated
- * according to a specific pattern so it can be easily verified even if it's
- * a multi-GB one.
- * During the read phase the incoming data stream is also checked against
- * this pattern.
- *
- * @param fs FileSystem
- * @param parent Test file parent dir path
- * @throws IOException
- * thrown if an I/O error occurs while writing or reading the test file
- */
- public static void createAndVerifyFile(FileSystem fs, Path parent, final long fileSize)
- throws IOException {
- int testBufferSize = fs.getConf()
- .getInt(IO_CHUNK_BUFFER_SIZE, DEFAULT_IO_CHUNK_BUFFER_SIZE);
- int modulus = fs.getConf()
- .getInt(IO_CHUNK_MODULUS_SIZE, DEFAULT_IO_CHUNK_MODULUS_SIZE);
-
- final String objectName = UUID.randomUUID().toString();
- final Path objectPath = new Path(parent, objectName);
-
- // Write test file in a specific pattern
- assertEquals(fileSize,
- generateTestFile(fs, objectPath, fileSize, testBufferSize, modulus));
- assertPathExists(fs, "not created successful", objectPath);
-
- // Now read the same file back and verify its content
- try {
- verifyReceivedData(fs, objectPath, fileSize, testBufferSize, modulus);
- } finally {
- // Delete test file
- fs.delete(objectPath, false);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java
deleted file mode 100644
index 06673d2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.utils;
-
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.slider.common.SliderXMLConfKeysForTesting;
-
-/**
- * Keys shared across tests.
- */
-public interface KeysForTests extends SliderKeys, SliderXMLConfKeysForTesting {
- /**
- * Username for all clusters, ZK, etc.
- */
- String USERNAME = "bigdataborat";
-
- int WAIT_TIME = 120;
- String WAIT_TIME_ARG = Integer.toString(WAIT_TIME);
-
- String SLIDER_TEST_XML = "slider-test.xml";
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java
deleted file mode 100644
index be452f1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.utils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.impl.zk.RegistryOperationsService;
-import org.apache.hadoop.registry.server.services.MicroZookeeperService;
-import org.apache.slider.common.tools.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-/**
- * Test ZK cluster.
- */
-public class MicroZKCluster implements Closeable {
- private static final Logger LOG =
- LoggerFactory.getLogger(MicroZKCluster.class);
-
- public static final String HOSTS = "127.0.0.1";
- private MicroZookeeperService zkService;
- private String zkBindingString;
- private final Configuration conf;
- private RegistryOperations registryOperations;
-
- MicroZKCluster() {
- this(SliderUtils.createConfiguration());
- }
-
- MicroZKCluster(Configuration conf) {
- this.conf = conf;
- }
-
- String getZkBindingString() {
- return zkBindingString;
- }
-
- void createCluster(String name) {
- zkService = new MicroZookeeperService(name);
-
- zkService.init(conf);
- zkService.start();
- zkBindingString = zkService.getConnectionString();
- LOG.info("Created {}", this);
- registryOperations = new RegistryOperationsService(
- "registry",
- zkService);
- registryOperations.init(conf);
- registryOperations.start();
- }
-
- @Override
- public void close() throws IOException {
- if (registryOperations != null) {
- registryOperations.stop();
- }
- if (zkService != null) {
- zkService.stop();
- }
- }
-
- @Override
- public String toString() {
- return "Micro ZK cluster as " + zkBindingString;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java
deleted file mode 100644
index 52875d3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.utils;
-
-/**
- * Outcome for probes.
- */
-public final class Outcome {
-
- private final String name;
-
- private Outcome(String name) {
- this.name = name;
- }
-
- public static final Outcome SUCCESS = new Outcome(
- "Success");
- public static final Outcome RETRY = new Outcome("Retry");
- public static final Outcome FAIL = new Outcome("Fail");
-
- /**
- * Build from a bool, where false is mapped to retry.
- * @param b boolean
- * @return an outcome
- */
- static Outcome fromBool(boolean b) {
- return b ? SUCCESS : RETRY;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java
deleted file mode 100644
index f7da585..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.utils;
-
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.slider.common.SliderXMLConfKeysForTesting;
-import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.rules.TestName;
-
-import java.io.File;
-
-
-/**
- * Base class for unit tests as well as ones starting mini clusters
- * -the foundational code and methods.
- *
- */
-public abstract class SliderTestBase extends SliderTestUtils {
-
- /**
- * Singleton metric registry.
- */
- public static final MetricsAndMonitoring METRICS = new MetricsAndMonitoring();
- public static final int WEB_STARTUP_TIME = 30000;
-
- @Rule
- public TestName methodName = new TestName();
-
- @BeforeClass
- public static void nameThread() {
- Thread.currentThread().setName("JUnit");
- }
-
- @Before
- public void setup() throws Exception {
- setSliderClientClassName(DEFAULT_SLIDER_CLIENT);
- FileUtil.fullyDelete(new File(SliderXMLConfKeysForTesting
- .TEST_SECURITY_DIR));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java
deleted file mode 100644
index 50d56b0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java
+++ /dev/null
@@ -1,1065 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.utils;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonParser;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.service.ServiceStateException;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Container;
-import org.apache.slider.client.SliderClient;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.slider.common.tools.Duration;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.main.LauncherExitCodes;
-import org.apache.slider.core.main.ServiceLaunchException;
-import org.apache.slider.core.main.ServiceLauncher;
-import org.apache.slider.core.persist.JsonSerDeser;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.server.services.workflow.ForkedProcessService;
-import org.codehaus.jackson.map.PropertyNamingStrategy;
-import org.junit.Assert;
-import org.junit.Assume;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_OPTION;
-
-/**
- * Static utils for tests in this package and in other test projects.
- *
- * It is designed to work with mini clusters as well as remote ones
- *
- * This class is not final and may be extended for test cases.
- *
- * Some of these methods are derived from the SwiftUtils and SwiftTestUtils
- * classes -replicated here so that they are available in Hadoop-2.0 code
- */
-public class SliderTestUtils extends Assert {
- private static final Logger LOG =
- LoggerFactory.getLogger(SliderTestUtils.class);
- public static final String DEFAULT_SLIDER_CLIENT = SliderClient.class
- .getName();
- private static String sliderClientClassName = DEFAULT_SLIDER_CLIENT;
-
- public static final Map<String, String> EMPTY_MAP = Collections.emptyMap();
- public static final Map<String, Integer> EMPTY_INT_MAP = Collections
- .emptyMap();
- public static final List<String> EMPTY_LIST = Collections.emptyList();
-
- public static final ObjectReader OBJECT_READER;
- public static final ObjectWriter OBJECT_WRITER;
-
- public static final JsonSerDeser<Application> JSON_SER_DESER =
- new JsonSerDeser<>(Application.class,
- PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
-
- static {
- ObjectMapper mapper = new ObjectMapper();
- OBJECT_READER = mapper.readerFor(Object.class);
- OBJECT_WRITER = mapper.writer();
- }
-
- /**
- * Action that returns an object.
- */
- public interface Action {
- Object invoke() throws Exception;
- }
-
- /**
- * Probe that returns an Outcome.
- */
- public interface Probe {
- Outcome invoke(Map args) throws Exception;
- }
-
- public static void setSliderClientClassName(String sliderClientClassName) {
- sliderClientClassName = sliderClientClassName;
- }
-
- public static void describe(String s) {
- LOG.info("");
- LOG.info("===============================");
- LOG.info(s);
- LOG.info("===============================");
- LOG.info("");
- }
-
- /**
- * Convert a JSON string to something readable.
- * @param json
- * @return a string for printing
- */
- public static String prettyPrintJson(String json) {
- Gson gson = new GsonBuilder().setPrettyPrinting().create();
- return gson.toJson(new JsonParser().parse(json));
- }
-
- /**
- * Convert an object to something readable.
- * @param src
- * @return a string for printing
- */
- public static String prettyPrintAsJson(Object src)
- throws JsonProcessingException, UnsupportedEncodingException {
- return new String(OBJECT_WRITER.writeValueAsBytes(src), "UTF8");
- }
-
- /**
- * Skip the test with a message.
- * @param message message logged and thrown
- */
- public static void skip(String message) {
- LOG.warn("Skipping test: {}", message);
- Assume.assumeTrue(message, false);
- }
-
- /**
- * Skip the test with a message if condition holds.
- * @param condition predicate
- * @param message message logged and thrown
- */
- public static void assume(boolean condition, String message) {
- if (!condition) {
- skip(message);
- }
- }
-
- /**
- * Skip a test if not running on Windows.
- */
- public static void assumeWindows() {
- assume(Shell.WINDOWS, "not windows");
- }
-
- /**
- * Skip a test if running on Windows.
- */
- public static void assumeNotWindows() {
- assume(!Shell.WINDOWS, "windows");
- }
-
- /**
- * Skip a test on windows.
- */
- public static void skipOnWindows() {
- assumeNotWindows();
- }
-
- /**
- * Equality size for a list.
- * @param left
- * @param right
- */
- public static void assertListEquals(List left, List right) {
- String lval = collectionToString(left);
- String rval = collectionToString(right);
- String text = "comparing " + lval + " to " + rval;
- assertEquals(text, left.size(), right.size());
- for (int i = 0; i < left.size(); i++) {
- assertEquals(text, left.get(i), right.get(i));
- }
- }
-
- /**
- * Assert a list has a given length.
- * @param list list
- * @param size size to have
- */
- public static void assertListLength(List list, int size) {
- String lval = collectionToString(list);
- assertEquals(lval, size, list.size());
- }
-
- /**
- * Stringify a collection with [ ] at either end.
- * @param collection collection
- * @return string value
- */
- public static String collectionToString(List collection) {
- return "[" + SliderUtils.join(collection, ", ", false) + "]";
- }
-
- /**
- * Assume that a string option is set and not equal to "".
- * @param conf configuration file
- * @param key key to look for
- */
- public static void assumeStringOptionSet(Configuration conf, String key) {
- if (SliderUtils.isUnset(conf.getTrimmed(key))) {
- skip("Configuration key " + key + " not set");
- }
- }
-
- /**
- * assert that a string option is set and not equal to "".
- * @param conf configuration file
- * @param key key to look for
- */
- public static void assertStringOptionSet(Configuration conf, String key) {
- getRequiredConfOption(conf, key);
- }
-
- /**
- * Assume that a boolean option is set and true.
- * Unset or false triggers a test skip
- * @param conf configuration file
- * @param key key to look for
- */
- public static void assumeBoolOptionTrue(Configuration conf, String key) {
- assumeBoolOption(conf, key, false);
- }
-
- /**
- * Assume that a boolean option is true.
- * False triggers a test skip
- * @param conf configuration file
- * @param key key to look for
- * @param defval default value if the property is not defined
- */
- public static void assumeBoolOption(
- Configuration conf, String key, boolean defval) {
- assume(conf.getBoolean(key, defval),
- "Configuration key " + key + " is false");
- }
-
- /**
- * Get a required config option (trimmed, incidentally).
- * Test will fail if not set
- * @param conf configuration
- * @param key key
- * @return the string
- */
- public static String getRequiredConfOption(Configuration conf, String key) {
- String val = conf.getTrimmed(key);
- if (SliderUtils.isUnset(val)) {
- fail("Missing configuration option " + key);
- }
- return val;
- }
-
- /**
- * Fails a test because required behavior has not been implemented.
- */
- public static void failNotImplemented() {
- fail("Not implemented");
- }
-
- /**
- * Assert that any needed libraries being present. On Unix none are needed;
- * on windows they must be present
- */
- public static void assertNativeLibrariesPresent() {
- String errorText = SliderUtils.checkForRequiredNativeLibraries();
- if (SliderUtils.isSet(errorText)) {
- fail(errorText);
- }
- }
-
- protected static String[] toArray(List<Object> args) {
- String[] converted = new String[args.size()];
- for (int i = 0; i < args.size(); i++) {
- Object elt = args.get(i);
- assertNotNull(args.get(i));
- converted[i] = elt.toString();
- }
- return converted;
- }
-
- public static void waitWhileClusterLive(SliderClient client, int timeout)
- throws IOException, YarnException {
- Duration duration = new Duration(timeout);
- duration.start();
- while (client.actionExists(client.getDeployedClusterName(), true) ==
- LauncherExitCodes.EXIT_SUCCESS && !duration.getLimitExceeded()) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- }
- }
- if (duration.getLimitExceeded()) {
- fail("Cluster " + client.getDeployedClusterName() + " still live after " +
- timeout + " ms");
- }
- }
-
- public static void waitUntilClusterLive(SliderClient client, int timeout)
- throws IOException, YarnException {
- Duration duration = new Duration(timeout);
- duration.start();
- while (LauncherExitCodes.EXIT_SUCCESS != client.actionExists(
- client.getDeployedClusterName(), true) &&
- !duration.getLimitExceeded()) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- }
- }
- if (duration.getLimitExceeded()) {
- fail("Cluster " + client.getDeployedClusterName() + " not live after " +
- timeout + " ms");
- }
- }
-
- public static void dumpClusterDescription(
- String text,
- Application status) throws IOException {
- describe(text);
- LOG.info(JSON_SER_DESER.toJson(status));
- }
-
- /**
- * Assert that a service operation succeeded.
- * @param service service
- */
- public static void assertSucceeded(ServiceLauncher service) {
- assertEquals(0, service.getServiceExitCode());
- }
-
- public static void assertContainersLive(Application application,
- String component, int expected) {
- LOG.info("Asserting component {} expected count {}", component, expected);
- int actual = extractLiveContainerCount(application, component);
- if (expected != actual) {
- LOG.warn("{} actual={}, expected {} in \n{}\n", component, actual,
- expected, application);
- }
- assertEquals(expected, actual);
- }
-
- /**
- * Robust extraction of live container count.
- * @param application status
- * @param component component to resolve
- * @return the number of containers live.
- */
- public static int extractLiveContainerCount(
- Application application,
- String component) {
- int actual = 0;
- if (application.getContainers() != null) {
- for (Container container : application.getContainers()) {
- if (container.getComponentName().equals(component)) {
- actual++;
- }
- }
- }
- return actual;
- }
-
- /**
- * Exec a set of commands, wait a few seconds for it to finish.
- * @param status code
- * @param commands
- * @return the process
- */
- public static ForkedProcessService exec(int status, List<String> commands)
- throws IOException, TimeoutException {
- ForkedProcessService process = exec(commands);
-
- Integer exitCode = process.getExitCode();
- assertNotNull(exitCode);
- assertEquals(status, exitCode.intValue());
- return process;
- }
-
- /**
- * Exec a set of commands, wait a few seconds for it to finish.
- * @param commands
- * @return
- */
- public static ForkedProcessService exec(List<String> commands)
- throws IOException, TimeoutException {
- ForkedProcessService process;
- process = new ForkedProcessService(
- commands.get(0),
- EMPTY_MAP,
- commands);
- process.init(new Configuration());
- process.start();
- int timeoutMillis = 5000;
- if (!process.waitForServiceToStop(timeoutMillis)) {
- throw new TimeoutException(
- "Process did not stop in " + timeoutMillis + "mS");
- }
- return process;
- }
-
- /**
- * Determine whether an application exists. Run the commands and if the
- * operation fails with a FileNotFoundException, then
- * this method returns false.
- * <p>
- * Run something harmless like a -version command, something
- * which must return 0
- *
- * @param commands
- * @return true if the command sequence succeeded
- * false if they failed with no file
- * @throws Exception on any other failure cause
- */
- public static boolean doesAppExist(List<String> commands)
- throws IOException, TimeoutException {
- try {
- exec(0, commands);
- return true;
- } catch (ServiceStateException e) {
- if (!(e.getCause() instanceof FileNotFoundException)) {
- throw e;
- }
- return false;
- }
- }
-
- /**
- * Locate an executable on the path.
- * @param exe executable name. If it is an absolute path which
- * exists then it will returned direct
- * @return the path to an exe or null for no match
- */
- public static File locateExecutable(String exe) {
- File exeNameAsPath = new File(exe).getAbsoluteFile();
- if (exeNameAsPath.exists()) {
- return exeNameAsPath;
- }
-
- File exepath = null;
- String path = extractPath();
- String[] dirs = path.split(System.getProperty("path.separator"));
- for (String dirname : dirs) {
- File dir = new File(dirname);
-
- File possible = new File(dir, exe);
- if (possible.exists()) {
- exepath = possible;
- }
- }
- return exepath;
- }
-
- /**
- * Lookup the PATH env var.
- * @return the path or null
- */
- public static String extractPath() {
- return extractEnvVar("PATH");
- }
-
- /**
- * Find an environment variable. Uses case independent checking for
- * the benefit of windows.
- * Will fail if the var is not found.
- * @param var path variable <i>in upper case</i>
- * @return the env var
- */
- public static String extractEnvVar(String var) {
- String realkey = "";
-
- for (String it : System.getenv().keySet()) {
- if (it.toUpperCase(Locale.ENGLISH).equals(var)) {
- realkey = it;
- }
- }
-
- if (SliderUtils.isUnset(realkey)) {
- fail("No environment variable " + var + " found");
- }
- String val = System.getenv(realkey);
-
- LOG.info("{} = {}", realkey, val);
- return val;
- }
-
- /**
- * Create a temp JSON file. After coming up with the name, the file
- * is deleted
- * @return the filename
- */
- public static File createTempJsonFile() throws IOException {
- return tmpFile(".json");
- }
-
- /**
- * Create a temp file with the specific name. It's deleted after creation,
- * to avoid "file exists exceptions"
- * @param suffix suffix, e.g. ".txt"
- * @return a path to a file which may be created
- */
- public static File tmpFile(String suffix) throws IOException {
- File reportFile = File.createTempFile(
- "temp",
- suffix,
- new File("target"));
- reportFile.delete();
- return reportFile;
- }
-
- /**
- * Execute a closure, assert it fails with a given exit code and text.
- * @param exitCode exit code
- * @param text text (can be "")
- * @param action action
- * @return
- */
- public void assertFailsWithException(int exitCode,
- String text,
- Action action) throws Exception {
- try {
- action.invoke();
- fail("Operation was expected to fail —but it succeeded");
- } catch (ServiceLaunchException e) {
- assertExceptionDetails(e, exitCode, text);
- }
- }
-
- /**
- * Execute a closure, assert it fails with a given exit code and text.
- * @param text text (can be "")
- * @param action action
- * @return
- */
- public void assertFailsWithExceptionClass(Class clazz,
- String text,
- Action action) throws Exception {
- try {
- action.invoke();
- fail("Operation was expected to fail —but it succeeded");
- } catch (Exception e) {
- assertExceptionDetails(e, clazz, text);
- }
- }
-
- public static void assertExceptionDetails(
- ServiceLaunchException ex,
- int exitCode) {
- assertExceptionDetails(ex, exitCode, null);
- }
-
- /**
- * Make an assertion about the exit code of an exception.
- * @param ex exception
- * @param exitCode exit code
- * @param text error text to look for in the exception
- */
- public static void assertExceptionDetails(
- ServiceLaunchException ex,
- int exitCode,
- String text) {
- if (exitCode != ex.getExitCode()) {
- String message = String.format("Wrong exit code, expected %d but" +
- " got %d in %s", exitCode, ex.getExitCode(), ex);
- LOG.warn(message, ex);
- throw new AssertionError(message, ex);
- }
- if (SliderUtils.isSet(text)) {
- if (!(ex.toString().contains(text))) {
- String message = String.format("String match for \"%s\"failed in %s",
- text, ex);
- LOG.warn(message, ex);
- throw new AssertionError(message, ex);
- }
- }
- }
-
- /**
- * Make an assertion about the class of an exception.
- * @param ex exception
- * @param clazz exit code
- * @param text error text to look for in the exception
- */
- static void assertExceptionDetails(
- Exception ex,
- Class clazz,
- String text) throws Exception {
- if (ex.getClass() != clazz) {
- throw ex;
- }
- if (SliderUtils.isSet(text) && !(ex.toString().contains(text))) {
- throw ex;
- }
- }
-
- /**
- * Launch the slider client with the specific args; no validation
- * of return code takes place.
- * @param conf configuration
- * @param args arg list
- * @return the launcher
- */
- protected static ServiceLauncher<SliderClient> execSliderCommand(
- Configuration conf,
- List args) throws Throwable {
- ServiceLauncher<SliderClient> serviceLauncher =
- new ServiceLauncher<>(sliderClientClassName);
-
- LOG.debug("slider {}", SliderUtils.join(args, " ", false));
- serviceLauncher.launchService(conf,
- toArray(args),
- false);
- return serviceLauncher;
- }
-
- /**
- * Launch a slider command to a given exit code.
- * Most failures will trigger exceptions; this is for the exit code of the
- * runService() call.
- * @param exitCode desired exit code
- * @param conf configuration
- * @param args arg list
- * @return the launcher
- */
- protected static ServiceLauncher<SliderClient> execSliderCommand(
- int exitCode,
- Configuration conf,
- List args) throws Throwable {
- ServiceLauncher<SliderClient> serviceLauncher = execSliderCommand(conf,
- args);
- assertEquals(exitCode, serviceLauncher.getServiceExitCode());
- return serviceLauncher;
- }
-
- public static ServiceLauncher launch(Class serviceClass,
- Configuration conf,
- List<Object> args) throws
- Throwable {
- ServiceLauncher serviceLauncher =
- new ServiceLauncher(serviceClass.getName());
-
- String joinedArgs = SliderUtils.join(args, " ", false);
- LOG.debug("slider {}", joinedArgs);
-
- serviceLauncher.launchService(conf,
- toArray(args),
- false);
- return serviceLauncher;
- }
-
- public static Throwable launchExpectingException(Class serviceClass,
- Configuration conf,
- String expectedText,
- List args)
- throws Throwable {
- try {
- ServiceLauncher launch = launch(serviceClass, conf, args);
- throw new AssertionError("Expected an exception with text containing " +
- expectedText + " -but the service completed with exit code " +
- launch.getServiceExitCode());
- } catch (AssertionError error) {
- throw error;
- } catch (Throwable thrown) {
- if (SliderUtils.isSet(expectedText) && !thrown.toString().contains(
- expectedText)) {
- //not the right exception -rethrow
- LOG.warn("Caught Exception did not contain expected text" +
- "\"" + expectedText + "\"");
- throw thrown;
- }
- return thrown;
- }
- }
-
-
- public static ServiceLauncher<SliderClient> launchClientAgainstRM(
- String address,
- List<String> args,
- Configuration conf) throws Throwable {
- assertNotNull(address);
- LOG.info("Connecting to rm at {}", address);
- if (!args.contains(Arguments.ARG_MANAGER)) {
- args.add(Arguments.ARG_MANAGER);
- args.add(address);
- }
- ServiceLauncher<SliderClient> launcher = execSliderCommand(conf, args);
- return launcher;
- }
-
- /**
- * Add a configuration parameter as a cluster configuration option.
- * @param extraArgs extra arguments
- * @param conf config
- * @param option option
- */
- public static void addClusterConfigOption(
- List<String> extraArgs,
- YarnConfiguration conf,
- String option) {
-
- conf.getTrimmed(option);
- extraArgs.add(ARG_OPTION);
- extraArgs.add(option);
- extraArgs.add(getRequiredConfOption(conf, option));
- }
-
- /**
- * Assert that a path refers to a directory.
- * @param fs filesystem
- * @param path path of the directory
- * @throws IOException on File IO problems
- */
- public static void assertIsDirectory(FileSystem fs,
- Path path) throws IOException {
- FileStatus fileStatus = fs.getFileStatus(path);
- assertIsDirectory(fileStatus);
- }
-
- /**
- * Assert that a path refers to a directory.
- * @param fileStatus stats to check
- */
- public static void assertIsDirectory(FileStatus fileStatus) {
- assertTrue("Should be a dir -but isn't: " + fileStatus,
- fileStatus.isDirectory());
- }
-
- /**
- * Assert that a path exists -but make no assertions as to the
- * type of that entry.
- *
- * @param fileSystem filesystem to examine
- * @param message message to include in the assertion failure message
- * @param path path in the filesystem
- * @throws IOException IO problems
- */
- public static void assertPathExists(
- FileSystem fileSystem,
- String message,
- Path path) throws IOException {
- if (!fileSystem.exists(path)) {
- //failure, report it
- fail(
- message + ": not found \"" + path + "\" in " + path.getParent() +
- "-" +
- ls(fileSystem, path.getParent()));
- }
- }
-
- /**
- * Assert that a path does not exist.
- *
- * @param fileSystem filesystem to examine
- * @param message message to include in the assertion failure message
- * @param path path in the filesystem
- * @throws IOException IO problems
- */
- public static void assertPathDoesNotExist(
- FileSystem fileSystem,
- String message,
- Path path) throws IOException {
- try {
- FileStatus status = fileSystem.getFileStatus(path);
- // a status back implies there is a file here
- fail(message + ": unexpectedly found " + path + " as " + status);
- } catch (FileNotFoundException expected) {
- //this is expected
-
- }
- }
-
- /**
- * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry.
- * @param fs filesystem
- * @param dir directory to scan
- * @param subdir full path to look for
- * @throws IOException IO probles
- */
- public static void assertListStatusFinds(FileSystem fs,
- Path dir,
- Path subdir) throws IOException {
- FileStatus[] stats = fs.listStatus(dir);
- boolean found = false;
- StringBuilder builder = new StringBuilder();
- for (FileStatus stat : stats) {
- builder.append(stat.toString()).append('\n');
- if (stat.getPath().equals(subdir)) {
- found = true;
- }
- }
- assertTrue("Path " + subdir
- + " not found in directory " + dir + ":" + builder,
- found);
- }
-
- /**
- * List a a path to string.
- * @param fileSystem filesystem
- * @param path directory
- * @return a listing of the filestatuses of elements in the directory, one
- * to a line, precedeed by the full path of the directory
- * @throws IOException connectivity problems
- */
- public static String ls(FileSystem fileSystem, Path path)
- throws IOException {
- if (path == null) {
- //surfaces when someone calls getParent() on something at the top of
- // the path
- return "/";
- }
- FileStatus[] stats;
- String pathtext = "ls " + path;
- try {
- stats = fileSystem.listStatus(path);
- } catch (FileNotFoundException e) {
- return pathtext + " -file not found";
- } catch (IOException e) {
- return pathtext + " -failed: " + e;
- }
- return pathtext + fileStatsToString(stats, "\n");
- }
-
- /**
- * Take an array of filestats and convert to a string (prefixed w/ a [01]
- * counter).
- * @param stats array of stats
- * @param separator separator after every entry
- * @return a stringified set
- */
- public static String fileStatsToString(FileStatus[] stats, String separator) {
- StringBuilder buf = new StringBuilder(stats.length * 128);
- for (int i = 0; i < stats.length; i++) {
- buf.append(String.format("[%02d] %s", i, stats[i])).append(separator);
- }
- return buf.toString();
- }
-
- public static void waitWhileClusterLive(SliderClient sliderClient)
- throws IOException, YarnException {
- waitWhileClusterLive(sliderClient, 30000);
- }
-
- public static void dumpRegistryInstances(
- Map<String, ServiceRecord> instances) {
- describe("service registry slider instances");
- for (Entry<String, ServiceRecord> it : instances.entrySet()) {
- LOG.info(" {} : {}", it.getKey(), it.getValue());
- }
- describe("end list service registry slider instances");
- }
-
-
- public static void dumpRegistryInstanceIDs(List<String> instanceIds) {
- describe("service registry instance IDs");
- dumpCollection(instanceIds);
- }
-
- public static void dumpRegistryServiceTypes(Collection<String> entries) {
- describe("service registry types");
- dumpCollection(entries);
- }
-
- public static <V> void dumpCollection(Collection<V> entries) {
- LOG.info("number of entries: {}", entries.size());
- for (V it : entries) {
- LOG.info(it.toString());
- }
- }
-
- public static void dumpArray(Object[] entries) {
- LOG.info("number of entries: {}", entries.length);
- for (Object it : entries) {
- LOG.info(it.toString());
- }
- }
-
- public static <K, V> void dumpMap(Map<K, V> map) {
- for (Entry<K, V> it : map.entrySet()) {
- LOG.info("\"{}\": \"{}\"", it.getKey().toString(), it.getValue()
- .toString());
- }
- }
-
- /**
- * Get a time option in seconds if set, otherwise the default value (also
- * in seconds).
- * This operation picks up the time value as a system property if set -that
- * value overrides anything in the test file
- * @param conf
- * @param key
- * @param defValMillis
- * @return
- */
- public static int getTimeOptionMillis(
- Configuration conf,
- String key,
- int defValMillis) {
- int val = conf.getInt(key, 0);
- val = Integer.getInteger(key, val);
- int time = 1000 * val;
- if (time == 0) {
- time = defValMillis;
- }
- return time;
- }
-
- public void dumpConfigurationSet(PublishedConfigSet confSet) {
- for (String key : confSet.keys()) {
- PublishedConfiguration config = confSet.get(key);
- LOG.info("{} -- {}", key, config.description);
- }
- }
-
- /**
- * Convert a file to a URI suitable for use in an argument.
- * @param file file
- * @return a URI string valid on all platforms
- */
- public String toURIArg(File file) {
- return file.getAbsoluteFile().toURI().toString();
- }
-
- /**
- * Assert a file exists; fails with a listing of the parent dir.
- * @param text text for front of message
- * @param file file to look for
- * @throws FileNotFoundException
- */
- public void assertFileExists(String text, File file)
- throws FileNotFoundException {
- if (!file.exists()) {
- File parent = file.getParentFile();
- String[] files = parent.list();
- StringBuilder builder = new StringBuilder();
- builder.append(parent.getAbsolutePath());
- builder.append(":\n");
- for (String name : files) {
- builder.append(" ");
- builder.append(name);
- builder.append("\n");
- }
- throw new FileNotFoundException(text + ": " + file + " not found in " +
- builder);
- }
- }
-
- /**
- * Repeat a probe until it succeeds, if it does not execute a failure
- * closure then raise an exception with the supplied message.
- * @param probe probe
- * @param timeout time in millis before giving up
- * @param sleepDur sleep between failing attempts
- * @param args map of arguments to the probe
- * @param failIfUnsuccessful if the probe fails after all the attempts
- * —should it raise an exception
- * @param failureMessage message to include in exception raised
- * @param failureHandler closure to invoke prior to the failure being raised
- */
- protected void repeatUntilSuccess(
- String action,
- Probe probe,
- int timeout,
- int sleepDur,
- Map args,
- boolean failIfUnsuccessful,
- String failureMessage,
- Action failureHandler) throws Exception {
- LOG.debug("Probe {} timelimit {}", action, timeout);
- if (timeout < 1000) {
- fail("Timeout " + timeout + " too low: milliseconds are expected, not " +
- "seconds");
- }
- int attemptCount = 1;
- boolean succeeded = false;
- boolean completed = false;
- Duration duration = new Duration(timeout);
- duration.start();
- while (!completed) {
- Outcome outcome = probe.invoke(args);
- if (outcome.equals(Outcome.SUCCESS)) {
- // success
- LOG.debug("Success after {} attempt(s)", attemptCount);
- succeeded = true;
- completed = true;
- } else if (outcome.equals(Outcome.RETRY)) {
- // failed but retry possible
- attemptCount++;
- completed = duration.getLimitExceeded();
- if (!completed) {
- LOG.debug("Attempt {} failed", attemptCount);
- try {
- Thread.sleep(sleepDur);
- } catch (InterruptedException e) {
- }
- }
- } else if (outcome.equals(Outcome.FAIL)) {
- // fast fail
- LOG.debug("Fast fail of probe");
- completed = true;
- }
- }
- if (!succeeded) {
- if (duration.getLimitExceeded()) {
- LOG.info("probe timed out after {} and {} attempts", timeout,
- attemptCount);
- }
- if (failureHandler != null) {
- failureHandler.invoke();
- }
- if (failIfUnsuccessful) {
- fail(failureMessage);
- }
- }
- }
-
- /**
- * Get a value from a map; raise an assertion if it is not there.
- * @param map map to look up
- * @param key key
- * @return the string value
- */
- public <K, V> String requiredMapValue(Map<K, V> map, String key) {
- assertNotNull(map.get(key));
- return map.get(key).toString();
- }
-
- public static void assertStringContains(String expected, String text) {
- assertNotNull("null text", text);
- if (!text.contains(expected)) {
- String message = String.format("did not find %s in \"%s\"", expected,
- text);
- LOG.error(message);
- fail(message);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java
deleted file mode 100644
index 2dfb204..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.utils;
-
-import org.apache.slider.api.resource.Application;
-import org.junit.Test;
-
-import java.util.Collections;
-
-/**
- * Test for some of the command test base operations.
- */
-public class TestAssertions {
-
- public static final String CLUSTER_JSON = "json/cluster.json";
-
- //@Test
- public void testNoInstances() throws Throwable {
- Application application = new Application();
- application.setContainers(null);
- SliderTestUtils.assertContainersLive(application, "example", 0);
- }
-
- //@Test
- public void testEmptyInstances() throws Throwable {
- Application application = new Application();
- application.setContainers(Collections.emptyList());
- SliderTestUtils.assertContainersLive(application, "example", 0);
- }
-
-// TODO test metrics retrieval
-// //@Test
-// public void testLiveInstances() throws Throwable {
-// InputStream stream = getClass().getClassLoader().getResourceAsStream(
-// CLUSTER_JSON);
-// assertNotNull("could not load " + CLUSTER_JSON, stream);
-// ClusterDescription liveCD = ClusterDescription.fromStream(stream);
-// assertNotNull(liveCD);
-// SliderTestUtils.assertContainersLive(liveCD, "SLEEP_LONG", 4);
-// assertEquals((Integer) 1, liveCD.statistics.get("SLEEP_LONG").get(
-// StatusKeys.STATISTICS_CONTAINERS_ANTI_AFFINE_PENDING));
-// }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
deleted file mode 100644
index bf71861..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.restclient;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.api.json.JSONConfiguration;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.slider.core.exceptions.ExceptionConverter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-
-/**
- * Class to bond to a Jersey client, for UGI integration and SPNEGO.
- * <p>
- * Usage: create an instance, then when creating a Jersey <code>Client</code>
- * pass in to the constructor the handler provided by {@link #getHandler()}
- *
- * see <a href="https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html">Jersey docs</a>
- */
-public class UgiJerseyBinding implements
- HttpURLConnectionFactory {
- private static final Logger log =
- LoggerFactory.getLogger(UgiJerseyBinding.class);
-
- private final UrlConnectionOperations operations;
- private final URLConnectionClientHandler handler;
-
- /**
- * Construct an instance
- * @param operations operations instance
- */
- @SuppressWarnings("ThisEscapedInObjectConstruction")
- public UgiJerseyBinding(UrlConnectionOperations operations) {
- Preconditions.checkArgument(operations != null, "Null operations");
- this.operations = operations;
- handler = new URLConnectionClientHandler(this);
- }
-
- /**
- * Create an instance off the configuration. The SPNEGO policy
- * is derived from the current UGI settings.
- * @param conf config
- */
- public UgiJerseyBinding(Configuration conf) {
- this(new UrlConnectionOperations(conf));
- }
-
- /**
- * Get a URL connection.
- * @param url URL to connect to
- * @return the connection
- * @throws IOException any problem. {@link AuthenticationException}
- * errors are wrapped
- */
- @Override
- public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
- try {
- // open a connection handling status codes and so redirections
- // but as it opens a connection, it's less useful than you think.
-
- return operations.openConnection(url);
- } catch (AuthenticationException e) {
- throw new IOException(e);
- }
- }
-
- public UrlConnectionOperations getOperations() {
- return operations;
- }
-
- public URLConnectionClientHandler getHandler() {
- return handler;
- }
-
- /**
- * Get the SPNEGO flag (as found in the operations instance
- * @return the spnego policy
- */
- public boolean isUseSpnego() {
- return operations.isUseSpnego();
- }
-
-
- /**
- * Uprate error codes 400 and up into faults;
- * <p>
- * see {@link ExceptionConverter#convertJerseyException(String, String, UniformInterfaceException)}
- */
- public static IOException uprateFaults(HttpVerb verb, String url,
- UniformInterfaceException ex)
- throws IOException {
- return ExceptionConverter.convertJerseyException(verb.getVerb(),
- url, ex);
- }
-
- /**
- * Create the standard Jersey client Config
- * @return the recommended Jersey Client config
- */
- public ClientConfig createJerseyClientConfig() {
- ClientConfig clientConfig = new DefaultClientConfig();
- clientConfig.getFeatures().put(JSONConfiguration.FEATURE_POJO_MAPPING, true);
- return clientConfig;
- }
-
- /**
- * Create a jersey client bonded to this handler, using the
- * supplied client config
- * @param clientConfig client configuratin
- * @return a new client instance to use
- */
- public Client createJerseyClient(ClientConfig clientConfig) {
- return new Client(getHandler(), clientConfig);
- }
-
- /**
- * Create a jersey client bonded to this handler, using the
- * client config created with {@link #createJerseyClientConfig()}
- * @return a new client instance to use
- */
- public Client createJerseyClient() {
- return createJerseyClient(createJerseyClientConfig());
- }
-
-}
-
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
deleted file mode 100644
index 46f0d02..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.restclient;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.hadoop.yarn.webapp.ForbiddenException;
-import org.apache.hadoop.yarn.webapp.NotFoundException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.ssl.SSLException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.HttpURLConnection;
-import java.net.URL;
-
-/**
- * Operations on the JDK UrlConnection class.
- *
- */
-public class UrlConnectionOperations extends Configured {
- private static final Logger log =
- LoggerFactory.getLogger(UrlConnectionOperations.class);
-
- private SliderURLConnectionFactory connectionFactory;
-
- private boolean useSpnego = false;
-
- /**
- * Create an instance off the configuration. The SPNEGO policy
- * is derived from the current UGI settings.
- * @param conf config
- */
- public UrlConnectionOperations(Configuration conf) {
- super(conf);
- connectionFactory = SliderURLConnectionFactory.newInstance(conf);
- if (UserGroupInformation.isSecurityEnabled()) {
- log.debug("SPNEGO is enabled");
- setUseSpnego(true);
- }
- }
-
-
- public boolean isUseSpnego() {
- return useSpnego;
- }
-
- public void setUseSpnego(boolean useSpnego) {
- this.useSpnego = useSpnego;
- }
-
- /**
- * Opens a url with cache disabled, redirect handled in
- * (JDK) implementation.
- *
- * @param url to open
- * @return URLConnection
- * @throws IOException
- * @throws AuthenticationException authentication failure
- */
- public HttpURLConnection openConnection(URL url) throws
- IOException,
- AuthenticationException {
- Preconditions.checkArgument(url.getPort() != 0, "no port");
- return (HttpURLConnection) connectionFactory.openConnection(url, useSpnego);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/BlockingZKWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/BlockingZKWatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/BlockingZKWatcher.java
deleted file mode 100644
index ca49888..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/BlockingZKWatcher.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.zk;
-
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.ConnectException;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public class BlockingZKWatcher implements Watcher {
-
- protected static final Logger log =
- LoggerFactory.getLogger(BlockingZKWatcher.class);
- private final AtomicBoolean connectedFlag = new AtomicBoolean(false);
-
- @Override
- public void process(WatchedEvent event) {
- log.info("ZK binding callback received");
- connectedFlag.set(true);
- synchronized (connectedFlag) {
- try {
- connectedFlag.notify();
- } catch (Exception e) {
- log.warn("failed while waiting for notification", e);
- }
- }
- }
-
- /**
- * Wait for a flag to go true
- * @param timeout timeout in millis
- */
-
- public void waitForZKConnection(int timeout)
- throws InterruptedException, ConnectException {
- synchronized (connectedFlag) {
- if (!connectedFlag.get()) {
- log.info("waiting for ZK event");
- //wait a bit
- connectedFlag.wait(timeout);
- }
- }
- if (!connectedFlag.get()) {
- throw new ConnectException("Unable to connect to ZK quorum");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
deleted file mode 100644
index 1af883e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.zk;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.zookeeper.server.NIOServerCnxnFactory;
-import org.apache.zookeeper.server.ZooKeeperServer;
-import org.apache.zookeeper.server.persistence.FileTxnLog;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.io.Reader;
-import java.net.BindException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-
-/**
- * This is a version of the HBase ZK cluster cut out to be standalone.
- *
- * <i>Important: keep this Java6 language level for now</i>
- */
-public class MiniZooKeeperCluster extends AbstractService {
- private static final Logger LOG = LoggerFactory.getLogger(
- MiniZooKeeperCluster.class);
-
- private static final int TICK_TIME = 2000;
- private static final int CONNECTION_TIMEOUT = 30000;
- public static final int MAX_CLIENT_CONNECTIONS = 1000;
-
- private boolean started;
-
- /** The default port. If zero, we use a random port. */
- private int defaultClientPort = 0;
-
- private int clientPort;
-
- private final List<NIOServerCnxnFactory> standaloneServerFactoryList;
- private final List<ZooKeeperServer> zooKeeperServers;
- private final List<Integer> clientPortList;
-
- private int activeZKServerIndex;
- private int tickTime = 0;
- private File baseDir;
- private final int numZooKeeperServers;
- private String zkQuorum = "";
-
- public MiniZooKeeperCluster(int numZooKeeperServers) {
- super("MiniZooKeeperCluster");
- this.numZooKeeperServers = numZooKeeperServers;
- this.started = false;
- activeZKServerIndex = -1;
- zooKeeperServers = new ArrayList<ZooKeeperServer>();
- clientPortList = new ArrayList<Integer>();
- standaloneServerFactoryList = new ArrayList<NIOServerCnxnFactory>();
- }
-
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- }
-
- public void setDefaultClientPort(int clientPort) {
- if (clientPort <= 0) {
- throw new IllegalArgumentException("Invalid default ZK client port: "
- + clientPort);
- }
- this.defaultClientPort = clientPort;
- }
-
- /**
- * Selects a ZK client port. Returns the default port if specified.
- * Otherwise, returns a random port. The random port is selected from the
- * range between 49152 to 65535. These ports cannot be registered with IANA
- * and are intended for dynamic allocation (see http://bit.ly/dynports).
- */
- private int selectClientPort(Random r) {
- if (defaultClientPort > 0) {
- return defaultClientPort;
- }
- return 0xc000 + r.nextInt(0x3f00);
- }
-
- public void setTickTime(int tickTime) {
- this.tickTime = tickTime;
- }
-
- public int getBackupZooKeeperServerNum() {
- return zooKeeperServers.size() - 1;
- }
-
- public int getZooKeeperServerNum() {
- return zooKeeperServers.size();
- }
-
- // / XXX: From o.a.zk.t.ClientBase
- private static void setupTestEnv() {
- // during the tests we run with 100K prealloc in the logs.
- // on windows systems prealloc of 64M was seen to take ~15seconds
- // resulting in test failure (client timeout on first session).
- // set env and directly in order to handle static init/gc issues
- System.setProperty("zookeeper.preAllocSize", "100");
- FileTxnLog.setPreallocSize(100 * 1024);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- startup();
- }
-
- /**
- * @return ClientPort server bound to, -1 if there was a
- * binding problem and we couldn't pick another port.
- * @throws IOException
- * @throws InterruptedException
- */
- private int startup() throws IOException,
- InterruptedException {
- if (numZooKeeperServers <= 0)
- return -1;
-
- setupTestEnv();
- started = true;
- baseDir = File.createTempFile("zookeeper", ".dir");
- recreateDir(baseDir);
-
- StringBuilder quorumList = new StringBuilder();
- Random rnd = new Random();
- int tentativePort = selectClientPort(rnd);
-
- // running all the ZK servers
- for (int i = 0; i < numZooKeeperServers; i++) {
- File dir = new File(baseDir, "zookeeper_" + i).getAbsoluteFile();
- recreateDir(dir);
- int tickTimeToUse;
- if (this.tickTime > 0) {
- tickTimeToUse = this.tickTime;
- } else {
- tickTimeToUse = TICK_TIME;
- }
- ZooKeeperServer server = new ZooKeeperServer(dir, dir, tickTimeToUse);
- NIOServerCnxnFactory standaloneServerFactory;
- while (true) {
- try {
- standaloneServerFactory = new NIOServerCnxnFactory();
- standaloneServerFactory.configure(
- new InetSocketAddress(tentativePort),
- MAX_CLIENT_CONNECTIONS
- );
- } catch (BindException e) {
- LOG.debug("Failed binding ZK Server to client port: " +
- tentativePort, e);
- // We're told to use some port but it's occupied, fail
- if (defaultClientPort > 0) return -1;
- // This port is already in use, try to use another.
- tentativePort = selectClientPort(rnd);
- continue;
- }
- break;
- }
-
- // Start up this ZK server
- standaloneServerFactory.startup(server);
- if (!waitForServerUp(tentativePort, CONNECTION_TIMEOUT)) {
- throw new IOException("Waiting for startup of standalone server");
- }
-
- // We have selected this port as a client port.
- clientPortList.add(tentativePort);
- standaloneServerFactoryList.add(standaloneServerFactory);
- zooKeeperServers.add(server);
- if (quorumList.length() > 0) {
- quorumList.append(",");
- }
- quorumList.append("localhost:").append(tentativePort);
- tentativePort++; //for the next server
- }
-
- // set the first one to be active ZK; Others are backups
- activeZKServerIndex = 0;
-
- clientPort = clientPortList.get(activeZKServerIndex);
- zkQuorum = quorumList.toString();
- LOG.info("Started MiniZK Cluster and connect 1 ZK server " +
- "on client port: " + clientPort);
- return clientPort;
- }
-
- private void recreateDir(File dir) throws IOException {
- if (dir.exists()) {
- if (!FileUtil.fullyDelete(dir)) {
- throw new IOException("Could not delete zk base directory: " + dir);
- }
- }
- try {
- dir.mkdirs();
- } catch (SecurityException e) {
- throw new IOException("creating dir: " + dir, e);
- }
- }
-
- @Override
- protected void serviceStop() throws Exception {
-
- if (!started) {
- return;
- }
- started = false;
-
- try {
- // shut down all the zk servers
- for (int i = 0; i < standaloneServerFactoryList.size(); i++) {
- NIOServerCnxnFactory standaloneServerFactory =
- standaloneServerFactoryList.get(i);
- int clientPort = clientPortList.get(i);
-
- standaloneServerFactory.shutdown();
- if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
- throw new IOException("Waiting for shutdown of standalone server");
- }
- }
- for (ZooKeeperServer zkServer : zooKeeperServers) {
- //explicitly close ZKDatabase since ZookeeperServer does not close them
- zkServer.getZKDatabase().close();
- }
- } finally {
- // clear everything
- activeZKServerIndex = 0;
- standaloneServerFactoryList.clear();
- clientPortList.clear();
- zooKeeperServers.clear();
- }
-
- LOG.info("Shutdown MiniZK cluster with all ZK servers");
- }
-
- /**@return clientPort return clientPort if there is another ZK backup can run
- * when killing the current active; return -1, if there is no backups.
- * @throws IOException
- * @throws InterruptedException
- */
- public int killCurrentActiveZooKeeperServer() throws IOException,
- InterruptedException {
- if (!started || activeZKServerIndex < 0) {
- return -1;
- }
-
- // Shutdown the current active one
- NIOServerCnxnFactory standaloneServerFactory =
- standaloneServerFactoryList.get(activeZKServerIndex);
- int clientPort = clientPortList.get(activeZKServerIndex);
-
- standaloneServerFactory.shutdown();
- if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
- throw new IOException("Waiting for shutdown of standalone server");
- }
-
- zooKeeperServers.get(activeZKServerIndex).getZKDatabase().close();
-
- // remove the current active zk server
- standaloneServerFactoryList.remove(activeZKServerIndex);
- clientPortList.remove(activeZKServerIndex);
- zooKeeperServers.remove(activeZKServerIndex);
- LOG.info("Kill the current active ZK servers in the cluster " +
- "on client port: " + clientPort);
-
- if (standaloneServerFactoryList.size() == 0) {
- // there is no backup servers;
- return -1;
- }
- clientPort = clientPortList.get(activeZKServerIndex);
- LOG.info("Activate a backup zk server in the cluster " +
- "on client port: " + clientPort);
- // return the next back zk server's port
- return clientPort;
- }
-
- /**
- * Kill one back up ZK servers
- * @throws IOException
- * @throws InterruptedException
- */
- public void killOneBackupZooKeeperServer() throws IOException,
- InterruptedException {
- if (!started || activeZKServerIndex < 0 ||
- standaloneServerFactoryList.size() <= 1) {
- return;
- }
-
- int backupZKServerIndex = activeZKServerIndex + 1;
- // Shutdown the current active one
- NIOServerCnxnFactory standaloneServerFactory =
- standaloneServerFactoryList.get(backupZKServerIndex);
- int clientPort = clientPortList.get(backupZKServerIndex);
-
- standaloneServerFactory.shutdown();
- if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
- throw new IOException("Waiting for shutdown of standalone server");
- }
-
- zooKeeperServers.get(backupZKServerIndex).getZKDatabase().close();
-
- // remove this backup zk server
- standaloneServerFactoryList.remove(backupZKServerIndex);
- clientPortList.remove(backupZKServerIndex);
- zooKeeperServers.remove(backupZKServerIndex);
- LOG.info("Kill one backup ZK servers in the cluster " +
- "on client port: " + clientPort);
- }
-
- // XXX: From o.a.zk.t.ClientBase
- private static boolean waitForServerDown(int port, long timeout) throws
- InterruptedException {
- long start = System.currentTimeMillis();
- while (true) {
- try {
- Socket sock = null;
- try {
- sock = new Socket("localhost", port);
- OutputStream outstream = sock.getOutputStream();
- outstream.write("stat".getBytes("UTF-8"));
- outstream.flush();
- } finally {
- IOUtils.closeSocket(sock);
- }
- } catch (IOException e) {
- return true;
- }
-
- if (System.currentTimeMillis() > start + timeout) {
- break;
- }
- Thread.sleep(250);
- }
- return false;
- }
-
- // XXX: From o.a.zk.t.ClientBase
- private static boolean waitForServerUp(int port, long timeout) throws
- InterruptedException {
- long start = System.currentTimeMillis();
- while (true) {
- try {
- Socket sock = null;
- sock = new Socket("localhost", port);
- BufferedReader reader = null;
- try {
- OutputStream outstream = sock.getOutputStream();
- outstream.write("stat".getBytes("UTF-8"));
- outstream.flush();
-
- Reader isr = new InputStreamReader(sock.getInputStream(), "UTF-8");
- reader = new BufferedReader(isr);
- String line = reader.readLine();
- if (line != null && line.startsWith("Zookeeper version:")) {
- return true;
- }
- } finally {
- IOUtils.closeSocket(sock);
- IOUtils.closeStream(reader);
- }
- } catch (IOException e) {
- // ignore as this is expected
- LOG.debug("server localhost:" + port + " not up " + e);
- }
-
- if (System.currentTimeMillis() > start + timeout) {
- break;
- }
- Thread.sleep(250);
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKCallback.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKCallback.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKCallback.java
deleted file mode 100644
index 045b72c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKCallback.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.zk;
-
-import org.apache.zookeeper.Watcher;
-
-/**
- * Relays ZK watcher events to a closure
- */
-public abstract class ZKCallback implements Watcher {
-
- public ZKCallback() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
deleted file mode 100644
index 519cd16..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.zk;
-
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.data.ACL;
-import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-
-public class ZKIntegration implements Watcher, Closeable {
-
-/**
- * Base path for services
- */
- public static final String ZK_SERVICES = "services";
- /**
- * Base path for all Slider references
- */
- public static final String ZK_SLIDER = "slider";
- public static final String ZK_USERS = "users";
- public static final String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
- public static final String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
-
- private static final List<String> ZK_USERS_PATH_LIST = new ArrayList<String>();
- static {
- ZK_USERS_PATH_LIST.add(ZK_SERVICES);
- ZK_USERS_PATH_LIST.add(ZK_SLIDER);
- ZK_USERS_PATH_LIST.add(ZK_USERS);
- }
-
- public static final int SESSION_TIMEOUT = 30000;
- protected static final Logger log =
- LoggerFactory.getLogger(ZKIntegration.class);
- private ZooKeeper zookeeper;
- private final String username;
- private final String clustername;
- private final String userPath;
- private int sessionTimeout = SESSION_TIMEOUT;
- private static final Map<String, ZooKeeper> ZK_SESSIONS = new HashMap<>();
-
-/**
- flag to set to indicate that the user path should be created if
- it is not already there
- */
- private final AtomicBoolean toInit = new AtomicBoolean(false);
- private final boolean createClusterPath;
- private final Watcher watchEventHandler;
- private final String zkConnection;
- private final boolean canBeReadOnly;
-
- protected ZKIntegration(String zkConnection,
- String username,
- String clustername,
- boolean canBeReadOnly,
- boolean createClusterPath,
- Watcher watchEventHandler,
- int sessionTimeout
- ) throws IOException {
- this.username = username;
- this.clustername = clustername;
- this.watchEventHandler = watchEventHandler;
- this.zkConnection = zkConnection;
- this.canBeReadOnly = canBeReadOnly;
- this.createClusterPath = createClusterPath;
- this.sessionTimeout = sessionTimeout;
- this.userPath = mkSliderUserPath(username);
- }
-
- /**
- * Returns true only if an active ZK session is available and retrieved from
- * cache, false when it has to create a new one.
- *
- * @return true if from cache, false when new session created
- * @throws IOException
- */
- public synchronized boolean init() throws IOException {
- if (zookeeper != null && getAlive()) {
- return true;
- }
-
- synchronized (ZK_SESSIONS) {
- if (ZK_SESSIONS.containsKey(zkConnection)) {
- zookeeper = ZK_SESSIONS.get(zkConnection);
- }
- if (zookeeper == null || !getAlive()) {
- log.info("Binding ZK client to {}", zkConnection);
- zookeeper = new ZooKeeper(zkConnection, sessionTimeout, this,
- canBeReadOnly);
- ZK_SESSIONS.put(zkConnection, zookeeper);
- return false;
- } else {
- return true;
- }
- }
- }
-
- /**
- * Create an instance bonded to the specific closure
- * @param zkConnection
- * @param username
- * @param clustername
- * @param canBeReadOnly
- * @param watchEventHandler
- * @return the new instance
- * @throws IOException
- */
- public static ZKIntegration newInstance(String zkConnection,
- String username,
- String clustername,
- boolean createClusterPath,
- boolean canBeReadOnly,
- Watcher watchEventHandler,
- int sessionTimeout) throws IOException {
-
- return new ZKIntegration(zkConnection,
- username,
- clustername,
- canBeReadOnly,
- createClusterPath,
- watchEventHandler,
- sessionTimeout);
- }
-
-
- @Override
- public synchronized void close() throws IOException {
- if (zookeeper != null) {
- try {
- zookeeper.close();
- } catch (InterruptedException ignored) {
-
- }
- zookeeper = null;
- }
- }
-
- public String getConnectionString() {
- return zkConnection;
- }
-
- public String getClusterPath() {
- return mkClusterPath(username, clustername);
- }
-
- public boolean getConnected() {
- return zookeeper.getState().isConnected();
- }
-
- public boolean getAlive() {
- return zookeeper.getState().isAlive();
- }
-
- public ZooKeeper.States getState() {
- return zookeeper.getState();
- }
-
- public Stat getClusterStat() throws KeeperException, InterruptedException {
- return stat(getClusterPath());
- }
-
- public boolean exists(String path) throws
- KeeperException,
- InterruptedException {
- return stat(path) != null;
- }
-
- public Stat stat(String path) throws KeeperException, InterruptedException {
- return zookeeper.exists(path, false);
- }
-
- @Override
- public String toString() {
- return "ZK integration bound @ " + zkConnection + ": " + zookeeper;
- }
-
-/**
- * Event handler to notify of state events
- * @param event
- */
- @Override
- public void process(WatchedEvent event) {
- log.debug("{}", event);
- try {
- maybeInit();
- } catch (Exception e) {
- log.error("Failed to init", e);
- }
- if (watchEventHandler != null) {
- watchEventHandler.process(event);
- }
- }
-
- private void maybeInit() throws KeeperException, InterruptedException {
- if (!toInit.getAndSet(true) && createClusterPath) {
- log.debug("initing");
- //create the user path
- mkPath(ZK_USERS_PATH_LIST, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- //create the specific user
- createPath(userPath, null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- }
- }
-
- /**
- * Create a path under a parent, don't care if it already exists
- * As the path isn't returned, this isn't the way to create sequentially
- * numbered nodes.
- * @param parent parent dir. Must have a trailing / if entry!=null||empty
- * @param entry entry -can be null or "", in which case it is not appended
- * @param acl
- * @param createMode
- * @return the path if created; null if not
- */
- public String createPath(String parent,
- String entry,
- List<ACL> acl,
- CreateMode createMode) throws KeeperException, InterruptedException {
- //initial create of full path
- assert acl != null;
- assert !acl.isEmpty();
- assert parent != null;
- String path = parent;
- if (entry != null) {
- path = path + entry;
- }
- try {
- log.debug("Creating ZK path {}", path);
- return zookeeper.create(path, null, acl, createMode);
- } catch (KeeperException.NodeExistsException ignored) {
- //node already there
- log.debug("node already present:{}",path);
- return null;
- }
- }
-
- /**
- * Recursive path create
- * @param paths path list
- * @param acl acl list
- * @param createMode create modes
- */
- public void mkPath(List<String> paths,
- List<ACL> acl,
- CreateMode createMode) throws KeeperException, InterruptedException {
- String history = "/";
- for (String entry : paths) {
- createPath(history, entry, acl, createMode);
- history = history + entry + "/";
- }
- }
-
- /**
- * Delete a node, does not throw an exception if the path is not fond
- * @param path path to delete
- * @return true if the path could be deleted, false if there was no node to delete
- *
- */
- public boolean delete(String path) throws
- InterruptedException,
- KeeperException {
- try {
- zookeeper.delete(path, -1);
- log.debug("Deleting {}", path);
- return true;
- } catch (KeeperException.NoNodeException ignored) {
- return false;
- }
- }
-
- /**
- * Recursively delete a node, does not throw exception if any node does not exist.
- * @param path
- * @return true if delete was successful
- */
- public boolean deleteRecursive(String path) throws KeeperException, InterruptedException {
-
- try {
- List<String> children = zookeeper.getChildren(path, false);
- for (String child : children) {
- deleteRecursive(path + "/" + child);
- }
- delete(path);
- } catch (KeeperException.NoNodeException ignored) {
- return false;
- }
-
- return true;
- }
-
- /**
- * Build the path to a cluster; exists once the cluster has come up.
- * Even before that, a ZK watcher could wait for it.
- * @param username user
- * @param clustername name of the cluster
- * @return a strin
- */
- public static String mkClusterPath(String username, String clustername) {
- return mkSliderUserPath(username) + "/" + clustername;
- }
-/**
- * Build the path to a cluster; exists once the cluster has come up.
- * Even before that, a ZK watcher could wait for it.
- * @param username user
- * @return a string
- */
- public static String mkSliderUserPath(String username) {
- return SVC_SLIDER_USERS + "/" + username;
- }
-
- /**
- * Blocking enum of users.
- * @return an unordered list of clusters under a user
- */
- public List<String> getClusters() throws KeeperException,
- InterruptedException {
- return zookeeper.getChildren(userPath, null);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKPathBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKPathBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKPathBuilder.java
deleted file mode 100644
index b088568..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKPathBuilder.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.zk;
-
-import java.util.Locale;
-
-public final class ZKPathBuilder {
-
- private final String username, appname, clustername;
- private final String quorum;
-
- private String appPath;
- private String registryPath;
- private final String appQuorum;
-
- public ZKPathBuilder(String username,
- String appname,
- String clustername,
- String quorum,
- String appQuorum) {
- this.username = username;
- this.appname = appname;
- this.clustername = clustername;
- this.quorum = quorum;
- appPath = buildAppPath();
- registryPath = buildRegistryPath();
- this.appQuorum = appQuorum;
- }
-
- public String buildAppPath() {
- return String.format(Locale.ENGLISH, "/yarnapps_%s_%s_%s", appname,
- username, clustername);
-
- }
-
- public String buildRegistryPath() {
- return String.format(Locale.ENGLISH, "/services_%s_%s_%s", appname,
- username, clustername);
-
- }
-
- public String getQuorum() {
- return quorum;
- }
-
- public String getAppQuorum() {
- return appQuorum;
- }
-
- public String getAppPath() {
- return appPath;
- }
-
- public void setAppPath(String appPath) {
- this.appPath = appPath;
- }
-
- public String getRegistryPath() {
- return registryPath;
- }
-
- public void setRegistryPath(String registryPath) {
- this.registryPath = registryPath;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
deleted file mode 100644
index cc1b2c9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.zk;
-
-import com.google.common.net.HostAndPort;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class ZookeeperUtils {
- public static final int DEFAULT_PORT = 2181;
-
- public static String buildConnectionString(String zkHosts, int port) {
- String zkPort = Integer.toString(port);
- //parse the hosts
- String[] hostlist = zkHosts.split(",", 0);
- String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
- return quorum;
- }
-
- /**
- * Take a quorum list and split it to (trimmed) pairs
- * @param hostPortQuorumList list of form h1:port, h2:port2,...
- * @return a possibly empty list of values between commas. They may not be
- * valid hostname:port pairs
- */
- public static List<String> splitToPairs(String hostPortQuorumList) {
- // split an address hot
- String[] strings = StringUtils.getStrings(hostPortQuorumList);
- int len = 0;
- if (strings != null) {
- len = strings.length;
- }
- List<String> tuples = new ArrayList<String>(len);
- if (strings != null) {
- for (String s : strings) {
- tuples.add(s.trim());
- }
- }
- return tuples;
- }
-
- /**
- * Split a quorum list into a list of hostnames and ports
- * @param hostPortQuorumList split to a list of hosts and ports
- * @return a list of values
- */
- public static List<HostAndPort> splitToHostsAndPorts(String hostPortQuorumList) {
- // split an address hot
- String[] strings = StringUtils.getStrings(hostPortQuorumList);
- int len = 0;
- if (strings != null) {
- len = strings.length;
- }
- List<HostAndPort> list = new ArrayList<HostAndPort>(len);
- if (strings != null) {
- for (String s : strings) {
- list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
- }
- }
- return list;
- }
-
- /**
- * Build up to a hosts only list
- * @param hostAndPorts
- * @return a list of the hosts only
- */
- public static String buildHostsOnlyList(List<HostAndPort> hostAndPorts) {
- StringBuilder sb = new StringBuilder();
- for (HostAndPort hostAndPort : hostAndPorts) {
- sb.append(hostAndPort.getHostText()).append(",");
- }
- if (sb.length() > 0) {
- sb.delete(sb.length() - 1, sb.length());
- }
- return sb.toString();
- }
-
- public static String buildQuorumEntry(HostAndPort hostAndPort,
- int defaultPort) {
- String s = hostAndPort.toString();
- if (hostAndPort.hasPort()) {
- return s;
- } else {
- return s + ":" + defaultPort;
- }
- }
-
- /**
- * Build a quorum list, injecting a ":defaultPort" ref if needed on
- * any entry without one
- * @param hostAndPorts
- * @param defaultPort
- * @return
- */
- public static String buildQuorum(List<HostAndPort> hostAndPorts, int defaultPort) {
- List<String> entries = new ArrayList<String>(hostAndPorts.size());
- for (HostAndPort hostAndPort : hostAndPorts) {
- entries.add(buildQuorumEntry(hostAndPort, defaultPort));
- }
- return SliderUtils.join(entries, ",", false);
- }
-
- public static String convertToHostsOnlyList(String quorum) throws
- BadConfigException {
- List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
- return ZookeeperUtils.buildHostsOnlyList(hostAndPorts);
- }
-
- public static List<HostAndPort> splitToHostsAndPortsStrictly(String quorum) throws
- BadConfigException {
- List<HostAndPort> hostAndPorts =
- ZookeeperUtils.splitToHostsAndPorts(quorum);
- if (hostAndPorts.isEmpty()) {
- throw new BadConfigException("empty zookeeper quorum");
- }
- return hostAndPorts;
- }
-
- public static int getFirstPort(String quorum, int defVal) throws
- BadConfigException {
- List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
- int port = hostAndPorts.get(0).getPortOrDefault(defVal);
- return port;
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
deleted file mode 100644
index 1e1b1b8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers;
-
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.provider.ProviderService;
-import org.apache.hadoop.yarn.service.provider.ProviderUtils;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.ContainerState;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.CommandLineBuilder;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
-
-public abstract class AbstractProviderService extends AbstractService
- implements ProviderService, SliderKeys {
-
- protected static final Logger log =
- LoggerFactory.getLogger(AbstractProviderService.class);
- private static final ProviderUtils providerUtils = new ProviderUtils();
- protected StateAccessForProviders amState;
- protected YarnRegistryViewForProviders yarnRegistry;
- private ServiceTimelinePublisher serviceTimelinePublisher;
-
- protected AbstractProviderService(String name) {
- super(name);
- }
-
- public abstract void processArtifact(ContainerLauncher launcher,
- Application application, RoleInstance roleInstance,
- SliderFileSystem fileSystem) throws IOException;
-
-
- public void buildContainerLaunchContext(ContainerLauncher launcher,
- Application application, Container container, ProviderRole providerRole,
- SliderFileSystem fileSystem, RoleInstance roleInstance)
- throws IOException, SliderException {
- Component component = providerRole.component;
- processArtifact(launcher, application, roleInstance, fileSystem);
-
- // Generate tokens (key-value pair) for config substitution.
- // Get pre-defined tokens
- Map<String, String> globalTokens = amState.getGlobalSubstitutionTokens();
- Map<String, String> tokensForSubstitution = providerUtils
- .initCompTokensForSubstitute(null);
- tokensForSubstitution.putAll(globalTokens);
- // Set the environment variables in launcher
- launcher.putEnv(SliderUtils
- .buildEnvMap(component.getConfiguration(), tokensForSubstitution));
- launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
- launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
- if (System.getenv(HADOOP_USER_NAME) != null) {
- launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
- }
- launcher.setEnv("LANG", "en_US.UTF-8");
- launcher.setEnv("LC_ALL", "en_US.UTF-8");
- launcher.setEnv("LANGUAGE", "en_US.UTF-8");
-
- for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
- tokensForSubstitution.put($(entry.getKey()), entry.getValue());
- }
- providerUtils.addComponentHostTokens(tokensForSubstitution, amState);
-
- // create config file on hdfs and add local resource
-
- // substitute launch command
- String launchCommand = ProviderUtils
- .substituteStrWithTokens(component.getLaunchCommand(),
- tokensForSubstitution);
- CommandLineBuilder operation = new CommandLineBuilder();
- operation.add(launchCommand);
- operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
- launcher.addCommand(operation.build());
-
- // publish exports
- providerUtils
- .substituteMapWithTokens(application.getQuicklinks(), tokensForSubstitution);
- PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS,
- application.getQuicklinks().entrySet());
- amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf);
- if (serviceTimelinePublisher != null) {
- serviceTimelinePublisher.serviceAttemptUpdated(application);
- }
- }
-
- public boolean processContainerStatus(ContainerId containerId,
- ContainerStatus status) {
- log.debug("Handling container status: {}", status);
- if (SliderUtils.isEmpty(status.getIPs()) ||
- SliderUtils.isUnset(status.getHost())) {
- return true;
- }
- RoleInstance instance = amState.getOwnedContainer(containerId);
- if (instance == null) {
- // container is completed?
- return false;
- }
-
- // TODO publish ip and host
- org.apache.slider.api.resource.Container container =
- instance.providerRole.component.getContainer(containerId.toString());
- if (container != null) {
- container.setIp(StringUtils.join(",", status.getIPs()));
- container.setHostname(status.getHost());
- container.setState(ContainerState.READY);
- } else {
- log.warn(containerId + " not found in Application!");
- }
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/MonitorDetail.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/MonitorDetail.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/MonitorDetail.java
deleted file mode 100644
index 27d3415..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/MonitorDetail.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers;
-
-/**
- * Details about some exported information from a provider to the AM web UI.
- */
-public class MonitorDetail {
-
- private final String value;
- private final boolean isUrl;
-
- public MonitorDetail(String value, boolean isUrl) {
- this.value = value;
- this.isUrl = isUrl;
- }
-
- public String getValue() {
- return value;
- }
-
- public boolean isUrl() {
- return isUrl;
- }
-
- public String toString() {
- return "MonitorDetail[" + value + " isUrl=" + isUrl + "]";
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
deleted file mode 100644
index 128dd5d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-/**
- * Placement values.
- * This is nominally a bitmask, though not all values make sense
- */
-public class PlacementPolicy {
-
- /**
- * Default value: history used, anti-affinity hinted at on rebuild/flex up
- */
- public static final int NONE = 0;
-
- /**
- * Default value: history used, anti-affinity hinted at on rebuild/flex up
- */
- public static final int DEFAULT = NONE;
-
- /**
- * Strict placement: when asking for an instance for which there is
- * history, mandate that it is strict
- */
- public static final int STRICT = 1;
-
- /**
- * No data locality; do not use placement history
- */
- public static final int ANYWHERE = 2;
-
- /**
- * @Deprecated: use {@link #ANYWHERE}
- */
- @Deprecated
- public static final int NO_DATA_LOCALITY = ANYWHERE;
-
- /**
- * Anti-affinity is mandatory.
- */
- public static final int ANTI_AFFINITY_REQUIRED = 4;
-
- /**
- * Exclude from flexing; used internally to mark AMs.
- */
- public static final int EXCLUDE_FROM_FLEXING = 16;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/PlacementPolicyOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/PlacementPolicyOptions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/PlacementPolicyOptions.java
deleted file mode 100644
index e61f944..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/PlacementPolicyOptions.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-public enum PlacementPolicyOptions {
-
- EXCLUDE_FROM_FLEXING,
- NO_DATA_LOCALITY,
- ANTI_AFFINITY_REQUIRED,
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCompleted.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCompleted.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCompleted.java
deleted file mode 100644
index f6ff4fd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCompleted.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-/**
- * This is the callback triggered by the {@link ProviderCompletedCallable}
- * when it generates a notification
- */
-public interface ProviderCompleted {
-
- public void eventCallbackEvent(Object parameter);
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCompletedCallable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCompletedCallable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCompletedCallable.java
deleted file mode 100644
index 47939c9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCompletedCallable.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-import java.util.concurrent.Callable;
-
-public class ProviderCompletedCallable implements Callable<Object> {
-
- private final ProviderCompleted callback;
- private final Object parameter;
-
- public ProviderCompletedCallable(ProviderCompleted callback, Object parameter) {
- this.callback = callback;
- this.parameter = parameter;
- }
-
- @Override
- public Object call() throws Exception {
- callback.eventCallbackEvent(parameter);
- return parameter;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
deleted file mode 100644
index b07fc29..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-import org.apache.hadoop.conf.Configuration;
-
-import java.util.List;
-public interface ProviderCore {
-
- String getName();
-
- List<ProviderRole> getRoles();
-
- Configuration getConf();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
deleted file mode 100644
index 6fd85bf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.servicemonitor.MonitorUtils;
-import org.apache.slider.server.servicemonitor.Probe;
-
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Provider role and key for use in app requests.
- *
- * This class uses the role name as the key for hashes and in equality tests,
- * and ignores the other values.
- */
-public final class ProviderRole {
- public final String name;
- public final int id;
- public int placementPolicy;
- public int nodeFailureThreshold;
- public final long placementTimeoutSeconds;
- public final String labelExpression;
- public final Component component;
- public AtomicLong componentIdCounter = null;
- public Queue<RoleInstance> failedInstances = new ConcurrentLinkedQueue<>();
- public Probe probe;
-
- public ProviderRole(String name, int id) {
- this(name,
- id,
- PlacementPolicy.DEFAULT,
- ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
- ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS,
- ResourceKeys.DEF_YARN_LABEL_EXPRESSION);
- }
-
- /**
- * Create a provider role
- * @param name role/component name
- * @param id ID. This becomes the YARN priority
- * @param policy placement policy
- * @param nodeFailureThreshold threshold for node failures (within a reset interval)
- * after which a node failure is considered an app failure
- * @param placementTimeoutSeconds for lax placement, timeout in seconds before
- * @param labelExpression label expression for requests; may be null
- */
- public ProviderRole(String name,
- int id,
- int policy,
- int nodeFailureThreshold,
- long placementTimeoutSeconds,
- String labelExpression) {
- this(name,
- id,
- policy,
- nodeFailureThreshold,
- placementTimeoutSeconds,
- labelExpression,
- new Component().name(name).numberOfContainers(0L));
- }
-
- /**
- * Create a provider role with a role group
- * @param name role/component name
- * @param id ID. This becomes the YARN priority
- * @param policy placement policy
- * @param nodeFailureThreshold threshold for node failures (within a reset interval)
- * after which a node failure is considered an app failure
- * @param placementTimeoutSeconds for lax placement, timeout in seconds before
- * @param labelExpression label expression for requests; may be null
- */
- public ProviderRole(String name, int id, int policy,
- int nodeFailureThreshold, long placementTimeoutSeconds,
- String labelExpression, Component component) {
- this.name = name;
- this.id = id;
- this.placementPolicy = policy;
- this.nodeFailureThreshold = nodeFailureThreshold;
- this.placementTimeoutSeconds = placementTimeoutSeconds;
- this.labelExpression = labelExpression;
- this.component = component;
- if(component.getUniqueComponentSupport()) {
- componentIdCounter = new AtomicLong(0);
- }
- this.probe = MonitorUtils.getProbe(component.getReadinessCheck());
- }
-
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- ProviderRole that = (ProviderRole) o;
- return name.equals(that.name);
- }
-
- @Override
- public int hashCode() {
- return name.hashCode();
- }
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder("ProviderRole{");
- sb.append("name='").append(name).append('\'');
- sb.append(", id=").append(id);
- sb.append(", placementPolicy=").append(placementPolicy);
- sb.append(", nodeFailureThreshold=").append(nodeFailureThreshold);
- sb.append(", placementTimeoutSeconds=").append(placementTimeoutSeconds);
- sb.append(", labelExpression='").append(labelExpression).append('\'');
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java
deleted file mode 100644
index 288f25a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
-
-/**
- * Interface of AM operations
- */
-public interface AppMasterActionOperations extends RMOperationHandlerActions {
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java
deleted file mode 100644
index 65b88cf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-
-import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
-
-import java.net.InetSocketAddress;
-import java.security.PrivilegedAction;
-
-/**
- * Implement privileged connection to the CM
- *
- */
-public class PrivilegedConnectToCM implements PrivilegedAction<ContainerManagementProtocol> {
- final SliderAppMaster appMaster;
- final InetSocketAddress cmAddress;
-
- public PrivilegedConnectToCM(SliderAppMaster appMaster,
- InetSocketAddress cmAddress) {
- this.appMaster = appMaster;
- this.cmAddress = cmAddress;
- }
-
-
- @Override //PrivilegedAction
- public ContainerManagementProtocol run() {
- return ((ContainerManagementProtocol) appMaster.getProxy(
- ContainerManagementProtocol.class,
- cmAddress));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
deleted file mode 100644
index 5d52441..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.slider.server.appmaster.state.AbstractClusterServices;
-
-public class ProtobufClusterServices extends AbstractClusterServices {
-
- public Resource newResource() {
- return Records.newRecord(Resource.class);
- }
-
- @Override
- public Resource newResource(int memory, int cores) {
- return Resources.createResource(memory, cores);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java
deleted file mode 100644
index fdc386f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-/**
- * This is the name of YARN artifacts that are published
- */
-public interface PublishedArtifacts {
-
- String COMPLETE_CONFIG = "complete-config";
- String CORE_SITE_CONFIG = "core-site";
- String HDFS_SITE_CONFIG = "hdfs-site";
- String YARN_SITE_CONFIG = "yarn-site";
- String LOG4J = "log4j";
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[86/86] [abbrv] hadoop git commit: YARN-7201. Added an apache httpd
example YARN service. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-7201. Added an apache httpd example YARN service. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f7a50d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f7a50d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f7a50d8
Branch: refs/heads/yarn-native-services
Commit: 3f7a50d8d50787331ae5806a946d6d1d52b5ec55
Parents: 360b2d2
Author: Jian He <ji...@apache.org>
Authored: Mon Sep 25 16:36:43 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 25 16:37:27 2017 -0700
----------------------------------------------------------------------
.../examples/httpd-no-dns/httpd-no-dns.json | 62 ++++++++
.../httpd-no-dns/httpd-proxy-no-dns.conf | 24 +++
.../examples/httpd/httpd-proxy.conf | 24 +++
.../examples/httpd/httpd.json | 55 +++++++
.../yarn/service/api/ServiceApiConstants.java | 5 +
.../yarn/service/component/Component.java | 31 ++++
.../yarn/service/provider/ProviderUtils.java | 1 +
.../hadoop/registry/server/dns/RegistryDNS.java | 4 +
.../src/site/markdown/yarn-service/Examples.md | 159 +++++++++++++++++++
.../src/site/markdown/yarn-service/Overview.md | 3 +-
10 files changed, 367 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f7a50d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
new file mode 100644
index 0000000..6b35538
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
@@ -0,0 +1,62 @@
+{
+ "name": "httpd-service-no-dns",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "httpd",
+ "number_of_containers": 2,
+ "artifact": {
+ "id": "centos/httpd-24-centos7:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/bin/run-httpd",
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "readiness_check": {
+ "type": "HTTP",
+ "props": {
+ "url": "http://${THIS_HOST}:8080"
+ }
+ },
+ "configuration": {
+ "files": [
+ {
+ "type": "ENV",
+ "dest_file": "/var/www/html/index.html",
+ "props": {
+ "content": "<html><header><title>Title</title></header><body>Hello from ${COMPONENT_INSTANCE_NAME}!</body></html>"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "httpd-proxy",
+ "number_of_containers": 1,
+ "dependencies": [ "httpd" ],
+ "artifact": {
+ "id": "centos/httpd-24-centos7:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/bin/run-httpd",
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "configuration": {
+ "files": [
+ {
+ "type": "TEMPLATE",
+ "dest_file": "/etc/httpd/conf.d/httpd-proxy.conf",
+ "src_file": "httpd-proxy-no-dns.conf"
+ }
+ ]
+ }
+ }
+ ],
+ "quicklinks": {
+ "Apache HTTP Server": "http://httpd-proxy-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080"
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f7a50d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf
new file mode 100644
index 0000000..9894e64
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+<Proxy balancer://test>
+ BalancerMember http://${HTTPD-0_IP}:8080
+ BalancerMember http://${HTTPD-1_IP}:8080
+ ProxySet lbmethod=bytraffic
+</Proxy>
+
+ProxyPass "/" "balancer://test/"
+ProxyPassReverse "/" "balancer://test/"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f7a50d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd-proxy.conf
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd-proxy.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd-proxy.conf
new file mode 100644
index 0000000..e8651a5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd-proxy.conf
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+<Proxy balancer://test>
+ BalancerMember http://httpd-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080
+ BalancerMember http://httpd-1.${SERVICE_NAME}.${USER}.${DOMAIN}:8080
+ ProxySet lbmethod=bytraffic
+</Proxy>
+
+ProxyPass "/" "balancer://test/"
+ProxyPassReverse "/" "balancer://test/"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f7a50d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json
new file mode 100644
index 0000000..e63376d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json
@@ -0,0 +1,55 @@
+{
+ "name": "httpd-service",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "httpd",
+ "number_of_containers": 2,
+ "artifact": {
+ "id": "centos/httpd-24-centos7:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/bin/run-httpd",
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "configuration": {
+ "files": [
+ {
+ "type": "ENV",
+ "dest_file": "/var/www/html/index.html",
+ "props": {
+ "content": "<html><header><title>Title</title></header><body>Hello from ${COMPONENT_INSTANCE_NAME}!</body></html>"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "httpd-proxy",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "centos/httpd-24-centos7:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/bin/run-httpd",
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "configuration": {
+ "files": [
+ {
+ "type": "TEMPLATE",
+ "dest_file": "/etc/httpd/conf.d/httpd-proxy.conf",
+ "src_file": "httpd-proxy.conf"
+ }
+ ]
+ }
+ }
+ ],
+ "quicklinks": {
+ "Apache HTTP Server": "http://httpd-proxy-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080"
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f7a50d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
index 0bfb220..a85191c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
@@ -52,6 +52,11 @@ public interface ServiceApiConstants {
String CONTAINER_ID = $("CONTAINER_ID");
+ // Templates for component instance host/IP
+ String COMPONENT_HOST = $("%s_HOST");
+
+ String COMPONENT_IP = $("%s_IP");
+
// Constants for default cluster ZK
String CLUSTER_ZK_QUORUM = $("CLUSTER_ZK_QUORUM");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f7a50d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index cb7131e..cbaf472 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -47,8 +47,10 @@ import org.slf4j.LoggerFactory;
import java.text.MessageFormat;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -58,6 +60,7 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.*;
+import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.*;
import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.START;
import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.STOP;
@@ -386,6 +389,34 @@ public class Component implements EventHandler<ComponentEvent> {
return true;
}
+ public Map<String, String> getDependencyHostIpTokens() {
+ Map<String, String> tokens = new HashMap<>();
+ List<String> dependencies = componentSpec.getDependencies();
+ if (SliderUtils.isEmpty(dependencies)) {
+ return tokens;
+ }
+ for (String dependency : dependencies) {
+ Collection<ComponentInstance> instances = scheduler.getAllComponents()
+ .get(dependency).getAllComponentInstances().values();
+ for (ComponentInstance instance : instances) {
+ if (instance.getContainerStatus() == null) {
+ continue;
+ }
+ if (SliderUtils.isEmpty(instance.getContainerStatus().getIPs()) ||
+ SliderUtils.isUnset(instance.getContainerStatus().getHost())) {
+ continue;
+ }
+ String ip = instance.getContainerStatus().getIPs().get(0);
+ String host = instance.getContainerStatus().getHost();
+ tokens.put(String.format(COMPONENT_IP,
+ instance.getCompInstanceName().toUpperCase()), ip);
+ tokens.put(String.format(COMPONENT_HOST,
+ instance.getCompInstanceName().toUpperCase()), host);
+ }
+ }
+ return tokens;
+ }
+
private void incRunningContainers() {
componentMetrics.containersRunning.incr();
scheduler.getServiceMetrics().containersRunning.incr();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f7a50d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index ec0c2ca..93abd73 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -397,6 +397,7 @@ public class ProviderUtils implements YarnServiceConstants {
tokens.put(CONTAINER_ID, instance.getContainer().getId().toString());
tokens.put(COMPONENT_ID,
String.valueOf(instance.getCompInstanceId().getId()));
+ tokens.putAll(instance.getComponent().getDependencyHostIpTokens());
return tokens;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f7a50d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index 9ffc9db..51139be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -1550,6 +1550,10 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
private final RegistryCommand removeRecordCommand = new RegistryCommand() {
@Override
public void exec(Zone zone, Record record) throws IOException {
+ if (zone == null) {
+ LOG.error("Unable to remove record because zone is null: {}", record);
+ return;
+ }
zone.removeRecord(record);
LOG.info("Removed {}", record);
if (isDNSSECEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f7a50d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
new file mode 100644
index 0000000..db6d553
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
@@ -0,0 +1,159 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# YARN Service Examples
+
+This document describes some example service definitions (`Yarnfile`).
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
+## Apache web server - httpd (with registry DNS)
+Below is the `Yarnfile` for a service called `httpd-service` with two `httpd` instances.
+There is also an httpd proxy instance (httpd-proxy-0) that proxies between the other two httpd instances (httpd-0 and httpd-1).
+
+Note this example requires registry DNS.
+```
+{
+ "name": "httpd-service",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "httpd",
+ "number_of_containers": 2,
+ "artifact": {
+ "id": "centos/httpd-24-centos7:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/bin/run-httpd",
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "configuration": {
+ "files": [
+ {
+ "type": "ENV",
+ "dest_file": "/var/www/html/index.html",
+ "props": {
+ "content": "<html><header><title>Title</title></header><body>Hello from ${COMPONENT_INSTANCE_NAME}!</body></html>"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "httpd-proxy",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "centos/httpd-24-centos7:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/bin/run-httpd",
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "configuration": {
+ "files": [
+ {
+ "type": "TEMPLATE",
+ "dest_file": "/etc/httpd/conf.d/httpd-proxy.conf",
+ "src_file": "httpd-proxy.conf"
+ }
+ ]
+ }
+ }
+ ],
+ "quicklinks": {
+ "Apache HTTP Server": "http://httpd-proxy-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080"
+ }
+}
+```
+This `Yarnfile` is already included in the Hadoop distribution, along with the required configuration template `httpd-proxy.conf`.
+First upload the configuration template file to HDFS:
+```
+hdfs dfs -copyFromLocal ${HADOOP_YARN_HOME}/share/hadoop/yarn/yarn-service-examples/httpd/httpd-proxy.conf .
+```
+
+The proxy configuration template looks like the following and will configure the httpd-proxy-0 container to balance between the httpd-0 and httpd-1 containers evenly:
+```
+<Proxy balancer://test>
+ BalancerMember http://httpd-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080
+ BalancerMember http://httpd-1.${SERVICE_NAME}.${USER}.${DOMAIN}:8080
+ ProxySet lbmethod=bytraffic
+</Proxy>
+
+ProxyPass "/" "balancer://test/"
+ProxyPassReverse "/" "balancer://test/"
+```
+
+Then run the service with the command:
+```
+yarn service create [service-name] --example httpd
+```
+where `service-name` is optional. If omitted, it uses the name defined in the `Yarnfile`.
+
+Once the service is running, navigate to `http://httpd-proxy-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080` to see the root page.
+The pages should alternately show "Hello from httpd-0!" or "Hello from httpd-1!"
+
+The individual httpd URLs can also be visited, `http://httpd-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080` and `http://httpd-1.${SERVICE_NAME}.${USER}.${DOMAIN}:8080`.
+
+If unsure of your hostnames, visit the apiserver REST endpoint `http://<apiserver host>:9191/ws/v1/services/httpd-service`.
+
+## Apache web server - httpd (without registry DNS)
+
+A similar IP-based example is provided for environments that do not have registry DNS set up.
+The service name for this example is `httpd-service-no-dns`.
+There are a couple of additions to the `Yarnfile` for the `httpd-service` described above.
+A readiness check is added for the `httpd` component:
+```
+ "readiness_check": {
+ "type": "HTTP",
+ "props": {
+ "url": "http://${THIS_HOST}:8080"
+ }
+ },
+```
+and `httpd` is added as a dependency for the `httpd-proxy` component:
+```
+ "dependencies": [ "httpd" ],
+```
+
+This means that the httpd-proxy-0 instance will not be started until after an HTTP probe has succeeded for the httpd-0 and httpd-1 containers.
+This is necessary so that the IPs of the containers can be used in the configuration of httpd-proxy-0.
+The proxy configuration is similar to that of the previous example, with the BalancerMember lines changed as follows:
+```
+ BalancerMember http://${HTTPD-0_IP}:8080
+ BalancerMember http://${HTTPD-1_IP}:8080
+```
+
+Note that IP and HOST variables such as `${HTTPD-0_IP}` and `${HTTPD-0_HOST}` should only be used by a component that has a dependency on the named component (`httpd` in this case) AND should only be used when the named component specifies a readiness check.
+Here, `httpd-proxy` has a dependency on `httpd` and `httpd` has an HTTP readiness check.
+Without the dependency and readiness check, the httpd-proxy-0 container would be started in parallel with the httpd-0 and http-1 containers, and the IPs and hosts would not be assigned yet for httpd-0 and httpd-1.
+
+Other variables can be used by any component.
+
+Before creating the service, upload the proxy configuration to HDFS:
+```
+hdfs dfs -copyFromLocal ${HADOOP_YARN_HOME}/share/hadoop/yarn/yarn-service-examples/httpd-no-dns/httpd-proxy-no-dns.conf .
+```
+
+Then run the service with the command:
+```
+yarn service create [service-name] --example httpd-no-dns
+```
+where `service-name` is optional. If omitted, it uses the name defined in the `Yarnfile`.
+
+Look up your IPs at the apiserver REST endpoint `http://<apiserver host>:9191/ws/v1/services/httpd-service`.
+Then visit port 8080 for each IP to view the pages.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f7a50d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
index 3dd4d8c..407fbc0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
@@ -52,7 +52,8 @@ The benefits of combining these workloads are two-fold:
* [Concepts](Concepts.md): Describes the internals of the framework and some features in YARN core to support running services on YARN.
* [Service REST API](YarnServiceAPI.md): The API doc for deploying/managing services on YARN.
-* [Service Discovery](ServiceDiscovery.md): Deep dives into the YARN DNS internals
+* [Service Discovery](ServiceDiscovery.md): Deep dives into the YARN DNS internals.
+* [Examples](Examples.md): List some example service definitions (`Yarnfile`).
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[85/86] [abbrv] hadoop git commit: YARN-7210. Some NPE fixes in
Registry DNS. Contributed by Jian He
Posted by ji...@apache.org.
YARN-7210. Some NPE fixes in Registry DNS. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/360b2d26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/360b2d26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/360b2d26
Branch: refs/heads/yarn-native-services
Commit: 360b2d263cd1eee10b9df991841031188fbce374
Parents: c329bba
Author: Billie Rinaldi <bi...@apache.org>
Authored: Thu Sep 21 10:18:42 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 25 16:37:26 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/service/ServiceScheduler.java | 6 +-----
.../hadoop/yarn/service/client/ServiceClient.java | 3 +++
.../params/AbstractClusterBuildingActionArgs.java | 2 +-
.../yarn/service/client/params/Arguments.java | 1 +
.../component/instance/ComponentInstance.java | 10 +++++-----
.../service/provider/AbstractProviderService.java | 18 +++++++++++-------
.../client/binding/RegistryTypeUtils.java | 3 ++-
.../dns/ApplicationServiceRecordProcessor.java | 15 ++++++++++++++-
.../server/dns/BaseServiceRecordProcessor.java | 8 ++++----
.../registry/server/dns/TestRegistryDNS.java | 12 ++++++++----
.../runtime/DockerLinuxContainerRuntime.java | 4 ++--
.../src/site/markdown/YarnCommands.md | 1 +
12 files changed, 53 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/360b2d26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 7b809b9..ec5f3ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -344,11 +344,7 @@ public class ServiceScheduler extends CompositeService {
attemptId.getApplicationId().toString());
serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
PersistencePolicies.APPLICATION);
- serviceRecord.description = "Yarn Service Master";
-
- serviceRecord.addExternalEndpoint(RegistryTypeUtils
- .ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc",
- context.clientAMService.getBindAddress()));
+ serviceRecord.description = "YarnServiceMaster";
// set any provided attributes
setUserProvidedServiceRecordAttributes(service.getConfiguration(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/360b2d26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 6890aef..a3a9fd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -170,6 +170,9 @@ public class ServiceClient extends CompositeService
if (!StringUtils.isEmpty(args.getServiceName())) {
service.setName(args.getServiceName());
}
+ if (!StringUtils.isEmpty(args.queue)) {
+ service.setQueue(args.queue);
+ }
return service;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/360b2d26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
index 434c6d5..4ecbe9c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
@@ -37,7 +37,7 @@ public abstract class AbstractClusterBuildingActionArgs
}
@Parameter(names = {
- ARG_QUEUE }, description = "Queue to submit the service")
+ ARG_QUEUE, ARG_SHORT_QUEUE}, description = "Queue to submit the service")
public String queue;
@Parameter(names = {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/360b2d26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
index 9672621..67571e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
@@ -77,6 +77,7 @@ public interface Arguments {
String ARG_PATH = "--path";
String ARG_PRINCIPAL = "--principal";
String ARG_QUEUE = "--queue";
+ String ARG_SHORT_QUEUE = "-q";
String ARG_LIFETIME = "--lifetime";
String ARG_RESOURCE = "--resource";
String ARG_RESOURCE_MANAGER = "--rm";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/360b2d26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
index 7d6525b..3c1e48f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -59,6 +59,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import static org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes.*;
import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.KILLED_BY_APPMASTER;
import static org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE;
import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.*;
@@ -356,12 +357,11 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
YarnRegistryViewForProviders yarnRegistry, ContainerStatus status) {
ServiceRecord record = new ServiceRecord();
String containerId = status.getContainerId().toString();
- record.set(YarnRegistryAttributes.YARN_ID, containerId);
+ record.set(YARN_ID, containerId);
record.description = getCompInstanceName();
- record.set(YarnRegistryAttributes.YARN_PERSISTENCE,
- PersistencePolicies.CONTAINER);
- record.set("yarn:ip", status.getIPs());
- record.set("yarn:hostname", status.getHost());
+ record.set(YARN_PERSISTENCE, PersistencePolicies.CONTAINER);
+ record.set(YARN_IP, status.getIPs().get(0));
+ record.set(YARN_HOSTNAME, status.getHost());
try {
yarnRegistry
.putComponent(RegistryPathUtils.encodeYarnID(containerId), record);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/360b2d26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
index 8d607ab..6ffb84d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.service.provider;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.service.api.records.Service;
@@ -91,13 +92,16 @@ public abstract class AbstractProviderService implements ProviderService,
component, tokensForSubstitution, instance, context);
// substitute launch command
- String launchCommand = ProviderUtils
- .substituteStrWithTokens(component.getLaunchCommand(),
- tokensForSubstitution);
- CommandLineBuilder operation = new CommandLineBuilder();
- operation.add(launchCommand);
- operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
- launcher.addCommand(operation.build());
+ String launchCommand = component.getLaunchCommand();
+ // docker container may have empty commands
+ if (!StringUtils.isEmpty(launchCommand)) {
+ launchCommand = ProviderUtils
+ .substituteStrWithTokens(launchCommand, tokensForSubstitution);
+ CommandLineBuilder operation = new CommandLineBuilder();
+ operation.add(launchCommand);
+ operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
+ launcher.addCommand(operation.build());
+ }
// By default retry forever every 30 seconds
launcher.setRetryContext(YarnServiceConf
http://git-wip-us.apache.org/repos/asf/hadoop/blob/360b2d26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java
index ec59d59..05df325 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java
@@ -22,7 +22,6 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
-import static org.apache.hadoop.registry.client.types.AddressTypes.*;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ProtocolTypes;
import org.apache.hadoop.registry.client.types.ServiceRecord;
@@ -36,6 +35,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static org.apache.hadoop.registry.client.types.AddressTypes.*;
+
/**
* Static methods to work with registry types —primarily endpoints and the
* list representation of addresses.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/360b2d26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java
index e6a1b5b..0b5f724 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java
@@ -18,6 +18,8 @@ package org.apache.hadoop.registry.server.dns;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.xbill.DNS.Name;
import org.xbill.DNS.Type;
@@ -32,7 +34,8 @@ import java.util.List;
*/
public class ApplicationServiceRecordProcessor extends
BaseServiceRecordProcessor {
-
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ApplicationServiceRecordProcessor.class);
/**
* Create an application service record processor.
*
@@ -57,6 +60,10 @@ public class ApplicationServiceRecordProcessor extends
*/
@Override public void initTypeToInfoMapping(ServiceRecord serviceRecord)
throws Exception {
+ if (serviceRecord.external.isEmpty()) {
+ LOG.info(serviceRecord.description + ": No external endpoints defined.");
+ return;
+ }
for (int type : getRecordTypes()) {
switch (type) {
case Type.A:
@@ -309,6 +316,9 @@ public class ApplicationServiceRecordProcessor extends
throws Exception {
this.setNames(new Name[] {getServiceName()});
List<Endpoint> endpoints = serviceRecord.external;
+ if (endpoints.isEmpty()) {
+ return;
+ }
// TODO: do we need a "hostname" attribute for an application record or
// can we rely on the first endpoint record.
this.setTarget(InetAddress.getByName(
@@ -342,6 +352,9 @@ public class ApplicationServiceRecordProcessor extends
@Override protected void init(ServiceRecord serviceRecord)
throws Exception {
super.init(serviceRecord);
+ if (getTarget() == null) {
+ return;
+ }
try {
this.setTarget(getIpv6Address(getTarget()));
} catch (UnknownHostException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/360b2d26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
index 2fe3a6c..fd5c74f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
@@ -52,8 +52,8 @@ public abstract class BaseServiceRecordProcessor
private String domain;
private static final Pattern USER_NAME = Pattern.compile("/users/(\\w*)/?");
- private static final String SLIDER_API_PREFIX =
- "classpath:org.apache.slider.";
+ private static final String YARN_SERVICE_API_PREFIX =
+ "classpath:org.apache.hadoop.yarn.service.";
private static final String HTTP_API_TYPE = "http://";
/**
@@ -425,8 +425,8 @@ public abstract class BaseServiceRecordProcessor
*/
protected String getDNSApiFragment(String api) {
String dnsApi = null;
- if (api.startsWith(SLIDER_API_PREFIX)) {
- dnsApi = api.substring(SLIDER_API_PREFIX.length());
+ if (api.startsWith(YARN_SERVICE_API_PREFIX)) {
+ dnsApi = api.substring(YARN_SERVICE_API_PREFIX.length());
} else if (api.startsWith(HTTP_API_TYPE)) {
dnsApi = "http";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/360b2d26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
index cc839cc..ac8d939 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
@@ -69,7 +69,8 @@ public class TestRegistryDNS extends Assert {
+ " \"type\" : \"JSONServiceRecord\",\n"
+ " \"description\" : \"Slider Application Master\",\n"
+ " \"external\" : [ {\n"
- + " \"api\" : \"classpath:org.apache.slider.appmaster.ipc\",\n"
+ + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.appmaster.ipc"
+ + "\",\n"
+ " \"addressType\" : \"host/port\",\n"
+ " \"protocolType\" : \"hadoop/IPC\",\n"
+ " \"addresses\" : [ {\n"
@@ -84,7 +85,8 @@ public class TestRegistryDNS extends Assert {
+ " \"uri\" : \"http://192.168.1.5:1027\"\n"
+ " } ]\n"
+ " }, {\n"
- + " \"api\" : \"classpath:org.apache.slider.management\",\n"
+ + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.management\""
+ + ",\n"
+ " \"addressType\" : \"uri\",\n"
+ " \"protocolType\" : \"REST\",\n"
+ " \"addresses\" : [ {\n"
@@ -92,14 +94,16 @@ public class TestRegistryDNS extends Assert {
+ " } ]\n"
+ " } ],\n"
+ " \"internal\" : [ {\n"
- + " \"api\" : \"classpath:org.apache.slider.agents.secure\",\n"
+ + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.agents.secure"
+ + "\",\n"
+ " \"addressType\" : \"uri\",\n"
+ " \"protocolType\" : \"REST\",\n"
+ " \"addresses\" : [ {\n"
+ " \"uri\" : \"https://192.168.1.5:47700/ws/v1/slider/agents\"\n"
+ " } ]\n"
+ " }, {\n"
- + " \"api\" : \"classpath:org.apache.slider.agents.oneway\",\n"
+ + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.agents.oneway"
+ + "\",\n"
+ " \"addressType\" : \"uri\",\n"
+ " \"protocolType\" : \"REST\",\n"
+ " \"addresses\" : [ {\n"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/360b2d26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 7b80e0b..90dde867 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -702,13 +702,13 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
.executePrivilegedOperation(null, privOp, null,
null, true, false);
LOG.info("Docker inspect output for " + containerId + ": " + output);
+ // strip off quotes if any
+ output = output.replaceAll("['\"]", "");
int index = output.lastIndexOf(',');
if (index == -1) {
LOG.error("Incorrect format for ip and host");
return null;
}
- // strip off quotes if any
- output = output.replaceAll("['\"]", "");
String ips = output.substring(0, index).trim();
String host = output.substring(index+1).trim();
String[] ipAndHost = new String[2];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/360b2d26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 55d007b..e464d54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -92,6 +92,7 @@ Usage `yarn service [sub-command] [service-name] [options]`
Options:
--file,-f The local path to the service definition file.
+ --queue,-q The queue to which the service is submitted.
--example,-e The name of the example service such as:
Sleeper A simple service that launches a few non-docker sleep containers on YARN.
```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
deleted file mode 100644
index da06c13..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
+++ /dev/null
@@ -1,168 +0,0 @@
-{
- "roleCommand": "{{COMMAND}}",
- "clusterName": "{{CLUSTER_NAME}}",
- "hostname": "{{HOST_NAME}}",
- "hostLevelParams": {
- "java_home": "/usr/jdk64/jdk1.7.0_45"
- },
- "commandType": "EXECUTION_COMMAND",
- "roleParams": {},
- "serviceName": "{{SERVICE_NAME}}",
- "role": "{{ROLE_NAME}}",
- "commandParams": {},
- "taskId": "{{TASK_ID}}",
- "public_hostname": "{{HOST_NAME}}",
- "configurations": {
- "hbase-log4j": {
- "log4j.threshold": "ALL",
- "log4j.rootLogger": "${hbase.root.logger}",
- "log4j.logger.org.apache.zookeeper": "INFO",
- "log4j.logger.org.apache.hadoop.hbase": "DEBUG",
- "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher": "INFO",
- "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil": "INFO",
- "log4j.category.SecurityLogger": "${hbase.security.logger}",
- "log4j.appender.console": "org.apache.log4j.ConsoleAppender",
- "log4j.appender.console.target": "System.err",
- "log4j.appender.console.layout": "org.apache.log4j.PatternLayout",
- "log4j.appender.console.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
- "log4j.appender.RFAS": "org.apache.log4j.RollingFileAppender",
- "log4j.appender.RFAS.layout": "org.apache.log4j.PatternLayout",
- "log4j.appender.RFAS.layout.ConversionPattern": "%d{ISO8601} %p %c: %m%n",
- "log4j.appender.RFAS.MaxFileSize": "${hbase.security.log.maxfilesize}",
- "log4j.appender.RFAS.MaxBackupIndex": "${hbase.security.log.maxbackupindex}",
- "log4j.appender.RFAS.File": "${hbase.log.dir}/${hbase.security.log.file}",
- "log4j.appender.RFA": "org.apache.log4j.RollingFileAppender",
- "log4j.appender.RFA.layout": "org.apache.log4j.PatternLayout",
- "log4j.appender.RFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
- "log4j.appender.RFA.MaxFileSize": "${hbase.log.maxfilesize}",
- "log4j.appender.RFA.MaxBackupIndex": "${hbase.log.maxbackupindex}",
- "log4j.appender.RFA.File": "${hbase.log.dir}/${hbase.log.file}",
- "log4j.appender.NullAppender": "org.apache.log4j.varia.NullAppender",
- "log4j.appender.DRFA": "org.apache.log4j.DailyRollingFileAppender",
- "log4j.appender.DRFA.layout": "org.apache.log4j.PatternLayout",
- "log4j.appender.DRFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
- "log4j.appender.DRFA.File": "${hbase.log.dir}/${hbase.log.file}",
- "log4j.appender.DRFA.DatePattern": ".yyyy-MM-dd",
- "log4j.additivity.SecurityLogger": "false",
- "hbase.security.logger": "INFO,console",
- "hbase.security.log.maxfilesize": "256MB",
- "hbase.security.log.maxbackupindex": "20",
- "hbase.security.log.file": "SecurityAuth.audit",
- "hbase.root.logger": "INFO,console",
- "hbase.log.maxfilesize": "256MB",
- "hbase.log.maxbackupindex": "20",
- "hbase.log.file": "hbase.log",
- "hbase.log.dir": "."
- },
- "global": {
- "hbase_root": "{{HBASE_HOME}}",
- "hbase_pid_dir": "{{PID_DIR}}",
- "proxyuser_group": "users",
- "syncLimit": "5",
- "hbase_regionserver_heapsize": "{{REGION_SERVER_HEAP_SIZE}}",
- "rca_enabled": "false",
- "tickTime": "2000",
- "hbase_master_heapsize": "{{MASTER_HEAP_SIZE}}",
- "initLimit": "10",
- "user_group": "{{GROUP_NAME}}",
- "hbase_user": "{{USER_NAME}}",
- "hbase_log_dir": "{{LOG_DIR}}"
- },
- "hdfs-site": {
- "dfs.namenode.checkpoint.period": "21600",
- "dfs.namenode.avoid.write.stale.datanode": "true",
- "dfs.namenode.checkpoint.txns": "1000000",
- "dfs.block.access.token.enable": "true",
- "dfs.support.append": "true",
- "dfs.datanode.address": "0.0.0.0:${ambari.dfs.datanode.port}",
- "dfs.cluster.administrators": " hdfs",
- "dfs.replication": "3",
- "ambari.dfs.datanode.http.port": "50075",
- "dfs.datanode.balance.bandwidthPerSec": "6250000",
- "dfs.namenode.safemode.threshold-pct": "1.0f",
- "dfs.namenode.checkpoint.edits.dir": "${dfs.namenode.checkpoint.dir}",
- "dfs.permissions.enabled": "true",
- "dfs.client.read.shortcircuit": "true",
- "dfs.namenode.https-address": "{{NAMENODE_HTTPS_ADDRESS}}",
- "dfs.journalnode.edits.dir": "/grid/0/hdfs/journal",
- "dfs.blocksize": "134217728",
- "dfs.datanode.max.transfer.threads": "1024",
- "dfs.datanode.du.reserved": "1073741824",
- "dfs.webhdfs.enabled": "true",
- "dfs.namenode.handler.count": "100",
- "dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary",
- "fs.permissions.umask-mode": "022",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
- "dfs.datanode.ipc.address": "0.0.0.0:8010",
- "dfs.datanode.data.dir": "/hadoop/hdfs/data",
- "dfs.namenode.http-address": "{{NAMENODE_HTTP_ADDRESS}}",
- "dfs.blockreport.initialDelay": "120",
- "dfs.datanode.failed.volumes.tolerated": "0",
- "dfs.namenode.accesstime.precision": "0",
- "ambari.dfs.datanode.port": "50010",
- "dfs.namenode.avoid.read.stale.datanode": "true",
- "dfs.namenode.secondary.http-address": "c6402.ambari.apache.org:50090",
- "dfs.namenode.stale.datanode.interval": "30000",
- "dfs.heartbeat.interval": "3",
- "dfs.client.read.shortcircuit.streams.cache.size": "4096",
- "dfs.permissions.superusergroup": "hdfs",
- "dfs.https.port": "50470",
- "dfs.journalnode.http-address": "0.0.0.0:8480",
- "dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket",
- "dfs.namenode.write.stale.datanode.ratio": "1.0f",
- "dfs.hosts.exclude": "/etc/hadoop/conf/dfs.exclude",
- "dfs.datanode.data.dir.perm": "750",
- "dfs.namenode.name.dir.restore": "true",
- "dfs.replication.max": "50",
- "dfs.namenode.name.dir": "/hadoop/hdfs/namenode"
- },
- "hbase-site": {
- "hbase.hstore.flush.retries.number": "120",
- "hbase.client.keyvalue.maxsize": "10485760",
- "hbase.hstore.compactionThreshold": "3",
- "hbase.rootdir": "{{HBASE_ROOT_DIR}}",
- "hbase.stagingdir": "{{HBASE_STAGING_DIR}}",
- "hbase.regionserver.handler.count": "60",
- "hbase.regionserver.global.memstore.lowerLimit": "0.38",
- "hbase.hregion.memstore.block.multiplier": "2",
- "hbase.hregion.memstore.flush.size": "134217728",
- "hbase.superuser": "{{HBASE_SUPERUSER}}",
- "hbase.zookeeper.property.clientPort": "{{ZK_CLIENT_PORT}}",
- "hbase.regionserver.global.memstore.upperLimit": "0.4",
- "zookeeper.session.timeout": "30000",
- "hbase.tmp.dir": "/hadoop/hbase",
- "hbase.hregion.max.filesize": "10737418240",
- "hfile.block.cache.size": "0.40",
- "hbase.security.authentication": "simple",
- "hbase.defaults.for.version.skip": "true",
- "hbase.zookeeper.quorum": "{{ZK_HOSTS}}",
- "zookeeper.znode.parent": "{{ZK_NODE_PARENT}}",
- "hbase.hstore.blockingStoreFiles": "10",
- "hbase.hregion.majorcompaction": "86400000",
- "hbase.security.authorization": "false",
- "hbase.cluster.distributed": "true",
- "hbase.hregion.memstore.mslab.enabled": "true",
- "hbase.client.scanner.caching": "100",
- "hbase.zookeeper.useMulti": "true",
- "hbase.regionserver.info.port": "{{REGION_SERVER_INFO_PORT}}",
- "hbase.master.info.port": "{{MASTER_INFO_PORT}}"
- },
- "core-site": {
- "io.serializations": "org.apache.hadoop.io.serializer.WritableSerialization",
- "gluster.daemon.user": "null",
- "fs.trash.interval": "360",
- "hadoop.security.authentication": "simple",
- "io.compression.codecs": "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec",
- "mapreduce.jobtracker.webinterface.trusted": "false",
- "fs.AbstractFileSystem.glusterfs.impl": "null",
- "fs.defaultFS": "{{DEFAULT_FS}}",
- "ipc.client.connect.max.retries": "50",
- "ipc.client.idlethreshold": "8000",
- "io.file.buffer.size": "131072",
- "hadoop.security.authorization": "false",
- "hadoop.security.auth_to_local": "\n RULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/\n RULE:[2:$1@$0](jhs@.*)s/.*/mapred/\n RULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/\n RULE:[2:$1@$0](hm@.*)s/.*/hbase/\n RULE:[2:$1@$0](rs@.*)s/.*/hbase/\n DEFAULT",
- "ipc.client.connection.maxidletime": "30000"
- }
- },
- "commandId": "{{COMMAND_ID}}"
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/role-node.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/role-node.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/role-node.xml
deleted file mode 100644
index aff1e05..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/role-node.xml
+++ /dev/null
@@ -1,65 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
- <!--
- Role options for an agent-managed node
- -->
-<configuration>
- <property>
- <name>role.name</name>
- <value>node</value>
- </property>
-
- <property>
- <name>role.instances</name>
- <value>1</value>
- </property>
-
- <property>
- <name>role.priority</name>
- <value>1</value>
- </property>
-
- <property>
- <name>role.placement.policy</name>
- <value>2</value>
- </property>
-
- <property>
- <name>yarn.memory</name>
- <value>256</value>
- </property>
-
- <property>
- <name>yarn.vcores</name>
- <value>1</value>
- </property>
-
- <property>
- <name>jvm.heapsize</name>
- <value>256M</value>
- </property>
-
- <property>
- <name>env.MALLOC_ARENA_MAX</name>
- <value>4</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/dynamic/application.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/dynamic/application.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/dynamic/application.properties
deleted file mode 100644
index d9b42de..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/dynamic/application.properties
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# gets updated at build time
-application.name=${pom.name}
-application.version=${pom.version}
-application.build=${buildNumber}
-application.build.java.version=${java.version}
-application.build.user=${user.name}
-application.build.info=${pom.name}-${pom.version} Built against commit# ${buildNumber} on Java ${java.version} by ${user.name}
-hadoop.build.info=${hadoop.version}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json
deleted file mode 100644
index 81239a2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
-
- "metadata": {
-
-
- },
-
- "global": {
- "env.MALLOC_ARENA_MAX": "4"
- },
-
- "components": {
- "slider-appmaster" : {
- "jvm.heapsize": "256M"
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/internal.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/internal.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/internal.json
deleted file mode 100644
index 2367d8f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/internal.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
-
- "metadata": {
- },
-
- "global": {
- "internal.container.failure.shortlife": "60000",
- "internal.container.failure.threshold": "5",
- "slider.cluster.directory.permissions": "0770",
- "slider.data.directory.permissions": "0770"
- },
-
- "components": {
-
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json
deleted file mode 100644
index 478ab7e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
-
- "metadata": {
-
- },
-
- "global": {
- },
-
- "components": {
- "slider-appmaster": {
- "yarn.component.instances": "1",
- "yarn.vcores": "1",
- "yarn.memory": "1024"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/webapps/slideram/.keep
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/webapps/slideram/.keep b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/webapps/slideram/.keep
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/scripts/slider_keytabs.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/scripts/slider_keytabs.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/scripts/slider_keytabs.sh
deleted file mode 100644
index f0a8fc2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/scripts/slider_keytabs.sh
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script exists to create the keytab set for a node on the cluster
-# including hbase and ZK alongside then YARN cores.
-
-# usage
-# keytabs <realm> <hostname>
-# validate the args
-
-num_vars=$#
-if [[ $num_vars < 2 ]]
-then
- echo "Usage: $0 <realm> <hostname>"
- exit -2
-fi
-
-realm="$1"
-hostname="$2"
-dest="."
-
-kadmin=kadmin.local
-
-${kadmin} <<EOF
-addprinc -randkey hdfs/${hostname}@${realm}
-addprinc -randkey yarn/${hostname}@${realm}
-addprinc -randkey HTTP/${hostname}@${realm}
-addprinc -randkey hbase/${hostname}@${realm}
-addprinc -randkey zookeeper/${hostname}@${realm}
-
-ktadd -norandkey -k ${dest}/hdfs.keytab \
- hdfs/${hostname}@${realm} \
- HTTP/${hostname}@${realm}
-
-ktadd -norandkey -k ${dest}/yarn.keytab \
- yarn/${hostname}@${realm} \
- HTTP/${hostname}@${realm}
-
-ktadd -norandkey -k ${dest}/hbase.keytab \
- hbase/${hostname}@${realm}
-
-ktadd -norandkey -k ${dest}/zookeeper.keytab \
- zookeeper/${hostname}@${realm}
-EOF
-
-exitcode=$?
-if [[ $exitcode != 0 ]]
-then
- echo "keytab generation from ${kadmin} failed with exit code $exitcode"
- exit $exitcode
-else
- echo "keytab files for ${hostname}@${realm} created"
-fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/scripts/yarnservice.py
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/scripts/yarnservice.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/scripts/yarnservice.py
deleted file mode 100644
index 1208c28..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/scripts/yarnservice.py
+++ /dev/null
@@ -1,383 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-"""Launches a yarn service
-
-WORK IN PROGRESS, IGNORE
-
-This is as work in progress project to build as new launcher script for
-any Hadoop service
-A key feature here is that the configs are defined in JSON files -
-files that are read in the order passed down, and merged into each other.
-
-The final merged file is used to define the java command to execute
--and hadoop XML files.
-
-
-It uses a JSON config file
- --jfile configuration file (JSON format)
- -class classname
- -Dname=value -arbitrary value to pass down to the JVM
- --java: any JVM arg
- -javaX: javaX value
-
-
- after an -- , all following commands are passed straight down to the invoked process.
- # -xJ name=value JVM options. No: this is just another param
- -xF file file to load next. Files are loaded in order.
- -xD name=value again, values are loaded in order
- -xU undefine
- -xX main class, 'eXecute'
-
- -- end of arguments
-
-
-"""
-
-import sys
-# see : http://simplejson.readthedocs.org/en/latest/
-# and install w/ easy_install simplejson
-import simplejson
-
-KEY_JFILE = "-xF"
-KEY_DEF = "-xD"
-KEY_UNDEF = "-xU"
-KEY_EXEC = "-xX"
-KEY_ARGS = "--"
-
-COMMANDS = [KEY_JFILE, KEY_DEF, KEY_EXEC]
-
-#
-
-def debug(string) :
- print string
-
-
-def pop_required_arg(arglist, previousArg) :
- """
- Pop the first element off the list and return it.
- If the list is empty, raise an exception about a missing argument after the $previousArgument
- """
- if not len(arglist) :
- raise Exception, "Missing required parameter after %s" % previousArg
- head = arglist[0]
- del arglist[0]
- return head
-
-
-def parse_one_jfile(filename) :
- """
- read in the given config file
- """
- parsed = simplejson.load(open(filename, "r"))
- return parsed
-
-# hand down sys.argv:
-def extract_jfiles(args) :
- """ takes a list of arg strings and separates them into jfile references
- and other arguments.
- """
- l = len(args)
- stripped = []
- jfiles = []
- index = 0
- while index < l :
- elt = args[index]
- index += 1
- if KEY_JFILE == elt :
- # a match
- if index == l :
- #overshoot
- raise Exception("Missing filename after " + KEY_JFILE)
- filename = args[index]
- debug("jfile " + filename)
- jfiles.append(filename)
- index += 1
- else :
- stripped.append(elt)
- return jfiles, stripped
-
-
-def extract_args(args) :
- """
- Take a list of args, parse them or fail, generating a dictionary of actions
- Return: dictionary and all leftover arguments
- """
- jfiles = []
- execs = []
- defs = []
- remainder = []
- while len(args) :
- # the next call cannot fail, because of the len(args)
- arg = pop_required_arg(args, "")
- if KEY_JFILE == arg :
- jfiles.append(pop_required_arg(args, KEY_JFILE))
- elif KEY_DEF == arg :
- defs.append((KEY_DEF, pop_required_arg(args, KEY_DEF)))
- elif KEY_UNDEF == arg :
- defs.append((KEY_UNDEF, pop_required_arg(args, KEY_UNDEF)))
- elif KEY_EXEC == arg :
- execs.append(pop_required_arg(args, KEY_EXEC))
- elif KEY_ARGS == arg :
- remainder += args
- args = []
- else :
- remainder.append(arg)
- #build the action list
- actions = {
- KEY_JFILE : jfiles,
- KEY_EXEC : execs,
- KEY_DEF : defs,
- KEY_ARGS : remainder
- }
- #end of the run, there's a dictionary and a list of unparsed values
- return actions
-
-
-def get(conf, key, defVal) :
- if conf.has_key(key) :
- return conf[key]
- else :
- return defVal
-
-
-def merge_json(conf, json) :
- """ merge in a json dict with the existing one
- in: configuration dict, json dict
- out: configuration'
- """
- for (key, val) in json.items() :
- if key in conf :
- #there's a match, do a more detailed merge
- oldval = conf[key]
- if type(oldval) == dict and type(val) == dict :
- # two dictionary instances -merge
- merge_json(oldval, val)
- else :
- conf[key] = val
- else :
- conf[key] = val
- return conf
-
-
-def merge_jfile(conf, filename) :
- json = parse_one_jfile(filename)
- return merge_json(conf, json)
-
-
-def merge_jfile_list(conf, jfiles) :
- """ merge a list of jfiles on top of a conf dict
- """
- for jfile in jfiles :
- conf = merge_jfile(conf, jfile)
- return conf
-
-
-def split_to_keyval_tuple(param) :
- """
- Split a key=value string into the (key,value) tuple
- * an exception is raised on any string "=value"
- * if there is no string: exception.
- * a key only definition maps to (key, None)
- * a "key=" definition maps to (key, "")
- """
- if not len(param) :
- raise Exception, "Empty string cannot be a key=value definition"
- equalsPos = param.find("=")
- if equalsPos < 0 :
- return param, None
- elif not equalsPos :
- raise Exception, "no key in argument %s" % param
- else :
- key = param[:(equalsPos - 1)]
- value = param[(equalsPos + 1) :]
- return key, value
-
-
-def recursive_define(conf, path, value) :
- if not len(path) :
- #fallen off the end of the world
- return
- entry = path[0]
- if len(path) == 1 :
- #end of list, apply it.
- conf[entry] = value
- else :
- #there's 1+ elements below, yet there's a subdir here.
- if conf.has_key(entry) and type(conf[entry]) == dict :
- #it's a subdir, simple: recurse.
- recursive_define(conf[entry], path[1 :], value)
- else :
- #either there is an entry that isn't a conf, or its not there. Same outcome.
- subconf = {}
- conf[entry] = subconf
- recursive_define(subconf, path[1 :], value)
-
-def recursive_undef(conf, path) :
- if not len(path) :
- #fallen off the end of the world
- return
- entry = path[0]
- if len(path) == 1 :
- #end of list, apply it.
- del conf[entry]
- else :
- #there's 1+ elements below, yet there's a subdir here.
- if conf.has_key(entry) and type(conf[entry]) == dict :
- #it's a subdir, simple: recurse.
- recursive_undef(conf[entry], path[1 :])
- else :
- #either there is an entry that isn't a conf, or its not there. Same outcome.
- pass
-
-def apply_action(conf, action, key, value) :
- """
- Apply either a def or undef action; splitting the key into a path and running through it.
- """
- keypath = key.split("/")
- #now have a split key,
- if KEY_DEF == action :
- recursive_define(conf, keypath, value)
- elif KEY_UNDEF == action :
- recursive_undef(conf, keypath)
-
-
-def apply_local_definitions(conf, definitions) :
- """
- Run through the definition actions and apply them one by one
- """
- for defn in definitions :
- # split into key=value; no value -> empty string
- (action, param) = defn
- if KEY_DEF == action :
- (key, val) = split_to_keyval_tuple(param)
- apply_action(conf, KEY_DEF, key, val)
-
- return conf
-
-
-#def parse_args(conf, args) :
-# """
-# split an arg string, parse the jfiles & merge over the conf
-# (configuration, args[]) -> (conf', stripped, jfiles[])
-# """
-# (jfiles, stripped) = extract_jfiles(args)
-#
-# actions = extract_args(args)
-# jfiles = actions[KEY_JFILE]
-# conf = merge_jfile_list(conf, jfiles)
-# return conf, actions
-
-
-def print_conf(conf) :
- """ dump the configuration to the console
- """
- print "{"
- for (key, val) in conf.items() :
- if type(val) == dict :
- print key
- print_conf(val)
- else :
- print "" + key + " => " + str(val)
- print "}"
-
-
-def list_to_str(l, spacer) :
- result = ""
- for elt in l :
- if len(result) > 0 :
- result += spacer
- result += elt
- return result
-
-
-def list_to_hxml_str(l) :
- return list_to_str(l, ",")
-
-
-def export_kv_xml(output, key, value) :
- line = "<property><name>" + key + "</name><value>" + str(value) + "</value>\n"
- print line
- output.write(line)
-
-
-def export_to_hadoop_xml(output, conf) :
- """ export the conf to hadoop XML
- dictionaries are skipped.
- """
- output.write("<configuration>\n")
- for (key, value) in conf.items() :
- if type(value) is list :
- # list print
- export_kv_xml(output, key, list_to_hxml_str(value))
- else :
- if type(value) is dict :
- print "skipping dict " + key
- else :
- export_kv_xml(output, key, value)
- output.write("</configuration>\n")
-
-
-def start(conf, stripped_args) :
- """
- start the process by grabbing exec/args for the arguments
- """
- ex = conf["exec"]
- args = []
- jsonargs = get(ex, "args", [])
- args.extend(jsonargs)
- args.extend(stripped_args)
- classname = get(ex, "classname", "")
- if not len(classname) :
- raise Exception, "No classname supplied"
- classname = get(ex, "classname", "")
- commandline = ["java"]
- classpath = []
- jvmargs = []
- commandline.extend(jvmargs)
- commandline.append("-classpath")
- commandline.append(list_to_str(classpath, ":"))
- commandline.append("org.apache.hadoop.yarn.service.launcher.ServiceLauncher")
- commandline.append(classname)
- commandline.extend(args)
- print "ready to exec : %s" % commandline
-
-
-def main() :
-# (conf, stripped, jfiles) = parse_args({}, sys.argv[1 :])
- actions = extract_args(sys.argv[1 :])
- jfiles = actions[KEY_JFILE]
- conf = merge_jfile_list({}, jfiles)
- apply_local_definitions(conf, actions[KEY_DEF])
- exec_args = actions[KEY_ARGS]
-
- print_conf(conf)
- # if len(stripped) > 0 :
- #got an output file
- # filename = stripped[0]
- # print "Writing XML configuration to " + filename
- # output = open(filename, "w")
- # export_to_hadoop_xml(output, conf["site"])
- start(conf, exec_args)
-
-
-if __name__ == "__main__" :
- main()
-
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/site/site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/site/site.xml
deleted file mode 100644
index 3b5df7a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/site/site.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-<?xml version="1.0"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<project name="Slider">
-
- <version position="right"/>
- <body>
- <menu ref="reports"/>
- </body>
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
index 9746d33..4fa81ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
@@ -38,14 +38,13 @@ import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.proto.*;
import org.apache.hadoop.yarn.proto.ClientAMProtocol;
+import org.apache.hadoop.yarn.service.api.records.Application;
import org.apache.hadoop.yarn.service.component.Component;
import org.apache.hadoop.yarn.service.component.ComponentState;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import java.io.IOException;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index ea75a90..73172bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -18,12 +18,18 @@
package org.apache.hadoop.yarn.service;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.Resource;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
public class ServiceTestUtils {
+ public static final JsonSerDeser<Application> JSON_SER_DESER =
+ new JsonSerDeser<>(Application.class,
+ PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
+
// Example service definition
// 2 components, each of which has 2 containers.
protected Application createExampleApplication() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
index d99e30e..1a22875 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
@@ -21,15 +21,14 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.Resource;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.persist.JsonSerDeser;
-import org.apache.slider.util.RestApiConstants;
-import org.apache.slider.util.RestApiErrorMessages;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -41,15 +40,10 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.List;
-import static org.apache.slider.util.RestApiConstants.DEFAULT_COMPONENT_NAME;
-import static org.apache.slider.util.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
-import static org.apache.slider.util.RestApiErrorMessages.*;
-import static org.apache.slider.util.RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID;
-import static org.apache.slider.util.RestApiErrorMessages.ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET;
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
+import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME;
+import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
+import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*;
+import static org.easymock.EasyMock.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -136,7 +130,7 @@ public class TestServiceApiUtil {
} catch (IllegalArgumentException e) {
assertEquals(String.format(
RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID,
- RestApiConstants.DEFAULT_COMPONENT_NAME), e.getMessage());
+ DEFAULT_COMPONENT_NAME), e.getMessage());
}
// memory not specified
@@ -148,7 +142,7 @@ public class TestServiceApiUtil {
} catch (IllegalArgumentException e) {
assertEquals(String.format(
RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID,
- RestApiConstants.DEFAULT_COMPONENT_NAME), e.getMessage());
+ DEFAULT_COMPONENT_NAME), e.getMessage());
}
// invalid no of cpus
@@ -161,7 +155,7 @@ public class TestServiceApiUtil {
} catch (IllegalArgumentException e) {
assertEquals(String.format(
RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE,
- RestApiConstants.DEFAULT_COMPONENT_NAME), e.getMessage());
+ DEFAULT_COMPONENT_NAME), e.getMessage());
}
// number of containers not specified
@@ -183,7 +177,7 @@ public class TestServiceApiUtil {
} catch (IllegalArgumentException e) {
assertEquals(String.format(RestApiErrorMessages
.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
- RestApiConstants.DEFAULT_COMPONENT_NAME),
+ DEFAULT_COMPONENT_NAME),
e.getMessage());
}
@@ -476,8 +470,7 @@ public class TestServiceApiUtil {
@Test
public void testInvalidComponent() throws IOException {
SliderFileSystem sfs = initMock(null);
- testComponent(sfs, false);
- testComponent(sfs, true);
+ testComponent(sfs);
}
@Test
@@ -496,17 +489,15 @@ public class TestServiceApiUtil {
}
}
- private static void testComponent(SliderFileSystem sfs, boolean unique)
+ private static void testComponent(SliderFileSystem sfs)
throws IOException {
int maxLen = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
- if (unique) {
- assertEquals(19, Long.toString(Long.MAX_VALUE).length());
- maxLen = maxLen - Long.toString(Long.MAX_VALUE).length();
- }
+ assertEquals(19, Long.toString(Long.MAX_VALUE).length());
+ maxLen = maxLen - Long.toString(Long.MAX_VALUE).length();
+
String compName = LEN_64_STR.substring(0, maxLen + 1);
Application app = createValidApplication(null);
- app.addComponent(createValidComponent(compName).uniqueComponentSupport(
- unique));
+ app.addComponent(createValidComponent(compName));
// invalid component name fails if dns is enabled
try {
@@ -526,8 +517,7 @@ public class TestServiceApiUtil {
compName = LEN_64_STR.substring(0, maxLen);
app = createValidApplication(null);
- app.addComponent(createValidComponent(compName).uniqueComponentSupport(
- unique));
+ app.addComponent(createValidComponent(compName));
// does not fail
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 28105b2..a36e0b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.service;
-import com.google.common.base.Supplier;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -36,17 +35,16 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
import org.apache.hadoop.yarn.service.client.ServiceClient;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.Container;
-import org.apache.slider.api.resource.ContainerState;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.exceptions.SliderException;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -71,8 +69,8 @@ import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
-import static org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys.KEY_AM_RESOURCE_MEM;
-import static org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys.KEY_SLIDER_BASE_PATH;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.AM_RESOURCE_MEM;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
/**
* End to end tests to test deploying services with MiniYarnCluster and a in-JVM
@@ -122,8 +120,8 @@ public class TestYarnNativeServices extends ServiceTestUtils{
conf.setBoolean(TIMELINE_SERVICE_ENABLED, false);
conf.setInt(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100);
conf.setLong(DEBUG_NM_DELETE_DELAY_SEC, 60000);
- conf.setLong(KEY_AM_RESOURCE_MEM, 526);
- conf.setLong(InternalKeys.MONITOR_INTERVAL, 5);
+ conf.setLong(AM_RESOURCE_MEM, 526);
+ conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 5);
// Disable vmem check to disallow NM killing the container
conf.setBoolean(NM_VMEM_CHECK_ENABLED, false);
conf.setBoolean(NM_PMEM_CHECK_ENABLED, false);
@@ -143,7 +141,7 @@ public class TestYarnNativeServices extends ServiceTestUtils{
basedir.mkdirs();
}
- conf.set(KEY_SLIDER_BASE_PATH, basedir.getAbsolutePath());
+ conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
if (yarnCluster == null) {
yarnCluster =
@@ -267,7 +265,7 @@ public class TestYarnNativeServices extends ServiceTestUtils{
// stop the service
LOG.info("Stop the service");
- client.actionStop(exampleApp.getName());
+ client.actionStop(exampleApp.getName(), true);
ApplicationReport report = client.getYarnClient()
.getApplicationReport(ApplicationId.fromString(exampleApp.getId()));
// AM unregisters with RM successfully
@@ -303,7 +301,7 @@ public class TestYarnNativeServices extends ServiceTestUtils{
// check that containers for compa are launched before containers for compb
checkContainerLaunchDependencies(client, exampleApp, "compa", "compb");
- client.actionStop(exampleApp.getName());
+ client.actionStop(exampleApp.getName(), true);
client.actionDestroy(exampleApp.getName());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
index 4bc9f26..a22c000 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
@@ -20,11 +20,11 @@ package org.apache.hadoop.yarn.service.client;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
-import org.apache.slider.api.resource.Component;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -37,7 +37,7 @@ import java.util.List;
import java.util.Set;
import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
-import static org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys.KEY_SLIDER_BASE_PATH;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
/**
* Test for building / resolving components of type APPLICATION.
@@ -87,7 +87,7 @@ public class TestBuildExternalComponents {
} else {
basedir.mkdirs();
}
- conf.set(KEY_SLIDER_BASE_PATH, basedir.getAbsolutePath());
+ conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
index 1f07301..20c06ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
@@ -20,14 +20,15 @@ package org.apache.hadoop.yarn.service.client;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.ClientAMProtocol;
+import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -38,7 +39,7 @@ import java.io.IOException;
import java.util.List;
import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
-import static org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys.KEY_SLIDER_BASE_PATH;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
import static org.mockito.Mockito.mock;
public class TestServiceCLI {
@@ -59,7 +60,7 @@ public class TestServiceCLI {
@Before
public void setup() throws Throwable {
basedir = new File("target", "apps");
- conf.set(KEY_SLIDER_BASE_PATH, basedir.getAbsolutePath());
+ conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
fs = new SliderFileSystem(conf);
if (basedir.exists()) {
FileUtils.deleteDirectory(basedir);
@@ -71,7 +72,11 @@ public class TestServiceCLI {
cli = new ServiceCLI() {
@Override protected void createServiceClient() {
client = new ServiceClient() {
- @Override protected ClientAMProtocol connectToAM(String appName)
+ @Override protected ClientAMProtocol getAMProxy(String appName,
+ ApplicationReport report) throws IOException {
+ return mock(ClientAMProtocol.class);
+ }
+ @Override protected ClientAMProtocol getAMProxy(String appName)
throws IOException, YarnException {
return mock(ClientAMProtocol.class);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
index 9791976..9e13200 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
@@ -18,13 +18,14 @@
package org.apache.hadoop.yarn.service.conf;
-import org.apache.slider.api.resource.Application;
+
+import org.apache.hadoop.yarn.service.api.records.Application;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER;
+import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
/**
* Names of the example configs.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
index 66939a1..954d117 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
@@ -21,14 +21,13 @@ package org.apache.hadoop.yarn.service.conf;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.api.resource.ConfigFile.TypeEnum;
-import org.apache.slider.api.resource.Configuration;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.persist.JsonSerDeser;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -41,18 +40,9 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import static org.apache.slider.api.InternalKeys.CHAOS_MONKEY_INTERVAL;
-import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS;
-import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS;
-import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES;
-import static org.apache.hadoop.yarn.service.conf.ExampleAppJson.APP_JSON;
-import static org.apache.hadoop.yarn.service.conf.ExampleAppJson.EXTERNAL_JSON_1;
-import static org.apache.hadoop.yarn.service.conf.ExampleAppJson.OVERRIDE_JSON;
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.reset;
+import static org.apache.hadoop.yarn.service.conf.ExampleAppJson.*;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
+import static org.easymock.EasyMock.*;
/**
* Test global configuration resolution.
@@ -115,9 +105,9 @@ public class TestAppJsonResolve extends Assert {
Map<String, String> props = new HashMap<>();
props.put("k1", "overridden");
props.put("k2", "v2");
- files.add(new ConfigFile().destFile("file1").type(TypeEnum
+ files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
.PROPERTIES).props(props));
- files.add(new ConfigFile().destFile("file2").type(TypeEnum
+ files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
.XML).props(Collections.singletonMap("k3", "v3")));
assertTrue(files.contains(simple.getFiles().get(0)));
assertTrue(files.contains(simple.getFiles().get(1)));
@@ -132,9 +122,9 @@ public class TestAppJsonResolve extends Assert {
props.put("k1", "v1");
files.clear();
- files.add(new ConfigFile().destFile("file1").type(TypeEnum
+ files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
.PROPERTIES).props(props));
- files.add(new ConfigFile().destFile("file2").type(TypeEnum
+ files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
.XML).props(Collections.singletonMap("k3", "v3")));
assertTrue(files.contains(master.getFiles().get(0)));
@@ -208,7 +198,7 @@ public class TestAppJsonResolve extends Assert {
assertEquals("a", simple.getProperty("g1"));
assertEquals("b", simple.getProperty("g2"));
assertEquals("60",
- simple.getProperty("internal.chaos.monkey.interval.seconds"));
+ simple.getProperty("yarn.service.failure-count-reset.window"));
master = orig.getComponent("master").getConfiguration();
assertEquals(5, master.getProperties().size());
@@ -217,7 +207,7 @@ public class TestAppJsonResolve extends Assert {
assertEquals("b", master.getProperty("g2"));
assertEquals("is-overridden", master.getProperty("g3"));
assertEquals("60",
- simple.getProperty("internal.chaos.monkey.interval.seconds"));
+ simple.getProperty("yarn.service.failure-count-reset.window"));
Configuration worker = orig.getComponent("worker").getConfiguration();
LOG.info("worker = {}", worker);
@@ -226,27 +216,9 @@ public class TestAppJsonResolve extends Assert {
assertEquals("overridden-by-worker", worker.getProperty("g1"));
assertEquals("b", worker.getProperty("g2"));
assertEquals("60",
- worker.getProperty("internal.chaos.monkey.interval.seconds"));
+ worker.getProperty("yarn.service.failure-count-reset.window"));
other = orig.getComponent("other").getConfiguration();
assertEquals(0, other.getProperties().size());
}
-
- @Test
- public void testTimeIntervalLoading() throws Throwable {
- Application orig = ExampleAppJson.loadResource(APP_JSON);
-
- Configuration conf = orig.getConfiguration();
- long s = conf.getPropertyLong(
- CHAOS_MONKEY_INTERVAL + SliderUtils.SECONDS,
- 0);
- assertEquals(60, s);
- long monkeyInterval = SliderUtils.getTimeRange(conf,
- CHAOS_MONKEY_INTERVAL,
- DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS,
- DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS,
- DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES,
- 0);
- assertEquals(60L, monkeyInterval);
- }
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
index b304901..8310530 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
@@ -21,9 +21,9 @@ package org.apache.hadoop.yarn.service.conf;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.hadoop.yarn.service.api.records.Application;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -32,11 +32,8 @@ import org.junit.runners.Parameterized;
import java.util.Arrays;
import java.util.Collection;
-import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER;
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
+import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
+import static org.easymock.EasyMock.*;
/**
* Test loading example resources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
index 9a8dbee..98c78d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.yarn.service.conf;
-import org.apache.slider.common.tools.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.junit.Assert;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
index 568a066..5b24a1d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
@@ -19,10 +19,9 @@ package org.apache.hadoop.yarn.service.providers;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.api.resource.ConfigFile.TypeEnum;
import org.junit.Assert;
import org.junit.Test;
@@ -70,7 +69,7 @@ public class TestAbstractClientProvider {
} catch (IllegalArgumentException e) {
}
- configFile.setType(TypeEnum.TEMPLATE);
+ configFile.setType(ConfigFile.TypeEnum.TEMPLATE);
try {
clientProvider.validateConfigFiles(configFiles, mockFs);
Assert.fail(EXCEPTION_PREFIX + "empty src_file for type template");
@@ -92,7 +91,7 @@ public class TestAbstractClientProvider {
}
configFile = new ConfigFile();
- configFile.setType(TypeEnum.JSON);
+ configFile.setType(ConfigFile.TypeEnum.JSON);
configFile.setSrcFile(null);
configFile.setDestFile("path/destfile2");
configFiles.add(configFile);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
index 5cb5793..489578d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.service.providers;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Artifact.TypeEnum;
import org.apache.hadoop.yarn.service.provider.ProviderFactory;
import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultClientProvider;
import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
@@ -28,8 +30,7 @@ import org.apache.hadoop.yarn.service.provider.docker.DockerProviderService;
import org.apache.hadoop.yarn.service.provider.tarball.TarballClientProvider;
import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderService;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.Artifact.TypeEnum;
+
import org.junit.Test;
import static org.junit.Assert.assertTrue;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/servicemonitor/TestServiceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/servicemonitor/TestServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/servicemonitor/TestServiceMonitor.java
index db83cb6..6f5653f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/servicemonitor/TestServiceMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/servicemonitor/TestServiceMonitor.java
@@ -20,33 +20,14 @@
package org.apache.hadoop.yarn.service.servicemonitor;
import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.service.MockServiceAM;
import org.apache.hadoop.yarn.service.ServiceTestUtils;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.exceptions.BadClusterStateException;
+
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -54,15 +35,7 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
-import java.util.List;
-
-import static org.mockito.Matchers.anyFloat;
-import static org.mockito.Matchers.anyInt;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.*;
public class TestServiceMonitor extends ServiceTestUtils {
@@ -77,7 +50,7 @@ public class TestServiceMonitor extends ServiceTestUtils {
} else {
basedir.mkdirs();
}
- conf.setLong(InternalKeys.MONITOR_INTERVAL, 2);
+ conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 2);
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
index 476727a..a891df8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
@@ -19,8 +19,8 @@
package org.apache.hadoop.yarn.service.timelineservice;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
import org.apache.hadoop.yarn.client.api.TimelineV2Client;
@@ -28,19 +28,16 @@ import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.hadoop.yarn.service.ServiceScheduler;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.ApplicationState;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.Container;
-import org.apache.slider.api.resource.ContainerState;
-import org.apache.slider.api.resource.PlacementPolicy;
-import org.apache.slider.api.resource.Resource;
-import org.apache.slider.server.appmaster.actions.ActionStopSlider;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.ApplicationState;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
+import org.apache.hadoop.yarn.service.api.records.PlacementPolicy;
+import org.apache.hadoop.yarn.service.api.records.Resource;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceId;
-import org.apache.slider.server.appmaster.state.AppState;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -103,16 +100,9 @@ public class TestServiceTimelinePublisher {
@Test
public void testServiceAttemptEntity() {
- AppState appState = createMockAppState();
- int exitCode = 0;
- String message = "Stopped by user";
- ActionStopSlider stopAction = mock(ActionStopSlider.class);
- when(stopAction.getExitCode()).thenReturn(exitCode);
- when(stopAction.getFinalApplicationStatus())
- .thenReturn(FinalApplicationStatus.SUCCEEDED);
- when(stopAction.getMessage()).thenReturn(message);
-
- serviceTimelinePublisher.serviceAttemptRegistered(appState.getClusterStatus());
+ Application application = createMockApplication();
+ serviceTimelinePublisher
+ .serviceAttemptRegistered(application, new YarnConfiguration());
Collection<TimelineEntity> lastPublishedEntities =
((DummyTimelineClient) timelineClient).getLastPublishedEntities();
@@ -123,17 +113,21 @@ public class TestServiceTimelinePublisher {
.toString()) {
verifyComponentTimelineEntity(timelineEntity);
} else {
- verifyServiceAttemptTimelineEntity(timelineEntity, 0, null, true);
+ verifyServiceAttemptTimelineEntity(timelineEntity, null, true);
}
}
- serviceTimelinePublisher.serviceAttemptUnregistered(appState, stopAction);
+ ServiceContext context = new ServiceContext();
+ context.attemptId = ApplicationAttemptId
+ .newInstance(ApplicationId.fromString(application.getId()), 1);
+ String exitDiags = "service killed";
+ serviceTimelinePublisher.serviceAttemptUnregistered(context, exitDiags);
lastPublishedEntities =
((DummyTimelineClient) timelineClient).getLastPublishedEntities();
for (TimelineEntity timelineEntity : lastPublishedEntities) {
if (timelineEntity.getType() == ServiceTimelineEntityType.SERVICE_ATTEMPT
.toString()) {
- verifyServiceAttemptTimelineEntity(timelineEntity, exitCode, message,
+ verifyServiceAttemptTimelineEntity(timelineEntity, exitDiags,
false);
}
}
@@ -180,7 +174,7 @@ public class TestServiceTimelinePublisher {
}
private void verifyServiceAttemptTimelineEntity(TimelineEntity timelineEntity,
- int exitCode, String message, boolean isRegistedEntity) {
+ String message, boolean isRegistedEntity) {
assertEquals(SERVICEID, timelineEntity.getId());
assertEquals(SERVICE_NAME,
timelineEntity.getInfo().get(ServiceTimelineMetricsConstants.NAME));
@@ -190,13 +184,10 @@ public class TestServiceTimelinePublisher {
assertEquals(ServiceTimelineEvent.SERVICE_ATTEMPT_REGISTERED.toString(),
timelineEntity.getEvents().iterator().next().getId());
} else {
- assertEquals("SUCCEEDED",
- timelineEntity.getInfo().get(ServiceTimelineMetricsConstants.STATE));
- assertEquals(exitCode, timelineEntity.getInfo()
- .get(ServiceTimelineMetricsConstants.EXIT_STATUS_CODE));
+ assertEquals("ENDED",
+ timelineEntity.getInfo().get(ServiceTimelineMetricsConstants.STATE).toString());
assertEquals(message, timelineEntity.getInfo()
- .get(ServiceTimelineMetricsConstants.EXIT_REASON));
-
+ .get(ServiceTimelineMetricsConstants.DIAGNOSTICS_INFO));
assertEquals(2, timelineEntity.getEvents().size());
assertEquals(ServiceTimelineEvent.SERVICE_ATTEMPT_UNREGISTERED.toString(),
timelineEntity.getEvents().iterator().next().getId());
@@ -218,23 +209,20 @@ public class TestServiceTimelinePublisher {
assertEquals("sleep 1",
info.get(ServiceTimelineMetricsConstants.LAUNCH_COMMAND));
assertEquals("false",
- info.get(ServiceTimelineMetricsConstants.UNIQUE_COMPONENT_SUPPORT));
- assertEquals("false",
info.get(ServiceTimelineMetricsConstants.RUN_PRIVILEGED_CONTAINER));
assertEquals("label",
info.get(ServiceTimelineMetricsConstants.PLACEMENT_POLICY));
}
- private static AppState createMockAppState() {
- AppState appState = mock(AppState.class);
+ private static Application createMockApplication() {
Application application = mock(Application.class);
when(application.getId()).thenReturn(SERVICEID);
when(application.getLaunchTime()).thenReturn(new Date());
when(application.getState()).thenReturn(ApplicationState.STARTED);
when(application.getName()).thenReturn(SERVICE_NAME);
- when(application.getConfiguration())
- .thenReturn(new org.apache.slider.api.resource.Configuration());
+ when(application.getConfiguration()).thenReturn(
+ new org.apache.hadoop.yarn.service.api.records.Configuration());
Component component = mock(Component.class);
Artifact artifact = new Artifact();
@@ -250,19 +238,13 @@ public class TestServiceTimelinePublisher {
PlacementPolicy placementPolicy = new PlacementPolicy();
placementPolicy.setLabel("label");
when(component.getPlacementPolicy()).thenReturn(placementPolicy);
- when(component.getConfiguration())
- .thenReturn(new org.apache.slider.api.resource.Configuration());
+ when(component.getConfiguration()).thenReturn(
+ new org.apache.hadoop.yarn.service.api.records.Configuration());
List<Component> components = new ArrayList<Component>();
components.add(component);
when(application.getComponents()).thenReturn(components);
- when(appState.getClusterStatus()).thenReturn(application);
- return appState;
- }
-
- public static void main(String[] args) {
- Application application = createMockAppState().getClusterStatus();
- System.out.println(application.getConfiguration());
+ return application;
}
protected static class DummyTimelineClient extends TimelineV2ClientImpl {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/api/TestRPCBinding.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/api/TestRPCBinding.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/api/TestRPCBinding.java
deleted file mode 100644
index 0c2a2aa..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/api/TestRPCBinding.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.slider.server.appmaster.rpc.RpcBinder;
-import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPB;
-import org.junit.Test;
-
-import java.net.InetSocketAddress;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests RPC work.
- */
-public class TestRPCBinding {
-
- //@Test
- public void testRegistration() throws Throwable {
- Configuration conf = new Configuration();
- RpcBinder.registerSliderAPI(conf);
- assertTrue(RpcBinder.verifyBondedToProtobuf(conf,
- SliderClusterProtocolPB.class));
- }
-
- //@Test
- public void testGetProxy() throws Throwable {
- Configuration conf = new Configuration();
- InetSocketAddress saddr = new InetSocketAddress("127.0.0.1", 9000);
- SliderClusterProtocol proxy =
- RpcBinder.connectToServer(saddr, null, conf, 1000);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[59/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
new file mode 100644
index 0000000..be36335
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
@@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME;
+import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
+import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*;
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for ServiceApiUtil helper methods.
+ */
+public class TestServiceApiUtil {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestServiceApiUtil.class);
+ private static final String EXCEPTION_PREFIX = "Should have thrown " +
+ "exception: ";
+ private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
+ "exception: ";
+
+ private static final String LEN_64_STR =
+ "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01";
+
+ private static final YarnConfiguration CONF_DEFAULT_DNS = new
+ YarnConfiguration();
+ private static final YarnConfiguration CONF_DNS_ENABLED = new
+ YarnConfiguration();
+
+ @BeforeClass
+ public static void init() {
+ CONF_DNS_ENABLED.setBoolean(RegistryConstants.KEY_DNS_ENABLED, true);
+ }
+
+ @Test(timeout = 90000)
+ public void testResourceValidation() throws Exception {
+ assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR
+ .length());
+
+ SliderFileSystem sfs = initMock(null);
+
+ Service app = new Service();
+
+ // no name
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no name");
+ } catch (IllegalArgumentException e) {
+ assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
+ }
+
+ // bad format name
+ String[] badNames = {"4finance", "Finance", "finance@home", LEN_64_STR};
+ for (String badName : badNames) {
+ app.setName(badName);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with bad name " + badName);
+ } catch (IllegalArgumentException e) {
+
+ }
+ }
+
+ // launch command not specified
+ app.setName(LEN_64_STR);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DEFAULT_DNS);
+ Assert.fail(EXCEPTION_PREFIX + "service with no launch command");
+ } catch (IllegalArgumentException e) {
+ assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
+ e.getMessage());
+ }
+
+ // launch command not specified
+ app.setName(LEN_64_STR.substring(0, RegistryConstants
+ .MAX_FQDN_LABEL_LENGTH));
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no launch command");
+ } catch (IllegalArgumentException e) {
+ assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
+ e.getMessage());
+ }
+
+ // resource not specified
+ app.setLaunchCommand("sleep 3600");
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no resource");
+ } catch (IllegalArgumentException e) {
+ assertEquals(String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID,
+ DEFAULT_COMPONENT_NAME), e.getMessage());
+ }
+
+ // memory not specified
+ Resource res = new Resource();
+ app.setResource(res);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no memory");
+ } catch (IllegalArgumentException e) {
+ assertEquals(String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID,
+ DEFAULT_COMPONENT_NAME), e.getMessage());
+ }
+
+ // invalid no of cpus
+ res.setMemory("100mb");
+ res.setCpus(-2);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(
+ EXCEPTION_PREFIX + "service with invalid no of cpus");
+ } catch (IllegalArgumentException e) {
+ assertEquals(String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE,
+ DEFAULT_COMPONENT_NAME), e.getMessage());
+ }
+
+ // number of containers not specified
+ res.setCpus(2);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no container count");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage()
+ .contains(ERROR_CONTAINERS_COUNT_INVALID));
+ }
+
+ // specifying profile along with cpus/memory raises exception
+ res.setProfile("hbase_finance_large");
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX
+ + "service with resource profile along with cpus/memory");
+ } catch (IllegalArgumentException e) {
+ assertEquals(String.format(RestApiErrorMessages
+ .ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
+ DEFAULT_COMPONENT_NAME),
+ e.getMessage());
+ }
+
+ // currently resource profile alone is not supported.
+ // TODO: remove the next test once resource profile alone is supported.
+ res.setCpus(null);
+ res.setMemory(null);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with resource profile only");
+ } catch (IllegalArgumentException e) {
+ assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET,
+ e.getMessage());
+ }
+
+ // unset profile here and add cpus/memory back
+ res.setProfile(null);
+ res.setCpus(2);
+ res.setMemory("2gb");
+
+ // null number of containers
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "null number of containers");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
+ }
+
+ // negative number of containers
+ app.setNumberOfContainers(-1L);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "negative number of containers");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
+ }
+
+ // everything valid here
+ app.setNumberOfContainers(5L);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ } catch (IllegalArgumentException e) {
+ LOG.error("service attributes specified should be valid here", e);
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testArtifacts() throws IOException {
+ SliderFileSystem sfs = initMock(null);
+
+ Service app = new Service();
+ app.setName("name");
+ Resource res = new Resource();
+ app.setResource(res);
+ res.setMemory("512M");
+ app.setNumberOfContainers(3L);
+
+ // no artifact id fails with default type
+ Artifact artifact = new Artifact();
+ app.setArtifact(artifact);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no artifact id");
+ } catch (IllegalArgumentException e) {
+ assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
+ }
+
+ // no artifact id fails with SERVICE type
+ artifact.setType(Artifact.TypeEnum.SERVICE);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no artifact id");
+ } catch (IllegalArgumentException e) {
+ assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
+ }
+
+ // no artifact id fails with TARBALL type
+ artifact.setType(Artifact.TypeEnum.TARBALL);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no artifact id");
+ } catch (IllegalArgumentException e) {
+ assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
+ }
+
+ // everything valid here
+ artifact.setType(Artifact.TypeEnum.DOCKER);
+ artifact.setId("docker.io/centos:centos7");
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ } catch (IllegalArgumentException e) {
+ LOG.error("service attributes specified should be valid here", e);
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ // defaults assigned
+ assertEquals(app.getComponents().get(0).getName(),
+ DEFAULT_COMPONENT_NAME);
+ assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME);
+ }
+
+ private static Resource createValidResource() {
+ Resource res = new Resource();
+ res.setMemory("512M");
+ return res;
+ }
+
+ private static Component createValidComponent(String compName) {
+ Component comp = new Component();
+ comp.setName(compName);
+ comp.setResource(createValidResource());
+ comp.setNumberOfContainers(1L);
+ return comp;
+ }
+
+ private static Service createValidApplication(String compName) {
+ Service app = new Service();
+ app.setLaunchCommand("sleep 3600");
+ app.setName("name");
+ app.setResource(createValidResource());
+ app.setNumberOfContainers(1L);
+ if (compName != null) {
+ app.addComponent(createValidComponent(compName));
+ }
+ return app;
+ }
+
+ private static SliderFileSystem initMock(Service ext) throws IOException {
+ SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ JsonSerDeser<Service> jsonSerDeser = createNiceMock(JsonSerDeser
+ .class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ if (ext != null) {
+ expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
+ .anyTimes();
+ }
+ replay(sfs, mockFs, jsonSerDeser);
+ ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
+ return sfs;
+ }
+
+ @Test
+ public void testExternalApplication() throws IOException {
+ Service ext = createValidApplication("comp1");
+ SliderFileSystem sfs = initMock(ext);
+
+ Service app = createValidApplication(null);
+
+ Artifact artifact = new Artifact();
+ artifact.setType(Artifact.TypeEnum.SERVICE);
+ artifact.setId("id");
+ app.setArtifact(artifact);
+
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ assertEquals(1, app.getComponents().size());
+ assertNotNull(app.getComponent("comp1"));
+ }
+
+ @Test
+ public void testDuplicateComponents() throws IOException {
+ SliderFileSystem sfs = initMock(null);
+
+ String compName = "comp1";
+ Service app = createValidApplication(compName);
+ app.addComponent(createValidComponent(compName));
+
+ // duplicate component name fails
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with component collision");
+ } catch (IllegalArgumentException e) {
+ assertEquals("Component name collision: " + compName, e.getMessage());
+ }
+ }
+
+ @Test
+ public void testExternalDuplicateComponent() throws IOException {
+ Service ext = createValidApplication("comp1");
+ SliderFileSystem sfs = initMock(ext);
+
+ Service app = createValidApplication("comp1");
+ Artifact artifact = new Artifact();
+ artifact.setType(Artifact.TypeEnum.SERVICE);
+ artifact.setId("id");
+ app.getComponent("comp1").setArtifact(artifact);
+
+ // duplicate component name okay in the case of SERVICE component
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testExternalComponent() throws IOException {
+ Service ext = createValidApplication("comp1");
+ SliderFileSystem sfs = initMock(ext);
+
+ Service app = createValidApplication("comp2");
+ Artifact artifact = new Artifact();
+ artifact.setType(Artifact.TypeEnum.SERVICE);
+ artifact.setId("id");
+ app.setArtifact(artifact);
+
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ assertEquals(1, app.getComponents().size());
+ // artifact ID not inherited from global
+ assertNotNull(app.getComponent("comp2"));
+
+ // set SERVICE artifact id on component
+ app.getComponent("comp2").setArtifact(artifact);
+
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ assertEquals(1, app.getComponents().size());
+ // original component replaced by external component
+ assertNotNull(app.getComponent("comp1"));
+ }
+
+ public static void verifyDependencySorting(List<Component> components,
+ Component... expectedSorting) {
+ Collection<Component> actualSorting = ServiceApiUtil.sortByDependencies(
+ components);
+ assertEquals(expectedSorting.length, actualSorting.size());
+ int i = 0;
+ for (Component component : actualSorting) {
+ assertEquals(expectedSorting[i++], component);
+ }
+ }
+
+ @Test
+ public void testDependencySorting() throws IOException {
+ Component a = new Component().name("a");
+ Component b = new Component().name("b");
+ Component c = new Component().name("c");
+ Component d = new Component().name("d").dependencies(Arrays.asList("c"));
+ Component e = new Component().name("e").dependencies(Arrays.asList("b",
+ "d"));
+
+ verifyDependencySorting(Arrays.asList(a, b, c), a, b, c);
+ verifyDependencySorting(Arrays.asList(c, a, b), c, a, b);
+ verifyDependencySorting(Arrays.asList(a, b, c, d, e), a, b, c, d, e);
+ verifyDependencySorting(Arrays.asList(e, d, c, b, a), c, b, a, d, e);
+
+ c.setDependencies(Arrays.asList("e"));
+ try {
+ verifyDependencySorting(Arrays.asList(a, b, c, d, e));
+ Assert.fail(EXCEPTION_PREFIX + "components with dependency cycle");
+ } catch (IllegalArgumentException ex) {
+ assertEquals(String.format(
+ RestApiErrorMessages.ERROR_DEPENDENCY_CYCLE, Arrays.asList(c, d,
+ e)), ex.getMessage());
+ }
+
+ SliderFileSystem sfs = initMock(null);
+ Service service = createValidApplication(null);
+ service.setComponents(Arrays.asList(c, d, e));
+ try {
+ ServiceApiUtil.validateAndResolveService(service, sfs,
+ CONF_DEFAULT_DNS);
+ Assert.fail(EXCEPTION_PREFIX + "components with bad dependencies");
+ } catch (IllegalArgumentException ex) {
+ assertEquals(String.format(
+ RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, "b", "e"), ex
+ .getMessage());
+ }
+ }
+
+ @Test
+ public void testInvalidComponent() throws IOException {
+ SliderFileSystem sfs = initMock(null);
+ testComponent(sfs);
+ }
+
+ @Test
+ public void testValidateCompName() {
+ String[] invalidNames = {
+ "EXAMPLE", // UPPER case not allowed
+ "example_app" // underscore not allowed.
+ };
+ for (String name : invalidNames) {
+ try {
+ ServiceApiUtil.validateNameFormat(name, new Configuration());
+ Assert.fail();
+ } catch (IllegalArgumentException ex) {
+ ex.printStackTrace();
+ }
+ }
+ }
+
+ private static void testComponent(SliderFileSystem sfs)
+ throws IOException {
+ int maxLen = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
+ assertEquals(19, Long.toString(Long.MAX_VALUE).length());
+ maxLen = maxLen - Long.toString(Long.MAX_VALUE).length();
+
+ String compName = LEN_64_STR.substring(0, maxLen + 1);
+ Service app = createValidApplication(null);
+ app.addComponent(createValidComponent(compName));
+
+ // invalid component name fails if dns is enabled
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with invalid component name");
+ } catch (IllegalArgumentException e) {
+ assertEquals(String.format(RestApiErrorMessages
+ .ERROR_COMPONENT_NAME_INVALID, maxLen, compName), e.getMessage());
+ }
+
+ // does not fail if dns is disabled
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DEFAULT_DNS);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ compName = LEN_64_STR.substring(0, maxLen);
+ app = createValidApplication(null);
+ app.addComponent(createValidComponent(compName));
+
+ // does not fail
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
new file mode 100644
index 0000000..63aa9c6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.curator.test.TestingCluster;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
+import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM;
+import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.AM_RESOURCE_MEM;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
+
+/**
+ * End to end tests to test deploying services with MiniYarnCluster and a in-JVM
+ * ZK testing cluster.
+ */
+public class TestYarnNativeServices extends ServiceTestUtils{
+
+ private static final Log LOG =
+ LogFactory.getLog(TestYarnNativeServices.class);
+
+ private MiniYARNCluster yarnCluster = null;
+ private MiniDFSCluster hdfsCluster = null;
+ private FileSystem fs = null;
+ protected Configuration conf = null;
+ private static final int NUM_NMS = 1;
+ private File basedir;
+
+ @Rule
+ public TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ @Before
+ public void setup() throws Exception {
+ setupInternal(NUM_NMS);
+ }
+
+ private void setupInternal(int numNodeManager)
+ throws Exception {
+ LOG.info("Starting up YARN cluster");
+// Logger rootLogger = LogManager.getRootLogger();
+// rootLogger.setLevel(Level.DEBUG);
+ conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
+ // reduce the teardown waiting time
+ conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000);
+ conf.set("yarn.log.dir", "target");
+ // mark if we need to launch the v1 timeline server
+ // disable aux-service based timeline aggregators
+ conf.set(YarnConfiguration.NM_AUX_SERVICES, "");
+ conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8");
+ // Enable ContainersMonitorImpl
+ conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR,
+ LinuxResourceCalculatorPlugin.class.getName());
+ conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE,
+ ProcfsBasedProcessTree.class.getName());
+ conf.setBoolean(
+ YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, true);
+ conf.setBoolean(TIMELINE_SERVICE_ENABLED, false);
+ conf.setInt(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100);
+ conf.setLong(DEBUG_NM_DELETE_DELAY_SEC, 60000);
+ conf.setLong(AM_RESOURCE_MEM, 526);
+ conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 5);
+ // Disable vmem check to disallow NM killing the container
+ conf.setBoolean(NM_VMEM_CHECK_ENABLED, false);
+ conf.setBoolean(NM_PMEM_CHECK_ENABLED, false);
+ // setup zk cluster
+ TestingCluster zkCluster;
+ zkCluster = new TestingCluster(1);
+ zkCluster.start();
+ conf.set(YarnConfiguration.RM_ZK_ADDRESS, zkCluster.getConnectString());
+ conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString());
+ LOG.info("ZK cluster: " + zkCluster.getConnectString());
+
+ fs = FileSystem.get(conf);
+ basedir = new File("target", "apps");
+ if (basedir.exists()) {
+ FileUtils.deleteDirectory(basedir);
+ } else {
+ basedir.mkdirs();
+ }
+
+ conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
+
+ if (yarnCluster == null) {
+ yarnCluster =
+ new MiniYARNCluster(TestYarnNativeServices.class.getSimpleName(), 1,
+ numNodeManager, 1, 1);
+ yarnCluster.init(conf);
+ yarnCluster.start();
+
+ waitForNMsToRegister();
+
+ URL url = Thread.currentThread().getContextClassLoader()
+ .getResource("yarn-site.xml");
+ if (url == null) {
+ throw new RuntimeException(
+ "Could not find 'yarn-site.xml' dummy file in classpath");
+ }
+ Configuration yarnClusterConfig = yarnCluster.getConfig();
+ yarnClusterConfig.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+ new File(url.getPath()).getParent());
+ //write the document to a buffer (not directly to the file, as that
+ //can cause the file being written to get read -which will then fail.
+ ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+ yarnClusterConfig.writeXml(bytesOut);
+ bytesOut.close();
+ //write the bytes to the file in the classpath
+ OutputStream os = new FileOutputStream(new File(url.getPath()));
+ os.write(bytesOut.toByteArray());
+ os.close();
+ LOG.info("Write yarn-site.xml configs to: " + url);
+ }
+ if (hdfsCluster == null) {
+ HdfsConfiguration hdfsConfig = new HdfsConfiguration();
+ hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig)
+ .numDataNodes(1).build();
+ }
+
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ LOG.info("setup thread sleep interrupted. message=" + e.getMessage());
+ }
+
+
+ }
+
+ private void waitForNMsToRegister() throws Exception {
+ int sec = 60;
+ while (sec >= 0) {
+ if (yarnCluster.getResourceManager().getRMContext().getRMNodes().size()
+ >= NUM_NMS) {
+ break;
+ }
+ Thread.sleep(1000);
+ sec--;
+ }
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (yarnCluster != null) {
+ try {
+ yarnCluster.stop();
+ } finally {
+ yarnCluster = null;
+ }
+ }
+ if (hdfsCluster != null) {
+ try {
+ hdfsCluster.shutdown();
+ } finally {
+ hdfsCluster = null;
+ }
+ }
+ if (basedir != null) {
+ FileUtils.deleteDirectory(basedir);
+ }
+ SliderFileSystem sfs = new SliderFileSystem(conf);
+ Path appDir = sfs.getBaseApplicationPath();
+ sfs.getFileSystem().delete(appDir, true);
+ }
+
+
+
+ // End-to-end test to use ServiceClient to deploy a service.
+ // 1. Create a service with 2 components, each of which has 2 containers
+ // 2. Flex up each component to 3 containers and check the component instance names
+ // 3. Flex down each component to 1 container and check the component instance names
+ // 4. Flex up each component to 2 containers and check the component instance names
+ // 5. Stop the service
+ // 6. Destroy the service
+ @Test (timeout = 200000)
+ public void testCreateFlexStopDestroyService() throws Exception {
+ ServiceClient client = createClient();
+ Service exampleApp = createExampleApplication();
+ client.actionCreate(exampleApp);
+ SliderFileSystem fileSystem = new SliderFileSystem(conf);
+ Path appDir = fileSystem.buildClusterDirPath(exampleApp.getName());
+ // check app.json is persisted.
+ Assert.assertTrue(
+ fs.exists(new Path(appDir, exampleApp.getName() + ".json")));
+ waitForAllCompToBeReady(client, exampleApp);
+
+ // Flex two components, each from 2 container to 3 containers.
+ flexComponents(client, exampleApp, 3L);
+ // wait for flex to be completed, increase from 2 to 3 containers.
+ waitForAllCompToBeReady(client, exampleApp);
+ // check all instances name for each component are in sequential order.
+ checkCompInstancesInOrder(client, exampleApp);
+
+ // flex down to 1
+ flexComponents(client, exampleApp, 1L);
+ waitForAllCompToBeReady(client, exampleApp);
+ checkCompInstancesInOrder(client, exampleApp);
+
+ // check component dir and registry are cleaned up.
+
+ // flex up again to 2
+ flexComponents(client, exampleApp, 2L);
+ waitForAllCompToBeReady(client, exampleApp);
+ checkCompInstancesInOrder(client, exampleApp);
+
+ // stop the service
+ LOG.info("Stop the service");
+ client.actionStop(exampleApp.getName(), true);
+ ApplicationReport report = client.getYarnClient()
+ .getApplicationReport(ApplicationId.fromString(exampleApp.getId()));
+ // AM unregisters with RM successfully
+ Assert.assertEquals(FINISHED, report.getYarnApplicationState());
+ Assert.assertEquals(FinalApplicationStatus.ENDED,
+ report.getFinalApplicationStatus());
+
+ LOG.info("Destroy the service");
+ //destroy the service and check the app dir is deleted from fs.
+ client.actionDestroy(exampleApp.getName());
+ // check the service dir on hdfs (in this case, local fs) are deleted.
+ Assert.assertFalse(fs.exists(appDir));
+ }
+
+ // Create compa with 2 containers
+ // Create compb with 2 containers which depends on compa
+ // Check containers for compa started before containers for compb
+ @Test (timeout = 200000)
+ public void testComponentStartOrder() throws Exception {
+ ServiceClient client = createClient();
+ Service exampleApp = new Service();
+ exampleApp.setName("teststartorder");
+ exampleApp.addComponent(createComponent("compa", 2, "sleep 1000"));
+ Component compb = createComponent("compb", 2, "sleep 1000");
+
+ // Let compb depedends on compa;
+ compb.setDependencies(Collections.singletonList("compa"));
+ exampleApp.addComponent(compb);
+
+ client.actionCreate(exampleApp);
+ waitForAllCompToBeReady(client, exampleApp);
+
+ // check that containers for compa are launched before containers for compb
+ checkContainerLaunchDependencies(client, exampleApp, "compa", "compb");
+
+ client.actionStop(exampleApp.getName(), true);
+ client.actionDestroy(exampleApp.getName());
+ }
+
+ // Check containers launched are in dependency order
+ // Get all containers into a list and sort based on container launch time e.g.
+ // compa-c1, compa-c2, compb-c1, compb-c2;
+ // check that the container's launch time are align with the dependencies.
+ private void checkContainerLaunchDependencies(ServiceClient client,
+ Service exampleApp, String... compOrder)
+ throws IOException, YarnException {
+ Service retrievedApp = client.getStatus(exampleApp.getName());
+ List<Container> containerList = new ArrayList<>();
+ for (Component component : retrievedApp.getComponents()) {
+ containerList.addAll(component.getContainers());
+ }
+ // sort based on launchTime
+ containerList
+ .sort((o1, o2) -> o1.getLaunchTime().compareTo(o2.getLaunchTime()));
+ LOG.info("containerList: " + containerList);
+ // check the containers are in the dependency order.
+ int index = 0;
+ for (String comp : compOrder) {
+ long num = retrievedApp.getComponent(comp).getNumberOfContainers();
+ for (int i = 0; i < num; i++) {
+ String compInstanceName = containerList.get(index).getComponentName();
+ String compName =
+ compInstanceName.substring(0, compInstanceName.lastIndexOf('-'));
+ Assert.assertEquals(comp, compName);
+ index++;
+ }
+ }
+ }
+
+
+ private Map<String, Long> flexComponents(ServiceClient client,
+ Service exampleApp, long count) throws YarnException, IOException {
+ Map<String, Long> compCounts = new HashMap<>();
+ compCounts.put("compa", count);
+ compCounts.put("compb", count);
+ // flex will update the persisted conf to reflect latest number of containers.
+ exampleApp.getComponent("compa").setNumberOfContainers(count);
+ exampleApp.getComponent("compb").setNumberOfContainers(count);
+ client.flexByRestService(exampleApp.getName(), compCounts);
+ return compCounts;
+ }
+
+ // Check each component's comp instances name are in sequential order.
+ // E.g. If there are two instances compA-1 and compA-2
+ // When flex up to 4 instances, it should be compA-1 , compA-2, compA-3, compA-4
+ // When flex down to 3 instances, it should be compA-1 , compA-2, compA-3.
+ private void checkCompInstancesInOrder(ServiceClient client,
+ Service exampleApp) throws IOException, YarnException {
+ Service service = client.getStatus(exampleApp.getName());
+ for (Component comp : service.getComponents()) {
+ checkEachCompInstancesInOrder(comp);
+ }
+ }
+
+ private void checkRegistryAndCompDirDeleted() {
+
+ }
+
+ private void checkEachCompInstancesInOrder(Component component) {
+ long expectedNumInstances = component.getNumberOfContainers();
+ Assert.assertEquals(expectedNumInstances, component.getContainers().size());
+ TreeSet<String> instances = new TreeSet<>();
+ for (Container container : component.getContainers()) {
+ instances.add(container.getComponentName());
+ }
+
+ int i = 0;
+ for (String s : instances) {
+ Assert.assertEquals(component.getName() + "-" + i, s);
+ i++;
+ }
+ }
+
+ private void waitForOneCompToBeReady(ServiceClient client,
+ Service exampleApp, String readyComp)
+ throws TimeoutException, InterruptedException {
+ long numExpectedContainers =
+ exampleApp.getComponent(readyComp).getNumberOfContainers();
+ GenericTestUtils.waitFor(() -> {
+ try {
+ Service retrievedApp = client.getStatus(exampleApp.getName());
+ Component retrievedComp = retrievedApp.getComponent(readyComp);
+
+ if (retrievedComp.getContainers() != null
+ && retrievedComp.getContainers().size() == numExpectedContainers) {
+ LOG.info(readyComp + " found " + numExpectedContainers
+ + " containers running");
+ return true;
+ } else {
+ LOG.info(" Waiting for " + readyComp + "'s containers to be running");
+ return false;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ }, 2000, 200000);
+ }
+
+ // wait until all the containers for all components become ready state
+ private void waitForAllCompToBeReady(ServiceClient client,
+ Service exampleApp) throws TimeoutException, InterruptedException {
+ int expectedTotalContainers = countTotalContainers(exampleApp);
+ GenericTestUtils.waitFor(() -> {
+ try {
+ Service retrievedApp = client.getStatus(exampleApp.getName());
+ int totalReadyContainers = 0;
+ LOG.info("Num Components " + retrievedApp.getComponents().size());
+ for (Component component : retrievedApp.getComponents()) {
+ LOG.info("looking for " + component.getName());
+ LOG.info(component);
+ if (component.getContainers() != null) {
+ if (component.getContainers().size() == exampleApp
+ .getComponent(component.getName()).getNumberOfContainers()) {
+ for (Container container : component.getContainers()) {
+ LOG.info(
+ "Container state " + container.getState() + ", component "
+ + component.getName());
+ if (container.getState() == ContainerState.READY) {
+ totalReadyContainers++;
+ LOG.info("Found 1 ready container " + container.getId());
+ }
+ }
+ } else {
+ LOG.info(component.getName() + " Expected number of containers "
+ + exampleApp.getComponent(component.getName())
+ .getNumberOfContainers() + ", current = " + component
+ .getContainers());
+ }
+ }
+ }
+ LOG.info("Exit loop, totalReadyContainers= " + totalReadyContainers
+ + " expected = " + expectedTotalContainers);
+ return totalReadyContainers == expectedTotalContainers;
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ }, 2000, 200000);
+ }
+
+ private ServiceClient createClient() throws Exception {
+ ServiceClient client = new ServiceClient() {
+ @Override protected Path addJarResource(String appName,
+ Map<String, LocalResource> localResources)
+ throws IOException, SliderException {
+ // do nothing, the Unit test will use local jars
+ return null;
+ }
+ };
+ client.init(conf);
+ client.start();
+ return client;
+ }
+
+
+ private int countTotalContainers(Service service) {
+ int totalContainers = 0;
+ for (Component component : service.getComponents()) {
+ totalContainers += component.getNumberOfContainers();
+ }
+ return totalContainers;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
new file mode 100644
index 0000000..93a15cc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
+import org.apache.hadoop.yarn.service.client.params.ClientArgs;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
+
+/**
+ * Test for building / resolving components of type SERVICE.
+ */
+public class TestBuildExternalComponents {
+
+ protected Configuration conf = new YarnConfiguration();
+ private File basedir;
+
+ // Check component names match with expected
+ private static void checkComponentNames(List<Component> components,
+ Set<String> expectedComponents) {
+ Assert.assertEquals(expectedComponents.size(), components.size());
+ for (Component comp : components) {
+ Assert.assertTrue(expectedComponents.contains(comp.getName()));
+ }
+ }
+
+ // 1. Build the appDef and store on fs
+ // 2. check component names
+ private void buildAndCheckComponents(String appName, String appDef,
+ SliderFileSystem sfs, Set<String> names) throws Throwable {
+ String[] args =
+ { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
+ ClientArgs clientArgs = new ClientArgs(args);
+ clientArgs.parse();
+ ServiceCLI cli = new ServiceCLI() {
+ @Override protected void createServiceClient() {
+ client = new ServiceClient();
+ client.init(conf);
+ client.start();
+ }
+ };
+ cli.exec(clientArgs);
+
+ // verify generated conf
+ List<Component> components =
+ ServiceApiUtil.getComponents(sfs, appName);
+ checkComponentNames(components, names);
+ }
+
+ @Before
+ public void setup() throws IOException {
+ basedir = new File("target", "apps");
+ if (basedir.exists()) {
+ FileUtils.deleteDirectory(basedir);
+ } else {
+ basedir.mkdirs();
+ }
+ conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (basedir != null) {
+ FileUtils.deleteDirectory(basedir);
+ }
+ }
+
+ // Test applications defining external components(SERVICE type)
+ // can be resolved correctly
+ @Test
+ public void testExternalComponentBuild() throws Throwable {
+ SliderFileSystem sfs = new SliderFileSystem(conf);
+
+ Set<String> nameSet = new HashSet<>();
+ nameSet.add("simple");
+ nameSet.add("master");
+ nameSet.add("worker");
+
+ // app-1 has 3 components: simple, master, worker
+ buildAndCheckComponents("app-1", ExampleAppJson.APP_JSON, sfs, nameSet);
+ buildAndCheckComponents("external-0", ExampleAppJson.EXTERNAL_JSON_0, sfs,
+ nameSet);
+
+ nameSet.add("other");
+
+ // external1 has 3 components: simple(SERVICE - app1), master and other
+ buildAndCheckComponents("external-1", ExampleAppJson.EXTERNAL_JSON_1, sfs,
+ nameSet);
+
+ nameSet.add("another");
+
+ // external2 has 2 components: ext(SERVICE - external1), another
+ buildAndCheckComponents("external-2", ExampleAppJson.EXTERNAL_JSON_2, sfs,
+ nameSet);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
new file mode 100644
index 0000000..ecc529d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.client.params.ClientArgs;
+import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.util.Records;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS;
+import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.*;
+
+public class TestServiceCLI {
+
+ protected Configuration conf = new YarnConfiguration();
+ private File basedir;
+ private ServiceCLI cli;
+ private SliderFileSystem fs;
+
+ private void buildApp(String appName, String appDef) throws Throwable {
+ String[] args =
+ { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
+ ClientArgs clientArgs = new ClientArgs(args);
+ clientArgs.parse();
+ cli.exec(clientArgs);
+ }
+
+ @Before
+ public void setup() throws Throwable {
+ basedir = new File("target", "apps");
+ conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
+ conf.setLong(RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 0);
+ conf.setLong(RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1);
+ conf.setInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
+ conf.setInt(CommonConfigurationKeysPublic.
+ IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
+ fs = new SliderFileSystem(conf);
+ if (basedir.exists()) {
+ FileUtils.deleteDirectory(basedir);
+ } else {
+ basedir.mkdirs();
+ }
+
+ // create a CLI and skip connection to AM
+ cli = new ServiceCLI() {
+ @Override protected void createServiceClient() {
+ client = new ServiceClient() {
+ @Override
+ protected void serviceInit(Configuration configuration)
+ throws Exception {
+ super.serviceInit(conf);
+ yarnClient = spy(yarnClient);
+ ApplicationReport report = Records.newRecord(ApplicationReport.class);
+ report.setYarnApplicationState(YarnApplicationState.RUNNING);
+ report.setHost("localhost");
+ doReturn(report).when(yarnClient).getApplicationReport(anyObject());
+ }
+ @Override
+ protected ClientAMProtocol createAMProxy(String host, int port)
+ throws IOException {
+ return mock(ClientAMProtocol.class);
+ }
+ };
+ client.init(conf);
+ client.start();
+ }
+ };
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (basedir != null) {
+ FileUtils.deleteDirectory(basedir);
+ }
+ }
+
+ // Test flex components count are persisted.
+ @Test
+ public void testFlexComponents() throws Throwable {
+ buildApp("service-1", ExampleAppJson.APP_JSON);
+
+ checkCompCount("master", 1L);
+
+ // increase by 2
+ String[] flexUpArgs = {"flex", "service-1", "--component", "master" , "+2"};
+ ClientArgs clientArgs = new ClientArgs(flexUpArgs);
+ clientArgs.parse();
+ cli.exec(clientArgs);
+ checkCompCount("master", 3L);
+
+ // decrease by 1
+ String[] flexDownArgs = {"flex", "service-1", "--component", "master", "-1"};
+ clientArgs = new ClientArgs(flexDownArgs);
+ clientArgs.parse();
+ cli.exec(clientArgs);
+ checkCompCount("master", 2L);
+
+ String[] flexAbsoluteArgs = {"flex", "service-1", "--component", "master", "10"};
+ clientArgs = new ClientArgs(flexAbsoluteArgs);
+ clientArgs.parse();
+ cli.exec(clientArgs);
+ checkCompCount("master", 10L);
+ }
+
+ private void checkCompCount(String compName, long count) throws IOException {
+ List<Component> components =
+ ServiceApiUtil.getComponents(fs, "service-1");
+ for (Component component : components) {
+ if (component.getName().equals(compName)) {
+ Assert.assertEquals(count, component.getNumberOfContainers().longValue());
+ return;
+ }
+ }
+ Assert.fail();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
new file mode 100644
index 0000000..5fdd2ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+
+import org.apache.hadoop.yarn.service.api.records.Service;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
+
+/**
+ * Names of the example configs.
+ */
+public final class ExampleAppJson {
+
+ public static final String APP_JSON = "app.json";
+ public static final String OVERRIDE_JSON = "app-override.json";
+ public static final String DEFAULT_JSON = "default.json";
+ public static final String EXTERNAL_JSON_0 = "external0.json";
+ public static final String EXTERNAL_JSON_1 = "external1.json";
+ public static final String EXTERNAL_JSON_2 = "external2.json";
+
+ public static final String PACKAGE = "/org/apache/hadoop/yarn/service/conf/examples/";
+
+
+ private static final String[] ALL_EXAMPLES = {APP_JSON, OVERRIDE_JSON,
+ DEFAULT_JSON};
+
+ public static final List<String> ALL_EXAMPLE_RESOURCES = new ArrayList<>();
+ static {
+ for (String example : ALL_EXAMPLES) {
+ ALL_EXAMPLE_RESOURCES.add(PACKAGE + example);
+ }
+ }
+
+ private ExampleAppJson() {
+ }
+
+ public static Service loadResource(String name) throws IOException {
+ return JSON_SER_DESER.fromResource(PACKAGE + name);
+ }
+
+ public static String resourceName(String name) {
+ return "target/test-classes" + PACKAGE + name;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
new file mode 100644
index 0000000..04ec526
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.service.conf.ExampleAppJson.*;
+import static org.easymock.EasyMock.*;
+
+/**
+ * Test global configuration resolution.
+ */
+public class TestAppJsonResolve extends Assert {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(TestAppJsonResolve.class);
+
+ @Test
+ public void testOverride() throws Throwable {
+ Service orig = ExampleAppJson.loadResource(OVERRIDE_JSON);
+
+ Configuration global = orig.getConfiguration();
+ assertEquals("a", global.getProperty("g1"));
+ assertEquals("b", global.getProperty("g2"));
+ assertEquals(2, global.getFiles().size());
+
+ Configuration simple = orig.getComponent("simple").getConfiguration();
+ assertEquals(0, simple.getProperties().size());
+ assertEquals(1, simple.getFiles().size());
+
+ Configuration master = orig.getComponent("master").getConfiguration();
+ assertEquals("m", master.getProperty("name"));
+ assertEquals("overridden", master.getProperty("g1"));
+ assertEquals(0, master.getFiles().size());
+
+ Configuration worker = orig.getComponent("worker").getConfiguration();
+ LOG.info("worker = {}", worker);
+ assertEquals(3, worker.getProperties().size());
+ assertEquals(0, worker.getFiles().size());
+
+ assertEquals("worker", worker.getProperty("name"));
+ assertEquals("overridden-by-worker", worker.getProperty("g1"));
+ assertNull(worker.getProperty("g2"));
+ assertEquals("1000", worker.getProperty("timeout"));
+
+ // here is the resolution
+ SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ replay(sfs, mockFs);
+ ServiceApiUtil.validateAndResolveService(orig, sfs, new
+ YarnConfiguration());
+
+ global = orig.getConfiguration();
+ LOG.info("global = {}", global);
+ assertEquals("a", global.getProperty("g1"));
+ assertEquals("b", global.getProperty("g2"));
+ assertEquals(2, global.getFiles().size());
+
+ simple = orig.getComponent("simple").getConfiguration();
+ assertEquals(2, simple.getProperties().size());
+ assertEquals("a", simple.getProperty("g1"));
+ assertEquals("b", simple.getProperty("g2"));
+ assertEquals(2, simple.getFiles().size());
+
+ Set<ConfigFile> files = new HashSet<>();
+ Map<String, String> props = new HashMap<>();
+ props.put("k1", "overridden");
+ props.put("k2", "v2");
+ files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
+ .PROPERTIES).props(props));
+ files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
+ .XML).props(Collections.singletonMap("k3", "v3")));
+ assertTrue(files.contains(simple.getFiles().get(0)));
+ assertTrue(files.contains(simple.getFiles().get(1)));
+
+ master = orig.getComponent("master").getConfiguration();
+ LOG.info("master = {}", master);
+ assertEquals(3, master.getProperties().size());
+ assertEquals("m", master.getProperty("name"));
+ assertEquals("overridden", master.getProperty("g1"));
+ assertEquals("b", master.getProperty("g2"));
+ assertEquals(2, master.getFiles().size());
+
+ props.put("k1", "v1");
+ files.clear();
+ files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
+ .PROPERTIES).props(props));
+ files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
+ .XML).props(Collections.singletonMap("k3", "v3")));
+
+ assertTrue(files.contains(master.getFiles().get(0)));
+ assertTrue(files.contains(master.getFiles().get(1)));
+
+ worker = orig.getComponent("worker").getConfiguration();
+ LOG.info("worker = {}", worker);
+ assertEquals(4, worker.getProperties().size());
+
+ assertEquals("worker", worker.getProperty("name"));
+ assertEquals("overridden-by-worker", worker.getProperty("g1"));
+ assertEquals("b", worker.getProperty("g2"));
+ assertEquals("1000", worker.getProperty("timeout"));
+ assertEquals(2, worker.getFiles().size());
+
+ assertTrue(files.contains(worker.getFiles().get(0)));
+ assertTrue(files.contains(worker.getFiles().get(1)));
+ }
+
+ @Test
+ public void testOverrideExternalConfiguration() throws IOException {
+ Service orig = ExampleAppJson.loadResource(EXTERNAL_JSON_1);
+
+ Configuration global = orig.getConfiguration();
+ assertEquals(0, global.getProperties().size());
+
+ assertEquals(3, orig.getComponents().size());
+
+ Configuration simple = orig.getComponent("simple").getConfiguration();
+ assertEquals(0, simple.getProperties().size());
+
+ Configuration master = orig.getComponent("master").getConfiguration();
+ assertEquals(1, master.getProperties().size());
+ assertEquals("is-overridden", master.getProperty("g3"));
+
+ Configuration other = orig.getComponent("other").getConfiguration();
+ assertEquals(0, other.getProperties().size());
+
+ // load the external service
+ SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ replay(sfs, mockFs);
+ Service ext = ExampleAppJson.loadResource(APP_JSON);
+ ServiceApiUtil.validateAndResolveService(ext, sfs, new
+ YarnConfiguration());
+ reset(sfs, mockFs);
+
+ // perform the resolution on original service
+ JsonSerDeser<Service> jsonSerDeser = createNiceMock(JsonSerDeser
+ .class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
+ .anyTimes();
+ replay(sfs, mockFs, jsonSerDeser);
+ ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
+ ServiceApiUtil.validateAndResolveService(orig, sfs, new
+ YarnConfiguration());
+
+ global = orig.getConfiguration();
+ assertEquals(0, global.getProperties().size());
+
+ assertEquals(4, orig.getComponents().size());
+
+ simple = orig.getComponent("simple").getConfiguration();
+ assertEquals(3, simple.getProperties().size());
+ assertEquals("a", simple.getProperty("g1"));
+ assertEquals("b", simple.getProperty("g2"));
+ assertEquals("60",
+ simple.getProperty("yarn.service.failure-count-reset.window"));
+
+ master = orig.getComponent("master").getConfiguration();
+ assertEquals(5, master.getProperties().size());
+ assertEquals("512M", master.getProperty("jvm.heapsize"));
+ assertEquals("overridden", master.getProperty("g1"));
+ assertEquals("b", master.getProperty("g2"));
+ assertEquals("is-overridden", master.getProperty("g3"));
+ assertEquals("60",
+ simple.getProperty("yarn.service.failure-count-reset.window"));
+
+ Configuration worker = orig.getComponent("worker").getConfiguration();
+ LOG.info("worker = {}", worker);
+ assertEquals(4, worker.getProperties().size());
+ assertEquals("512M", worker.getProperty("jvm.heapsize"));
+ assertEquals("overridden-by-worker", worker.getProperty("g1"));
+ assertEquals("b", worker.getProperty("g2"));
+ assertEquals("60",
+ worker.getProperty("yarn.service.failure-count-reset.window"));
+
+ other = orig.getComponent("other").getConfiguration();
+ assertEquals(0, other.getProperties().size());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
new file mode 100644
index 0000000..83e9502
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
+import static org.easymock.EasyMock.*;
+
+/**
+ * Test loading example resources.
+ */
+@RunWith(value = Parameterized.class)
+public class TestLoadExampleAppJson extends Assert {
+ private String resource;
+
+ public TestLoadExampleAppJson(String resource) {
+ this.resource = resource;
+ }
+
+ @Parameterized.Parameters
+ public static Collection<String[]> filenames() {
+ String[][] stringArray = new String[ExampleAppJson
+ .ALL_EXAMPLE_RESOURCES.size()][1];
+ int i = 0;
+ for (String s : ExampleAppJson.ALL_EXAMPLE_RESOURCES) {
+ stringArray[i++][0] = s;
+ }
+ return Arrays.asList(stringArray);
+ }
+
+ @Test
+ public void testLoadResource() throws Throwable {
+ try {
+ Service service = JSON_SER_DESER.fromResource(resource);
+
+ SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ replay(sfs, mockFs);
+
+ ServiceApiUtil.validateAndResolveService(service, sfs,
+ new YarnConfiguration());
+ } catch (Exception e) {
+ throw new Exception("exception loading " + resource + ":" + e.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
new file mode 100644
index 0000000..6159215
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Test cluster name validation.
+ */
+public class TestValidateServiceNames {
+
+ void assertValidName(String name) {
+ ServiceApiUtil.validateNameFormat(name, new Configuration());
+ }
+
+ void assertInvalidName(String name) {
+ try {
+ ServiceApiUtil.validateNameFormat(name, new Configuration());
+ Assert.fail();
+ } catch (IllegalArgumentException e) {
+ //
+ }
+ }
+
+ void assertInvalid(List<String> names) {
+ for (String name : names) {
+ assertInvalidName(name);
+ }
+ }
+
+ void assertValid(List<String> names) {
+ for (String name : names) {
+ assertValidName(name);
+ }
+ }
+
+ @Test
+ public void testEmptyName() throws Throwable {
+ assertInvalidName("");
+ }
+
+ @Test
+ public void testSpaceName() throws Throwable {
+ assertInvalidName(" ");
+ }
+
+
+ @Test
+ public void testLeadingHyphen() throws Throwable {
+ assertInvalidName("-hyphen");
+ }
+
+ @Test
+ public void testTitleLetters() throws Throwable {
+ assertInvalidName("Title");
+ }
+
+ @Test
+ public void testCapitalLetters() throws Throwable {
+ assertInvalidName("UPPER-CASE-CLUSTER");
+ }
+
+ @Test
+ public void testInnerBraced() throws Throwable {
+ assertInvalidName("a[a");
+ }
+
+ @Test
+ public void testLeadingBrace() throws Throwable {
+ assertInvalidName("[");
+ }
+
+ @Test
+ public void testNonalphaLeadingChars() throws Throwable {
+ assertInvalid(Arrays.asList(
+ "[a", "#", "@", "=", "*", "."
+ ));
+ }
+
+ @Test
+ public void testNonalphaInnerChars() throws Throwable {
+ assertInvalid(Arrays.asList(
+ "a[a", "b#", "c@", "d=", "e*", "f.", "g ", "h i"
+ ));
+ }
+
+ @Test
+ public void testClusterValid() throws Throwable {
+ assertValidName("cluster");
+ }
+
+ @Test
+ public void testValidNames() throws Throwable {
+ assertValid(Arrays.asList(
+ "cluster",
+ "cluster1",
+ "very-very-very-long-cluster-name",
+ "c1234567890"
+ ));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java
new file mode 100644
index 0000000..0e03a2c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.yarn.service.monitor;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.MockServiceAM;
+import org.apache.hadoop.yarn.service.ServiceTestUtils;
+
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+
+public class TestServiceMonitor extends ServiceTestUtils {
+
+ private File basedir;
+ YarnConfiguration conf = new YarnConfiguration();
+
+ @Before
+ public void setup() throws Exception {
+ basedir = new File("target", "apps");
+ if (basedir.exists()) {
+ FileUtils.deleteDirectory(basedir);
+ } else {
+ basedir.mkdirs();
+ }
+ conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 2);
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (basedir != null) {
+ FileUtils.deleteDirectory(basedir);
+ }
+ }
+
+ // Create compa with 1 container
+ // Create compb with 1 container
+ // Verify compb dependency satisfied
+ // Increase compa to 2 containers
+ // Verify compb dependency becomes unsatisfied.
+ @Test
+ public void testComponentDependency() throws Exception{
+ ApplicationId applicationId = ApplicationId.newInstance(123456, 1);
+ Service exampleApp = new Service();
+ exampleApp.setId(applicationId.toString());
+ exampleApp.setName("testComponentDependency");
+ exampleApp.addComponent(createComponent("compa", 1, "sleep 1000"));
+ Component compb = createComponent("compb", 1, "sleep 1000");
+
+ // Let compb depends on compa;
+ compb.setDependencies(Collections.singletonList("compa"));
+ exampleApp.addComponent(compb);
+
+ MockServiceAM am = new MockServiceAM(exampleApp);
+ am.init(conf);
+ am.start();
+
+ // compa ready
+ Assert.assertTrue(am.getComponent("compa").areDependenciesReady());
+ //compb not ready
+ Assert.assertFalse(am.getComponent("compb").areDependenciesReady());
+
+ // feed 1 container to compa,
+ am.feedContainerToComp(exampleApp, 1, "compa");
+ // waiting for compb's dependencies are satisfied
+ am.waitForDependenciesSatisfied("compb");
+
+ // feed 1 container to compb
+ am.feedContainerToComp(exampleApp, 2, "compb");
+ am.flexComponent("compa", 2);
+ am.waitForNumDesiredContainers("compa", 2);
+
+ // compb dependencies not satisfied again.
+ Assert.assertFalse(am.getComponent("compb").areDependenciesReady());
+ am.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
new file mode 100644
index 0000000..5b24a1d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.providers;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.easymock.EasyMock.*;
+
+/**
+ * Test the AbstractClientProvider shared methods.
+ */
+public class TestAbstractClientProvider {
+ private static final String EXCEPTION_PREFIX = "Should have thrown " +
+ "exception: ";
+ private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
+ "exception: ";
+
+ private static class ClientProvider extends AbstractClientProvider {
+ @Override
+ public void validateArtifact(Artifact artifact, FileSystem fileSystem)
+ throws IOException {
+ }
+
+ @Override
+ protected void validateConfigFile(ConfigFile configFile,
+ FileSystem fileSystem) throws IOException {
+ }
+ }
+
+ @Test
+ public void testConfigFiles() throws IOException {
+ ClientProvider clientProvider = new ClientProvider();
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes();
+ replay(mockFs);
+
+ ConfigFile configFile = new ConfigFile();
+ List<ConfigFile> configFiles = new ArrayList<>();
+ configFiles.add(configFile);
+
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "null file type");
+ } catch (IllegalArgumentException e) {
+ }
+
+ configFile.setType(ConfigFile.TypeEnum.TEMPLATE);
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "empty src_file for type template");
+ } catch (IllegalArgumentException e) {
+ }
+
+ configFile.setSrcFile("srcfile");
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "empty dest file");
+ } catch (IllegalArgumentException e) {
+ }
+
+ configFile.setDestFile("destfile");
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ configFile = new ConfigFile();
+ configFile.setType(ConfigFile.TypeEnum.JSON);
+ configFile.setSrcFile(null);
+ configFile.setDestFile("path/destfile2");
+ configFiles.add(configFile);
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "dest file with multiple path elements");
+ } catch (IllegalArgumentException e) {
+ }
+
+ configFile.setDestFile("/path/destfile2");
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ configFile.setDestFile("destfile");
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "duplicate dest file");
+ } catch (IllegalArgumentException e) {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
new file mode 100644
index 0000000..56f4555
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.providers;
+
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Artifact.TypeEnum;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultClientProvider;
+import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
+import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderService;
+import org.apache.hadoop.yarn.service.provider.docker.DockerClientProvider;
+import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
+import org.apache.hadoop.yarn.service.provider.docker.DockerProviderService;
+import org.apache.hadoop.yarn.service.provider.tarball.TarballClientProvider;
+import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
+import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderService;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test provider factories.
+ */
+public class TestProviderFactory {
+ @Test
+ public void testDockerFactory() throws Throwable {
+ ProviderFactory factory = ProviderFactory
+ .createServiceProviderFactory(new Artifact().type(TypeEnum.DOCKER));
+ assertTrue(factory instanceof DockerProviderFactory);
+ assertTrue(factory.createClientProvider() instanceof DockerClientProvider);
+ assertTrue(factory.createServerProvider() instanceof DockerProviderService);
+ assertTrue(ProviderFactory.getProviderService(new Artifact()
+ .type(TypeEnum.DOCKER)) instanceof DockerProviderService);
+ }
+
+ @Test
+ public void testTarballFactory() throws Throwable {
+ ProviderFactory factory = ProviderFactory
+ .createServiceProviderFactory(new Artifact().type(TypeEnum.TARBALL));
+ assertTrue(factory instanceof TarballProviderFactory);
+ assertTrue(factory.createClientProvider() instanceof TarballClientProvider);
+ assertTrue(factory.createServerProvider() instanceof
+ TarballProviderService);
+ assertTrue(ProviderFactory.getProviderService(new Artifact()
+ .type(TypeEnum.TARBALL)) instanceof TarballProviderService);
+ }
+
+ @Test
+ public void testDefaultFactory() throws Throwable {
+ ProviderFactory factory = ProviderFactory
+ .createServiceProviderFactory(null);
+ assertTrue(factory instanceof DefaultProviderFactory);
+ assertTrue(factory.createClientProvider() instanceof DefaultClientProvider);
+ assertTrue(factory.createServerProvider() instanceof DefaultProviderService);
+ assertTrue(ProviderFactory.getProviderService(null) instanceof
+ DefaultProviderService);
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[61/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
new file mode 100644
index 0000000..7b22e3e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Support for marshalling objects to and from JSON.
+ * This class is NOT thread safe; it constructs an object mapper
+ * as an instance field.
+ * @param <T>
+ */
+public class JsonSerDeser<T> {
+
+ private static final Logger log = LoggerFactory.getLogger(JsonSerDeser.class);
+ private static final String UTF_8 = "UTF-8";
+
+ private final Class<T> classType;
+ private final ObjectMapper mapper;
+
+ /**
+ * Create an instance bound to a specific type
+ * @param classType class type
+ */
+ public JsonSerDeser(Class<T> classType) {
+ this.classType = classType;
+ this.mapper = new ObjectMapper();
+ mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ public JsonSerDeser(Class<T> classType, PropertyNamingStrategy namingStrategy) {
+ this(classType);
+ mapper.setPropertyNamingStrategy(namingStrategy);
+ }
+
+ /**
+ * Convert from JSON
+ * @param json input
+ * @return the parsed JSON
+ * @throws IOException IO
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public T fromJson(String json)
+ throws IOException, JsonParseException, JsonMappingException {
+ try {
+ return mapper.readValue(json, classType);
+ } catch (IOException e) {
+ log.error("Exception while parsing json : " + e + "\n" + json, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Convert from a JSON file
+ * @param jsonFile input file
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public T fromFile(File jsonFile)
+ throws IOException, JsonParseException, JsonMappingException {
+ File absoluteFile = jsonFile.getAbsoluteFile();
+ try {
+ return mapper.readValue(absoluteFile, classType);
+ } catch (IOException e) {
+ log.error("Exception while parsing json file {}", absoluteFile, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Convert from a JSON file
+ * @param resource input file
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public T fromResource(String resource)
+ throws IOException, JsonParseException, JsonMappingException {
+ try(InputStream resStream = this.getClass().getResourceAsStream(resource)) {
+ if (resStream == null) {
+ throw new FileNotFoundException(resource);
+ }
+ return (T) (mapper.readValue(resStream, classType));
+ } catch (IOException e) {
+ log.error("Exception while parsing json resource {}", resource, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Convert from an input stream, closing the stream afterwards.
+ * @param stream
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ */
+ public T fromStream(InputStream stream) throws IOException {
+ try {
+ return (T) (mapper.readValue(stream, classType));
+ } catch (IOException e) {
+ log.error("Exception while parsing json input stream", e);
+ throw e;
+ } finally {
+ IOUtils.closeStream(stream);
+ }
+ }
+
+ /**
+ * clone by converting to JSON and back again.
+ * This is much less efficient than any Java clone process.
+ * @param instance instance to duplicate
+ * @return a new instance
+ * @throws IOException problems.
+ */
+ public T fromInstance(T instance) throws IOException {
+ return fromJson(toJson(instance));
+ }
+
+ /**
+ * Deserialize from a byte array
+ * @param b
+ * @return the deserialized value
+ * @throws IOException parse problems
+ */
+ public T fromBytes(byte[] b) throws IOException {
+ String json = new String(b, 0, b.length, UTF_8);
+ return fromJson(json);
+ }
+
+ /**
+ * Load from a Hadoop filesystem
+ * @param fs filesystem
+ * @param path path
+ * @return a loaded CD
+ * @throws IOException IO problems
+ * @throws JsonParseException parse problems
+ * @throws JsonMappingException O/J mapping problems
+ */
+ public T load(FileSystem fs, Path path)
+ throws IOException, JsonParseException, JsonMappingException {
+ FileStatus status = fs.getFileStatus(path);
+ long len = status.getLen();
+ byte[] b = new byte[(int) len];
+ FSDataInputStream dataInputStream = fs.open(path);
+ int count = dataInputStream.read(b);
+ if (count != len) {
+ throw new EOFException("Read of " + path +" finished prematurely");
+ }
+ return fromBytes(b);
+ }
+
+
+ /**
+ * Save to a hadoop filesystem
+ * @param fs filesystem
+ * @param path path
+ * @param instance instance to save
+ * @param overwrite should any existing file be overwritten
+ * @throws IOException IO exception
+ */
+ public void save(FileSystem fs, Path path, T instance,
+ boolean overwrite) throws
+ IOException {
+ FSDataOutputStream dataOutputStream = fs.create(path, overwrite);
+ writeJsonAsBytes(instance, dataOutputStream);
+ }
+
+ /**
+ * Save an instance to a file
+ * @param instance instance to save
+ * @param file file
+ * @throws IOException
+ */
+ public void save(T instance, File file) throws
+ IOException {
+ writeJsonAsBytes(instance, new FileOutputStream(file.getAbsoluteFile()));
+ }
+
+ /**
+ * Write the json as bytes -then close the file
+ * @param dataOutputStream an outout stream that will always be closed
+ * @throws IOException on any failure
+ */
+ private void writeJsonAsBytes(T instance,
+ OutputStream dataOutputStream) throws IOException {
+ try {
+ String json = toJson(instance);
+ byte[] b = json.getBytes(UTF_8);
+ dataOutputStream.write(b);
+ dataOutputStream.flush();
+ dataOutputStream.close();
+ } finally {
+ IOUtils.closeStream(dataOutputStream);
+ }
+ }
+
+ /**
+ * Convert an object to a JSON string
+ * @param instance instance to convert
+ * @return a JSON string description
+ * @throws JsonParseException parse problems
+ * @throws JsonMappingException O/J mapping problems
+ */
+ public String toJson(T instance) throws IOException,
+ JsonGenerationException,
+ JsonMappingException {
+ mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
+ return mapper.writeValueAsString(instance);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
new file mode 100644
index 0000000..c0712c3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
@@ -0,0 +1,680 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SaslPropertiesResolver;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.crypto.Cipher;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.lang.reflect.InvocationTargetException;
+import java.net.InetAddress;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.security.UserGroupInformation.*;
+import static org.apache.hadoop.security.authentication.util.KerberosUtil.*;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
+
+/**
+ * Kerberos diagnostics
+ * At some point this may move to hadoop core, so please keep use of slider
+ * methods and classes to ~0.
+ *
+ * This operation expands some of the diagnostic output of the security code,
+ * but not all. For completeness
+ *
+ * Set the environment variable {@code HADOOP_JAAS_DEBUG=true}
+ * Set the log level for {@code org.apache.hadoop.security=DEBUG}
+ */
+public class KerberosDiags implements Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KerberosDiags.class);
+ public static final String KRB5_CCNAME = "KRB5CCNAME";
+ public static final String JAVA_SECURITY_KRB5_CONF
+ = "java.security.krb5.conf";
+ public static final String JAVA_SECURITY_KRB5_REALM
+ = "java.security.krb5.realm";
+ public static final String SUN_SECURITY_KRB5_DEBUG
+ = "sun.security.krb5.debug";
+ public static final String SUN_SECURITY_SPNEGO_DEBUG
+ = "sun.security.spnego.debug";
+ public static final String SUN_SECURITY_JAAS_FILE
+ = "java.security.auth.login.config";
+ public static final String KERBEROS_KINIT_COMMAND
+ = "hadoop.kerberos.kinit.command";
+ public static final String HADOOP_AUTHENTICATION_IS_DISABLED
+ = "Hadoop authentication is disabled";
+ public static final String UNSET = "(unset)";
+ public static final String NO_DEFAULT_REALM = "Cannot locate default realm";
+
+ private final Configuration conf;
+ private final List<String> services;
+ private final PrintStream out;
+ private final File keytab;
+ private final String principal;
+ private final long minKeyLength;
+ private final boolean securityRequired;
+
+ public static final String CAT_JVM = "JVM";
+ public static final String CAT_JAAS = "JAAS";
+ public static final String CAT_CONFIG = "CONFIG";
+ public static final String CAT_LOGIN = "LOGIN";
+ public static final String CAT_KERBEROS = "KERBEROS";
+ public static final String CAT_SASL = "SASL";
+
+ @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
+ public KerberosDiags(Configuration conf,
+ PrintStream out,
+ List<String> services,
+ File keytab,
+ String principal,
+ long minKeyLength,
+ boolean securityRequired) {
+ this.conf = conf;
+ this.services = services;
+ this.keytab = keytab;
+ this.principal = principal;
+ this.out = out;
+ this.minKeyLength = minKeyLength;
+ this.securityRequired = securityRequired;
+ }
+
+ @Override
+ public void close() throws IOException {
+ flush();
+ }
+
+ /**
+ * Execute diagnostics.
+ * <p>
+ * Things it would be nice if UGI made accessible
+ * <ol>
+ * <li>A way to enable JAAS debug programatically</li>
+ * <li>Access to the TGT</li>
+ * </ol>
+ * @return true if security was enabled and all probes were successful
+ * @throws KerberosDiagsFailure explicitly raised failure
+ * @throws Exception other security problems
+ */
+ @SuppressWarnings("deprecation")
+ public boolean execute() throws Exception {
+
+ title("Kerberos Diagnostics scan at %s",
+ new Date(System.currentTimeMillis()));
+
+ // check that the machine has a name
+ println("Hostname: %s",
+ InetAddress.getLocalHost().getCanonicalHostName());
+
+ // Fail fast on a JVM without JCE installed.
+ validateKeyLength();
+
+ // look at realm
+ println("JVM Kerberos Login Module = %s", getKrb5LoginModuleName());
+ printDefaultRealm();
+
+ title("System Properties");
+ for (String prop : new String[]{
+ JAVA_SECURITY_KRB5_CONF,
+ JAVA_SECURITY_KRB5_REALM,
+ SUN_SECURITY_KRB5_DEBUG,
+ SUN_SECURITY_SPNEGO_DEBUG,
+ SUN_SECURITY_JAAS_FILE
+ }) {
+ printSysprop(prop);
+ }
+
+ title("Environment Variables");
+ for (String env : new String[]{
+ "HADOOP_JAAS_DEBUG",
+ KRB5_CCNAME,
+ "HADOOP_USER_NAME",
+ "HADOOP_PROXY_USER",
+ HADOOP_TOKEN_FILE_LOCATION,
+ }) {
+ printEnv(env);
+ }
+
+ for (String prop : new String[]{
+ KERBEROS_KINIT_COMMAND,
+ HADOOP_SECURITY_AUTHENTICATION,
+ HADOOP_SECURITY_AUTHORIZATION,
+ "hadoop.kerberos.min.seconds.before.relogin", // not in 2.6
+ "hadoop.security.dns.interface", // not in 2.6
+ "hadoop.security.dns.nameserver", // not in 2.6
+ HADOOP_RPC_PROTECTION,
+ HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
+ HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX,
+ HADOOP_SECURITY_GROUP_MAPPING,
+ "hadoop.security.impersonation.provider.class", // not in 2.6
+ "dfs.data.transfer.protection" // HDFS
+ }) {
+ printConfOpt(prop);
+ }
+
+ // check that authentication is enabled
+ if (SecurityUtil.getAuthenticationMethod(conf)
+ .equals(AuthenticationMethod.SIMPLE)) {
+ println(HADOOP_AUTHENTICATION_IS_DISABLED);
+ failif(securityRequired, CAT_CONFIG, HADOOP_AUTHENTICATION_IS_DISABLED);
+ // no security, skip rest of test
+ return false;
+ }
+
+ validateKrb5File();
+ validateSasl(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS);
+ validateSasl("dfs.data.transfer.saslproperties.resolver.class");
+ validateKinitExecutable();
+ validateJAAS();
+ // now the big test: login, then try again
+ boolean krb5Debug = getAndSet(SUN_SECURITY_KRB5_DEBUG);
+ boolean spnegoDebug = getAndSet(SUN_SECURITY_SPNEGO_DEBUG);
+ try {
+ title("Logging in");
+
+ if (keytab != null) {
+ dumpKeytab(keytab);
+ loginFromKeytab();
+ } else {
+ UserGroupInformation loginUser = getLoginUser();
+ dumpUGI("Log in user", loginUser);
+ validateUGI("Login user", loginUser);
+ println("Ticket based login: %b", isLoginTicketBased());
+ println("Keytab based login: %b", isLoginKeytabBased());
+ }
+
+ return true;
+ } finally {
+ // restore original system properties
+ System.setProperty(SUN_SECURITY_KRB5_DEBUG,
+ Boolean.toString(krb5Debug));
+ System.setProperty(SUN_SECURITY_SPNEGO_DEBUG,
+ Boolean.toString(spnegoDebug));
+ }
+ }
+
+ /**
+ * Fail fast on a JVM without JCE installed.
+ *
+ * This is a recurrent problem
+ * (that is: it keeps creeping back with JVM updates);
+ * a fast failure is the best tactic
+ * @throws NoSuchAlgorithmException
+ */
+
+ protected void validateKeyLength() throws NoSuchAlgorithmException {
+ int aesLen = Cipher.getMaxAllowedKeyLength("AES");
+ println("Maximum AES encryption key length %d bits", aesLen);
+ failif (aesLen < minKeyLength,
+ CAT_JVM,
+ "Java Cryptography Extensions are not installed on this JVM."
+ +" Maximum supported key length %s - minimum required %d",
+ aesLen, minKeyLength);
+ }
+
+ /**
+ * Get the default realm.
+ * <p>
+ * Not having a default realm may be harmless, so is noted at info.
+ * All other invocation failures are downgraded to warn, as
+ * follow-on actions may still work.
+ * failure to invoke the method via introspection is rejected,
+ * as it's a sign of JVM compatibility issues that may have other
+ * consequences
+ */
+ protected void printDefaultRealm() {
+ try {
+ println("Default Realm = %s",
+ getDefaultRealm());
+ } catch (ClassNotFoundException
+ | IllegalAccessException
+ | NoSuchMethodException e) {
+
+ throw new KerberosDiagsFailure(CAT_JVM, e,
+ "Failed to invoke krb5.Config.getDefaultRealm: %s", e);
+ } catch (InvocationTargetException e) {
+ Throwable cause = e.getCause() != null ? e.getCause() : e;
+ if (cause.toString().contains(NO_DEFAULT_REALM)) {
+ // exception raised if there is no default realm. This is not
+ // always a problem, so downgrade to a message.
+ println("Host has no default realm");
+ LOG.debug(cause.toString(), cause);
+ } else {
+ println("Kerberos.getDefaultRealm() failed: %s\n%s",
+ cause,
+ org.apache.hadoop.util.StringUtils.stringifyException(cause));
+ }
+ }
+ }
+
+ /**
+ * Locate the krb5.conf file and dump it.
+ * No-op on windows.
+ * @throws IOException
+ */
+ private void validateKrb5File() throws IOException {
+ if (!Shell.WINDOWS) {
+ title("Locating Kerberos configuration file");
+ String krbPath = "/etc/krb5.conf";
+ String jvmKrbPath = System.getProperty(JAVA_SECURITY_KRB5_CONF);
+ if (jvmKrbPath != null) {
+ println("Setting kerberos path from sysprop %s: %s",
+ JAVA_SECURITY_KRB5_CONF, jvmKrbPath);
+ krbPath = jvmKrbPath;
+ }
+
+ String krb5name = System.getenv(KRB5_CCNAME);
+ if (krb5name != null) {
+ println("Setting kerberos path from environment variable %s: %s",
+ KRB5_CCNAME, krb5name);
+ krbPath = krb5name;
+ if (jvmKrbPath != null) {
+ println("Warning - both %s and %s were set - %s takes priority",
+ JAVA_SECURITY_KRB5_CONF, KRB5_CCNAME, KRB5_CCNAME);
+ }
+ }
+
+ File krbFile = new File(krbPath);
+ println("Kerberos configuration file = %s", krbFile);
+ failif(!krbFile.exists(),
+ CAT_KERBEROS,
+ "Kerberos configuration file %s not found", krbFile);
+ dump(krbFile);
+ }
+ }
+
+ /**
+ * Dump a keytab: list all principals.
+ * @param keytabFile the keytab file
+ * @throws IOException IO problems
+ */
+ public void dumpKeytab(File keytabFile) throws IOException {
+ title("Examining keytab %s", keytabFile);
+ File kt = keytabFile.getCanonicalFile();
+ failif(!kt.exists(), CAT_CONFIG, "Keytab not found: %s", kt);
+ failif(!kt.isFile(), CAT_CONFIG, "Keytab is not a valid file: %s", kt);
+
+ String[] names = getPrincipalNames(keytabFile.getCanonicalPath(),
+ Pattern.compile(".*"));
+ println("keytab entry count: %d", names.length);
+ for (String name : names) {
+ println(" %s", name);
+ }
+ println("-----");
+ }
+
+ /**
+ * Log in from a keytab, dump the UGI, validate it, then try and log in again.
+ * That second-time login catches JVM/Hadoop compatibility problems.
+ * @throws IOException
+ */
+ private void loginFromKeytab() throws IOException {
+ UserGroupInformation ugi;
+ String identity;
+ if (keytab != null) {
+ File kt = keytab.getCanonicalFile();
+ println("Using keytab %s principal %s", kt, principal);
+ identity = principal;
+
+ failif(StringUtils.isEmpty(principal), CAT_KERBEROS,
+ "No principal defined");
+ ugi = loginUserFromKeytabAndReturnUGI(principal, kt.getPath());
+ dumpUGI(identity, ugi);
+ validateUGI(principal, ugi);
+
+ title("Attempting to log in from keytab again");
+ // package scoped -hence the reason why this class must be in the
+ // hadoop.security package
+ setShouldRenewImmediatelyForTests(true);
+ // attempt a new login
+ ugi.reloginFromKeytab();
+ } else {
+ println("No keytab: logging is as current user");
+ }
+ }
+
+ /**
+ * Dump a UGI.
+ * @param title title of this section
+ * @param ugi UGI to dump
+ * @throws IOException
+ */
+ private void dumpUGI(String title, UserGroupInformation ugi)
+ throws IOException {
+ title(title);
+ println("UGI instance = %s", ugi);
+ println("Has kerberos credentials: %b", ugi.hasKerberosCredentials());
+ println("Authentication method: %s", ugi.getAuthenticationMethod());
+ println("Real Authentication method: %s",
+ ugi.getRealAuthenticationMethod());
+ title("Group names");
+ for (String name : ugi.getGroupNames()) {
+ println(name);
+ }
+ title("Credentials");
+ Credentials credentials = ugi.getCredentials();
+ List<Text> secretKeys = credentials.getAllSecretKeys();
+ title("Secret keys");
+ if (!secretKeys.isEmpty()) {
+ for (Text secret: secretKeys) {
+ println("%s", secret);
+ }
+ } else {
+ println("(none)");
+ }
+
+ dumpTokens(ugi);
+ }
+
+ /**
+ * Validate the UGI: verify it is kerberized.
+ * @param messagePrefix message in exceptions
+ * @param user user to validate
+ */
+ private void validateUGI(String messagePrefix, UserGroupInformation user) {
+ failif(!user.hasKerberosCredentials(),
+ CAT_LOGIN, "%s: No kerberos credentials for %s", messagePrefix, user);
+ failif(user.getAuthenticationMethod() == null,
+ CAT_LOGIN, "%s: Null AuthenticationMethod for %s", messagePrefix, user);
+ }
+
+ /**
+ * A cursory look at the {@code kinit} executable.
+ * If it is an absolute path: it must exist with a size > 0.
+ * If it is just a command, it has to be on the path. There's no check
+ * for that -but the PATH is printed out.
+ */
+ private void validateKinitExecutable() {
+ String kinit = conf.getTrimmed(KERBEROS_KINIT_COMMAND, "");
+ if (!kinit.isEmpty()) {
+ File kinitPath = new File(kinit);
+ println("%s = %s", KERBEROS_KINIT_COMMAND, kinitPath);
+ if (kinitPath.isAbsolute()) {
+ failif(!kinitPath.exists(), CAT_KERBEROS,
+ "%s executable does not exist: %s",
+ KERBEROS_KINIT_COMMAND, kinitPath);
+ failif(!kinitPath.isFile(), CAT_KERBEROS,
+ "%s path does not refer to a file: %s",
+ KERBEROS_KINIT_COMMAND, kinitPath);
+ failif(kinitPath.length() == 0, CAT_KERBEROS,
+ "%s file is empty: %s",
+ KERBEROS_KINIT_COMMAND, kinitPath);
+ } else {
+ println("Executable %s is relative -must be on the PATH", kinit);
+ printEnv("PATH");
+ }
+ }
+ }
+
+ /**
+ * Try to load the SASL resolver.
+ * @param saslPropsResolverKey key for the SASL resolver
+ */
+ private void validateSasl(String saslPropsResolverKey) {
+ title("Resolving SASL property %s", saslPropsResolverKey);
+ String saslPropsResolver = conf.getTrimmed(saslPropsResolverKey);
+ try {
+ Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass(
+ saslPropsResolverKey,
+ SaslPropertiesResolver.class, SaslPropertiesResolver.class);
+ println("Resolver is %s", resolverClass);
+ } catch (RuntimeException e) {
+ throw new KerberosDiagsFailure(CAT_SASL, e,
+ "Failed to load %s class %s",
+ saslPropsResolverKey, saslPropsResolver);
+ }
+ }
+
+ /**
+ * Validate any JAAS entry referenced in the {@link #SUN_SECURITY_JAAS_FILE}
+ * property.
+ */
+ private void validateJAAS() {
+ String jaasFilename = System.getProperty(SUN_SECURITY_JAAS_FILE);
+ if (jaasFilename != null) {
+ title("JAAS");
+ File jaasFile = new File(jaasFilename);
+ println("JAAS file is defined in %s: %s",
+ SUN_SECURITY_JAAS_FILE, jaasFile);
+ failif(!jaasFile.exists(), CAT_JAAS,
+ "JAAS file does not exist: %s", jaasFile);
+ failif(!jaasFile.isFile(), CAT_JAAS,
+ "Specified JAAS file is not a file: %s", jaasFile);
+ }
+ }
+
+ /**
+ * Dump all tokens of a user
+ * @param user user
+ */
+ public void dumpTokens(UserGroupInformation user) {
+ Collection<Token<? extends TokenIdentifier>> tokens
+ = user.getCredentials().getAllTokens();
+ title("Token Count: %d", tokens.size());
+ for (Token<? extends TokenIdentifier> token : tokens) {
+ println("Token %s", token.getKind());
+ }
+ }
+
+ /**
+ * Set the System property to true; return the old value for caching
+ * @param sysprop property
+ * @return the previous value
+ */
+ private boolean getAndSet(String sysprop) {
+ boolean old = Boolean.getBoolean(sysprop);
+ System.setProperty(sysprop, "true");
+ return old;
+ }
+
+ /**
+ * Flush all active output channels, including {@Code System.err},
+ * so as to stay in sync with any JRE log messages.
+ */
+ private void flush() {
+ if (out != null) {
+ out.flush();
+ } else {
+ System.out.flush();
+ }
+ System.err.flush();
+ }
+
+ /**
+ * Format and print a line of output.
+ * This goes to any output file, or
+ * is logged at info. The output is flushed before and after, to
+ * try and stay in sync with JRE logging.
+ * @param format format string
+ * @param args any arguments
+ */
+ @VisibleForTesting
+ public void println(String format, Object... args) {
+ println(format(format, args));
+ }
+
+ /**
+ * Print a line of output. This goes to any output file, or
+ * is logged at info. The output is flushed before and after, to
+ * try and stay in sync with JRE logging.
+ * @param msg message string
+ */
+ @VisibleForTesting
+ private void println(String msg) {
+ flush();
+ if (out != null) {
+ out.println(msg);
+ } else {
+ LOG.info(msg);
+ }
+ flush();
+ }
+
+ /**
+ * Print a title entry
+ * @param format format string
+ * @param args any arguments
+ */
+ private void title(String format, Object... args) {
+ println("");
+ println("");
+ String msg = "== " + format(format, args) + " ==";
+ println(msg);
+ println("");
+ }
+
+ /**
+ * Print a system property, or {@link #UNSET} if unset.
+ * @param property property to print
+ */
+ private void printSysprop(String property) {
+ println("%s = \"%s\"", property,
+ System.getProperty(property, UNSET));
+ }
+
+ /**
+ * Print a configuration option, or {@link #UNSET} if unset.
+ * @param option option to print
+ */
+ private void printConfOpt(String option) {
+ println("%s = \"%s\"", option, conf.get(option, UNSET));
+ }
+
+ /**
+ * Print an environment variable's name and value; printing
+ * {@link #UNSET} if it is not set
+ * @param variable environment variable
+ */
+ private void printEnv(String variable) {
+ String env = System.getenv(variable);
+ println("%s = \"%s\"", variable, env != null ? env : UNSET);
+ }
+
+ /**
+ * Dump any file to standard out; add a trailing newline
+ * @param file file to dump
+ * @throws IOException IO problems
+ */
+ public void dump(File file) throws IOException {
+ try (FileInputStream in = new FileInputStream(file)) {
+ for (String line : IOUtils.readLines(in)) {
+ println("%s", line);
+ }
+ }
+ println("");
+ }
+
+ /**
+ * Format and raise a failure
+ *
+ * @param category category for exception
+ * @param message string formatting message
+ * @param args any arguments for the formatting
+ * @throws KerberosDiagsFailure containing the formatted text
+ */
+ private void fail(String category, String message, Object... args)
+ throws KerberosDiagsFailure {
+ throw new KerberosDiagsFailure(category, message, args);
+ }
+
+ /**
+ * Conditional failure with string formatted arguments
+ * @param condition failure condition
+ * @param category category for exception
+ * @param message string formatting message
+ * @param args any arguments for the formatting
+ * @throws KerberosDiagsFailure containing the formatted text
+ * if the condition was met
+ */
+ private void failif(boolean condition,
+ String category,
+ String message,
+ Object... args)
+ throws KerberosDiagsFailure {
+ if (condition) {
+ fail(category, message, args);
+ }
+ }
+
+ /**
+ * Format a string, treating a call where there are no varags values
+ * as a string to pass through unformatted.
+ * @param message message, which is either a format string + args, or
+ * a general string
+ * @param args argument array
+ * @return a string for printing.
+ */
+ public static String format(String message, Object... args) {
+ if (args.length == 0) {
+ return message;
+ } else {
+ return String.format(message, args);
+ }
+ }
+
+ /**
+ * Diagnostics failures return the exit code 41, "unauthorized".
+ *
+ * They have a category, initially for testing: the category can be
+ * validated without having to match on the entire string.
+ */
+ public static class KerberosDiagsFailure extends ExitUtil.ExitException {
+ private final String category;
+
+ public KerberosDiagsFailure(String category, String message) {
+ super(41, category + ": " + message);
+ this.category = category;
+ }
+
+ public KerberosDiagsFailure(String category, String message, Object... args) {
+ this(category, format(message, args));
+ }
+
+ public KerberosDiagsFailure(String category, Throwable throwable,
+ String message, Object... args) {
+ this(category, message, args);
+ initCause(throwable);
+ }
+
+ public String getCategory() {
+ return category;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java
new file mode 100644
index 0000000..108ca22
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import java.util.regex.Pattern;
+
+/**
+ * Utility class to validate strings against a predefined pattern.
+ */
+public class PatternValidator {
+
+ public static final String E_INVALID_NAME =
+ "Invalid name %s does not match the pattern %s ";
+ private final Pattern valid;
+ private final String pattern;
+
+ public PatternValidator(String pattern) {
+ this.pattern = pattern;
+ valid = Pattern.compile(pattern);
+ }
+
+ /**
+ * Validate the name -restricting it to the set defined in
+ * @param name name to validate
+ * @throws IllegalArgumentException if not a valid name
+ */
+ public void validate(String name) {
+ if (!matches(name)) {
+ throw new IllegalArgumentException(
+ String.format(E_INVALID_NAME, name, pattern));
+ }
+ }
+
+ /**
+ * Query to see if the pattern matches
+ * @param name name to validate
+ * @return true if the string matches the pattern
+ */
+ public boolean matches(String name) {
+ return valid.matcher(name).matches();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
new file mode 100644
index 0000000..2dbf37f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * a scanner which can take an input string for a range or scan the lot.
+ */
+public class PortScanner {
+ private static Pattern NUMBER_RANGE = Pattern.compile("^(\\d+)\\s*-\\s*(\\d+)$");
+ private static Pattern SINGLE_NUMBER = Pattern.compile("^\\d+$");
+
+ private List<Integer> remainingPortsToCheck;
+
+ public PortScanner() {
+ }
+
+ public void setPortRange(String input) throws BadConfigException {
+ // first split based on commas
+ Set<Integer> inputPorts= new TreeSet<Integer>();
+ String[] ranges = input.split(",");
+ for ( String range : ranges ) {
+ if (range.trim().isEmpty()) {
+ continue;
+ }
+ Matcher m = SINGLE_NUMBER.matcher(range.trim());
+ if (m.find()) {
+ inputPorts.add(Integer.parseInt(m.group()));
+ continue;
+ }
+ m = NUMBER_RANGE.matcher(range.trim());
+ if (m.find()) {
+ String[] boundaryValues = m.group(0).split("-");
+ int start = Integer.parseInt(boundaryValues[0].trim());
+ int end = Integer.parseInt(boundaryValues[1].trim());
+ if (end < start) {
+ throw new BadConfigException("End of port range is before start: "
+ + range + " in input: " + input);
+ }
+ for (int i = start; i < end + 1; i++) {
+ inputPorts.add(i);
+ }
+ continue;
+ }
+ throw new BadConfigException("Bad port range: " + range + " in input: "
+ + input);
+ }
+ if (inputPorts.size() == 0) {
+ throw new BadConfigException("No ports found in range: " + input);
+ }
+ this.remainingPortsToCheck = new ArrayList<Integer>(inputPorts);
+ }
+
+ public List<Integer> getRemainingPortsToCheck() {
+ return remainingPortsToCheck;
+ }
+
+ public int getAvailablePort() throws SliderException, IOException {
+ if (remainingPortsToCheck != null) {
+ return getAvailablePortViaPortArray();
+ } else {
+ return SliderUtils.getOpenPort();
+ }
+ }
+
+ private int getAvailablePortViaPortArray() throws SliderException {
+ boolean found = false;
+ int availablePort = -1;
+ Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator();
+ while (portsToCheck.hasNext() && !found) {
+ int portToCheck = portsToCheck.next();
+ found = SliderUtils.isPortAvailable(portToCheck);
+ if (found) {
+ availablePort = portToCheck;
+ portsToCheck.remove();
+ }
+ }
+
+ if (availablePort < 0) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "No available ports found in configured range {}",
+ remainingPortsToCheck);
+ }
+
+ return availablePort;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java
new file mode 100644
index 0000000..9d00b3c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * JSON-serializable description of a published key-val configuration.
+ *
+ * The values themselves are not serialized in the external view; they have
+ * to be served up by the far end
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedConfiguration {
+
+ public String description;
+ public long updated;
+
+ public String updatedTime;
+
+ public Map<String, String> entries = new HashMap<>();
+
+ public PublishedConfiguration() {
+ }
+
+ /**
+ * build an empty published configuration
+ * @param description configuration description
+ */
+ public PublishedConfiguration(String description) {
+ this.description = description;
+ }
+
+ /**
+ * Build a configuration from the entries
+ * @param description configuration description
+ * @param entries entries to put
+ */
+ public PublishedConfiguration(String description,
+ Iterable<Map.Entry<String, String>> entries) {
+ this.description = description;
+ putValues(entries);
+ }
+
+ /**
+ * Build a published configuration, using the keys from keysource,
+ * but resolving the values from the value source, via Configuration.get()
+ * @param description configuration description
+ * @param keysource source of keys
+ * @param valuesource source of values
+ */
+ public PublishedConfiguration(String description,
+ Iterable<Map.Entry<String, String>> keysource,
+ Configuration valuesource) {
+ this.description = description;
+ putValues(ConfigHelper.resolveConfiguration(keysource, valuesource));
+ }
+
+
+ /**
+ * Is the configuration empty. This means either that it has not
+ * been given any values, or it is stripped down copy set down over the
+ * wire.
+ * @return true if it is empty
+ */
+ public boolean isEmpty() {
+ return entries.isEmpty();
+ }
+
+
+ public void setUpdated(long updated) {
+ this.updated = updated;
+ this.updatedTime = new Date(updated).toString();
+ }
+
+ public long getUpdated() {
+ return updated;
+ }
+
+ /**
+ * Set the values from an iterable (this includes a Hadoop Configuration
+ * and Java properties object).
+ * Any existing value set is discarded
+ * @param entries entries to put
+ */
+ public void putValues(Iterable<Map.Entry<String, String>> entries) {
+ this.entries = new HashMap<String, String>();
+ for (Map.Entry<String, String> entry : entries) {
+ this.entries.put(entry.getKey(), entry.getValue());
+ }
+
+ }
+
+ /**
+ * Convert to Hadoop XML
+ * @return the configuration as a Hadoop Configuratin
+ */
+ public Configuration asConfiguration() {
+ Configuration conf = new Configuration(false);
+ try {
+ ConfigHelper.addConfigMap(conf, entries, "");
+ } catch (BadConfigException e) {
+ // triggered on a null value; switch to a runtime (and discard the stack)
+ throw new RuntimeException(e.toString());
+ }
+ return conf;
+ }
+
+ public String asConfigurationXML() throws IOException {
+ return ConfigHelper.toXml(asConfiguration());
+ }
+
+ /**
+ * Convert values to properties
+ * @return a property file
+ */
+ public Properties asProperties() {
+ Properties props = new Properties();
+ props.putAll(entries);
+ return props;
+ }
+
+ /**
+ * Return the values as json string
+ * @return the JSON representation
+ * @throws IOException marshalling failure
+ */
+ public String asJson() throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
+ String json = mapper.writeValueAsString(entries);
+ return json;
+ }
+
+
+ /**
+ * This makes a copy without the nested content -so is suitable
+ * for returning as part of the list of a parent's values
+ * @return the copy
+ */
+ public PublishedConfiguration shallowCopy() {
+ PublishedConfiguration that = new PublishedConfiguration();
+ that.description = this.description;
+ that.updated = this.updated;
+ that.updatedTime = this.updatedTime;
+ return that;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("PublishedConfiguration{");
+ sb.append("description='").append(description).append('\'');
+ sb.append(" entries = ").append(entries.size());
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Create an outputter for a given format
+ * @param format format to use
+ * @return an instance of output
+ */
+ public PublishedConfigurationOutputter createOutputter(ConfigFormat format) {
+ return PublishedConfigurationOutputter.createOutputter(format, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
new file mode 100644
index 0000000..88ecf2c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.DumperOptions.FlowStyle;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.StringWriter;
+import java.util.Properties;
+
+/**
+ * Output a published configuration
+ */
+public abstract class PublishedConfigurationOutputter {
+
+ private static final String COMMENTS = "Generated by Apache Slider";
+
+ protected final PublishedConfiguration owner;
+
+ protected PublishedConfigurationOutputter(PublishedConfiguration owner) {
+ this.owner = owner;
+ }
+
+ /**
+ * Save the config to a destination file, in the format of this outputter
+ * @param dest destination file
+ * @throws IOException
+ */
+/* JDK7
+ public void save(File dest) throws IOException {
+ try(FileOutputStream out = new FileOutputStream(dest)) {
+ save(out);
+ out.close();
+ }
+ }
+*/
+ public void save(File dest) throws IOException {
+ FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
+ }
+
+ /**
+ * Save the content. The default saves the asString() value
+ * to the output stream
+ * @param out output stream
+ * @throws IOException
+ */
+ public void save(OutputStream out) throws IOException {
+ IOUtils.write(asString(), out, Charsets.UTF_8);
+ }
+ /**
+ * Convert to a string
+ * @return the string form
+ * @throws IOException
+ */
+ public abstract String asString() throws IOException;
+
+ /**
+ * Create an outputter for the chosen format
+ * @param format format enumeration
+ * @param owner owning config
+ * @return the outputter
+ */
+
+ public static PublishedConfigurationOutputter createOutputter(ConfigFormat format,
+ PublishedConfiguration owner) {
+ Preconditions.checkNotNull(owner);
+ switch (format) {
+ case XML:
+ case HADOOP_XML:
+ return new XmlOutputter(owner);
+ case PROPERTIES:
+ return new PropertiesOutputter(owner);
+ case JSON:
+ return new JsonOutputter(owner);
+ case ENV:
+ return new EnvOutputter(owner);
+ case TEMPLATE:
+ return new TemplateOutputter(owner);
+ case YAML:
+ return new YamlOutputter(owner);
+ default:
+ throw new RuntimeException("Unsupported format :" + format);
+ }
+ }
+
+ public static class XmlOutputter extends PublishedConfigurationOutputter {
+
+
+ private final Configuration configuration;
+
+ public XmlOutputter(PublishedConfiguration owner) {
+ super(owner);
+ configuration = owner.asConfiguration();
+ }
+
+ @Override
+ public void save(OutputStream out) throws IOException {
+ configuration.writeXml(out);
+ }
+
+ @Override
+ public String asString() throws IOException {
+ return ConfigHelper.toXml(configuration);
+ }
+
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+ }
+
+ public static class PropertiesOutputter extends PublishedConfigurationOutputter {
+
+ private final Properties properties;
+
+ public PropertiesOutputter(PublishedConfiguration owner) {
+ super(owner);
+ properties = owner.asProperties();
+ }
+
+ @Override
+ public void save(OutputStream out) throws IOException {
+ properties.store(out, COMMENTS);
+ }
+
+
+ public String asString() throws IOException {
+ StringWriter sw = new StringWriter();
+ properties.store(sw, COMMENTS);
+ return sw.toString();
+ }
+ }
+
+
+ public static class JsonOutputter extends PublishedConfigurationOutputter {
+
+ public JsonOutputter(PublishedConfiguration owner) {
+ super(owner);
+ }
+
+ @Override
+ public String asString() throws IOException {
+ return owner.asJson();
+ }
+ }
+
+
+ public static class EnvOutputter extends PublishedConfigurationOutputter {
+
+ public EnvOutputter(PublishedConfiguration owner) {
+ super(owner);
+ }
+
+ @Override
+ public String asString() throws IOException {
+ if (!owner.entries.containsKey("content")) {
+ throw new IOException("Configuration has no content field and cannot " +
+ "be retrieved as type 'env'");
+ }
+ String content = owner.entries.get("content");
+ return ConfigUtils.replaceProps(owner.entries, content);
+ }
+ }
+
+ public static class TemplateOutputter extends EnvOutputter {
+ public TemplateOutputter(PublishedConfiguration owner) {
+ super(owner);
+ }
+ }
+
+ public static class YamlOutputter extends PublishedConfigurationOutputter {
+
+ private final Yaml yaml;
+
+ public YamlOutputter(PublishedConfiguration owner) {
+ super(owner);
+ DumperOptions options = new DumperOptions();
+ options.setDefaultFlowStyle(FlowStyle.BLOCK);
+ yaml = new Yaml(options);
+ }
+
+ public String asString() throws IOException {
+ return yaml.dump(owner.entries);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
new file mode 100644
index 0000000..140204a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.service.utils.ApplicationReportSerDeser;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+
+/**
+ * Serialized form of an service report which can be persisted
+ * and then parsed. It can not be converted back into a
+ * real YARN service report
+ *
+ * Useful for testing
+ */
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+
+public class SerializedApplicationReport {
+
+ public String applicationId;
+ public String applicationAttemptId;
+ public String name;
+ public String applicationType;
+ public String user;
+ public String queue;
+ public String host;
+ public Integer rpcPort;
+ public String state;
+ public String diagnostics;
+ public String url;
+ /**
+ * This value is non-null only when a report is generated from a submission context.
+ * The YARN {@link ApplicationReport} structure does not propagate this value
+ * from the RM.
+ */
+ public Long submitTime;
+ public Long startTime;
+ public Long finishTime;
+ public String finalStatus;
+ public String origTrackingUrl;
+ public Float progress;
+
+ public SerializedApplicationReport() {
+ }
+
+ public SerializedApplicationReport(ApplicationReport report) {
+ this.applicationId = report.getApplicationId().toString();
+ ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId();
+ this.applicationAttemptId = attemptId != null ? attemptId.toString() : "N/A";
+ this.name = report.getName();
+ this.applicationType = report.getApplicationType();
+ this.user = report.getUser();
+ this.queue = report.getQueue();
+ this.host = report.getHost();
+ this.rpcPort = report.getRpcPort();
+ this.state = report.getYarnApplicationState().toString();
+ this.diagnostics = report.getDiagnostics();
+ this.startTime = report.getStartTime();
+ this.finishTime = report.getFinishTime();
+ FinalApplicationStatus appStatus = report.getFinalApplicationStatus();
+ this.finalStatus = appStatus == null ? "" : appStatus.toString();
+ this.progress = report.getProgress();
+ this.url = report.getTrackingUrl();
+ this.origTrackingUrl= report.getOriginalTrackingUrl();
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return ApplicationReportSerDeser.toString(this);
+ } catch (IOException e) {
+ return super.toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
new file mode 100644
index 0000000..de82580
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -0,0 +1,446 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils;
+import org.apache.hadoop.yarn.service.conf.RestApiConstants;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class ServiceApiUtil {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceApiUtil.class);
+ public static JsonSerDeser<Service> jsonSerDeser =
+ new JsonSerDeser<>(Service.class,
+ PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
+ private static final PatternValidator namePattern
+ = new PatternValidator("[a-z][a-z0-9-]*");
+
+ @VisibleForTesting
+ public static void setJsonSerDeser(JsonSerDeser jsd) {
+ jsonSerDeser = jsd;
+ }
+
+ @VisibleForTesting
+ public static void validateAndResolveService(Service service,
+ SliderFileSystem fs, org.apache.hadoop.conf.Configuration conf) throws
+ IOException {
+ boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED,
+ RegistryConstants.DEFAULT_DNS_ENABLED);
+ if (dnsEnabled && RegistryUtils.currentUser().length() > RegistryConstants
+ .MAX_FQDN_LABEL_LENGTH) {
+ throw new IllegalArgumentException(RestApiErrorMessages
+ .ERROR_USER_NAME_INVALID);
+ }
+ if (StringUtils.isEmpty(service.getName())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID);
+ }
+
+ validateNameFormat(service.getName(), conf);
+
+ // If the service has no components do top-level checks
+ if (!hasComponent(service)) {
+ // If artifact is of type SERVICE, read other service components
+ if (service.getArtifact() != null && service.getArtifact()
+ .getType() == Artifact.TypeEnum.SERVICE) {
+ if (StringUtils.isEmpty(service.getArtifact().getId())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+ }
+ Service otherService = loadService(fs,
+ service.getArtifact().getId());
+ service.setComponents(otherService.getComponents());
+ service.setArtifact(null);
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(service.getQuicklinks(),
+ otherService.getQuicklinks());
+ } else {
+ // Since it is a simple service with no components, create a default
+ // component
+ Component comp = createDefaultComponent(service);
+ validateComponent(comp, fs.getFileSystem(), conf);
+ service.getComponents().add(comp);
+ if (service.getLifetime() == null) {
+ service.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME);
+ }
+ return;
+ }
+ }
+
+ // Validate there are no component name collisions (collisions are not
+ // currently supported) and add any components from external services
+ // TODO allow name collisions? see AppState#roles
+ // TODO or add prefix to external component names?
+ Configuration globalConf = service.getConfiguration();
+ Set<String> componentNames = new HashSet<>();
+ List<Component> componentsToRemove = new ArrayList<>();
+ List<Component> componentsToAdd = new ArrayList<>();
+ for (Component comp : service.getComponents()) {
+ int maxCompLength = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
+ maxCompLength = maxCompLength - Long.toString(Long.MAX_VALUE).length();
+ if (dnsEnabled && comp.getName().length() > maxCompLength) {
+ throw new IllegalArgumentException(String.format(RestApiErrorMessages
+ .ERROR_COMPONENT_NAME_INVALID, maxCompLength, comp.getName()));
+ }
+ if (componentNames.contains(comp.getName())) {
+ throw new IllegalArgumentException("Component name collision: " +
+ comp.getName());
+ }
+ // If artifact is of type SERVICE (which cannot be filled from
+ // global), read external service and add its components to this
+ // service
+ if (comp.getArtifact() != null && comp.getArtifact().getType() ==
+ Artifact.TypeEnum.SERVICE) {
+ if (StringUtils.isEmpty(comp.getArtifact().getId())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+ }
+ LOG.info("Marking {} for removal", comp.getName());
+ componentsToRemove.add(comp);
+ List<Component> externalComponents = getComponents(fs,
+ comp.getArtifact().getId());
+ for (Component c : externalComponents) {
+ Component override = service.getComponent(c.getName());
+ if (override != null && override.getArtifact() == null) {
+ // allow properties from external components to be overridden /
+ // augmented by properties in this component, except for artifact
+ // which must be read from external component
+ override.mergeFrom(c);
+ LOG.info("Merging external component {} from external {}", c
+ .getName(), comp.getName());
+ } else {
+ if (componentNames.contains(c.getName())) {
+ throw new IllegalArgumentException("Component name collision: " +
+ c.getName());
+ }
+ componentNames.add(c.getName());
+ componentsToAdd.add(c);
+ LOG.info("Adding component {} from external {}", c.getName(),
+ comp.getName());
+ }
+ }
+ } else {
+ // otherwise handle as a normal component
+ componentNames.add(comp.getName());
+ // configuration
+ comp.getConfiguration().mergeFrom(globalConf);
+ }
+ }
+ service.getComponents().removeAll(componentsToRemove);
+ service.getComponents().addAll(componentsToAdd);
+
+ // Validate components and let global values take effect if component level
+ // values are not provided
+ Artifact globalArtifact = service.getArtifact();
+ Resource globalResource = service.getResource();
+ Long globalNumberOfContainers = service.getNumberOfContainers();
+ String globalLaunchCommand = service.getLaunchCommand();
+ for (Component comp : service.getComponents()) {
+ // fill in global artifact unless it is type SERVICE
+ if (comp.getArtifact() == null && service.getArtifact() != null
+ && service.getArtifact().getType() != Artifact.TypeEnum
+ .SERVICE) {
+ comp.setArtifact(globalArtifact);
+ }
+ // fill in global resource
+ if (comp.getResource() == null) {
+ comp.setResource(globalResource);
+ }
+ // fill in global container count
+ if (comp.getNumberOfContainers() == null) {
+ comp.setNumberOfContainers(globalNumberOfContainers);
+ }
+ // fill in global launch command
+ if (comp.getLaunchCommand() == null) {
+ comp.setLaunchCommand(globalLaunchCommand);
+ }
+ // validate dependency existence
+ if (comp.getDependencies() != null) {
+ for (String dependency : comp.getDependencies()) {
+ if (!componentNames.contains(dependency)) {
+ throw new IllegalArgumentException(String.format(
+ RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, dependency,
+ comp.getName()));
+ }
+ }
+ }
+ validateComponent(comp, fs.getFileSystem(), conf);
+ }
+
+ // validate dependency tree
+ sortByDependencies(service.getComponents());
+
+ // Service lifetime if not specified, is set to unlimited lifetime
+ if (service.getLifetime() == null) {
+ service.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME);
+ }
+ }
+
+ private static void validateComponent(Component comp, FileSystem fs,
+ org.apache.hadoop.conf.Configuration conf)
+ throws IOException {
+ validateNameFormat(comp.getName(), conf);
+
+ AbstractClientProvider compClientProvider = ProviderFactory
+ .getClientProvider(comp.getArtifact());
+ compClientProvider.validateArtifact(comp.getArtifact(), fs);
+
+ if (comp.getLaunchCommand() == null && (comp.getArtifact() == null || comp
+ .getArtifact().getType() != Artifact.TypeEnum.DOCKER)) {
+ throw new IllegalArgumentException(RestApiErrorMessages
+ .ERROR_ABSENT_LAUNCH_COMMAND);
+ }
+
+ validateServiceResource(comp.getResource(), comp);
+
+ if (comp.getNumberOfContainers() == null
+ || comp.getNumberOfContainers() < 0) {
+ throw new IllegalArgumentException(String.format(
+ RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID
+ + ": " + comp.getNumberOfContainers(), comp.getName()));
+ }
+ compClientProvider.validateConfigFiles(comp.getConfiguration()
+ .getFiles(), fs);
+
+ MonitorUtils.getProbe(comp.getReadinessCheck());
+ }
+
+ // Check component or service name format and transform to lower case.
+ public static void validateNameFormat(String name,
+ org.apache.hadoop.conf.Configuration conf) {
+ if (StringUtils.isEmpty(name)) {
+ throw new IllegalArgumentException("Name can not be empty!");
+ }
+ // validate component name
+ if (name.contains("_")) {
+ throw new IllegalArgumentException(
+ "Invalid format: " + name
+ + ", can not use '_', as DNS hostname does not allow '_'. Use '-' Instead. ");
+ }
+ boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED,
+ RegistryConstants.DEFAULT_DNS_ENABLED);
+ if (dnsEnabled && name.length() > RegistryConstants.MAX_FQDN_LABEL_LENGTH) {
+ throw new IllegalArgumentException(String
+ .format("Invalid format %s, must be no more than 63 characters ",
+ name));
+ }
+ namePattern.validate(name);
+ }
+
+ @VisibleForTesting
+ public static List<Component> getComponents(SliderFileSystem
+ fs, String serviceName) throws IOException {
+ return loadService(fs, serviceName).getComponents();
+ }
+
+ public static Service loadService(SliderFileSystem fs, String
+ serviceName) throws IOException {
+ Path serviceJson = getServiceJsonPath(fs, serviceName);
+ LOG.info("Loading service definition from " + serviceJson);
+ return jsonSerDeser.load(fs.getFileSystem(), serviceJson);
+ }
+
+ public static Service loadServiceFrom(SliderFileSystem fs,
+ Path appDefPath) throws IOException {
+ LOG.info("Loading service definition from " + appDefPath);
+ return jsonSerDeser.load(fs.getFileSystem(), appDefPath);
+ }
+
+ public static Path getServiceJsonPath(SliderFileSystem fs, String serviceName) {
+ Path serviceDir = fs.buildClusterDirPath(serviceName);
+ return new Path(serviceDir, serviceName + ".json");
+ }
+
+ private static void validateServiceResource(Resource resource,
+ Component comp) {
+ // Only services/components of type SERVICE can skip resource requirement
+ if (resource == null) {
+ throw new IllegalArgumentException(
+ comp == null ? RestApiErrorMessages.ERROR_RESOURCE_INVALID : String
+ .format(RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID,
+ comp.getName()));
+ }
+ // One and only one of profile OR cpus & memory can be specified. Specifying
+ // both raises validation error.
+ if (StringUtils.isNotEmpty(resource.getProfile()) && (
+ resource.getCpus() != null || StringUtils
+ .isNotEmpty(resource.getMemory()))) {
+ throw new IllegalArgumentException(comp == null ?
+ RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED :
+ String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
+ comp.getName()));
+ }
+ // Currently resource profile is not supported yet, so we will raise
+ // validation error if only resource profile is specified
+ if (StringUtils.isNotEmpty(resource.getProfile())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET);
+ }
+
+ String memory = resource.getMemory();
+ Integer cpus = resource.getCpus();
+ if (StringUtils.isEmpty(memory)) {
+ throw new IllegalArgumentException(
+ comp == null ? RestApiErrorMessages.ERROR_RESOURCE_MEMORY_INVALID :
+ String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID,
+ comp.getName()));
+ }
+ if (cpus == null) {
+ throw new IllegalArgumentException(
+ comp == null ? RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID :
+ String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID,
+ comp.getName()));
+ }
+ if (cpus <= 0) {
+ throw new IllegalArgumentException(comp == null ?
+ RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID_RANGE : String
+ .format(
+ RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE,
+ comp.getName()));
+ }
+ }
+
+ // check if comp mem size exceeds cluster limit
+ public static void validateCompResourceSize(
+ org.apache.hadoop.yarn.api.records.Resource maxResource,
+ Service service) throws YarnException {
+ for (Component component : service.getComponents()) {
+ // only handle mem now.
+ long mem = Long.parseLong(component.getResource().getMemory());
+ if (mem > maxResource.getMemorySize()) {
+ throw new YarnException(
+ "Component " + component.getName() + " memory size (" + mem
+ + ") is larger than configured max container memory size ("
+ + maxResource.getMemorySize() + ")");
+ }
+ }
+ }
+
+ public static boolean hasComponent(Service service) {
+ if (service.getComponents() == null || service.getComponents()
+ .isEmpty()) {
+ return false;
+ }
+ return true;
+ }
+
+ public static Component createDefaultComponent(Service service) {
+ Component comp = new Component();
+ comp.setName(RestApiConstants.DEFAULT_COMPONENT_NAME);
+ comp.setArtifact(service.getArtifact());
+ comp.setResource(service.getResource());
+ comp.setNumberOfContainers(service.getNumberOfContainers());
+ comp.setLaunchCommand(service.getLaunchCommand());
+ comp.setConfiguration(service.getConfiguration());
+ return comp;
+ }
+
+ public static Collection<Component> sortByDependencies(List<Component>
+ components) {
+ Map<String, Component> sortedComponents =
+ sortByDependencies(components, null);
+ return sortedComponents.values();
+ }
+
+ /**
+ * Each internal call of sortByDependencies will identify all of the
+ * components with the same dependency depth (the lowest depth that has not
+ * been processed yet) and add them to the sortedComponents list, preserving
+ * their original ordering in the components list.
+ *
+ * So the first time it is called, all components with no dependencies
+ * (depth 0) will be identified. The next time it is called, all components
+ * that have dependencies only on the the depth 0 components will be
+ * identified (depth 1). This will be repeated until all components have
+ * been added to the sortedComponents list. If no new components are
+ * identified but the sortedComponents list is not complete, an error is
+ * thrown.
+ */
+ private static Map<String, Component> sortByDependencies(List<Component>
+ components, Map<String, Component> sortedComponents) {
+ if (sortedComponents == null) {
+ sortedComponents = new LinkedHashMap<>();
+ }
+
+ Map<String, Component> componentsToAdd = new LinkedHashMap<>();
+ List<Component> componentsSkipped = new ArrayList<>();
+ for (Component component : components) {
+ String name = component.getName();
+ if (sortedComponents.containsKey(name)) {
+ continue;
+ }
+ boolean dependenciesAlreadySorted = true;
+ if (!SliderUtils.isEmpty(component.getDependencies())) {
+ for (String dependency : component.getDependencies()) {
+ if (!sortedComponents.containsKey(dependency)) {
+ dependenciesAlreadySorted = false;
+ break;
+ }
+ }
+ }
+ if (dependenciesAlreadySorted) {
+ componentsToAdd.put(name, component);
+ } else {
+ componentsSkipped.add(component);
+ }
+ }
+
+ if (componentsToAdd.size() == 0) {
+ throw new IllegalArgumentException(String.format(RestApiErrorMessages
+ .ERROR_DEPENDENCY_CYCLE, componentsSkipped));
+ }
+ sortedComponents.putAll(componentsToAdd);
+ if (sortedComponents.size() == components.size()) {
+ return sortedComponents;
+ }
+ return sortByDependencies(components, sortedComponents);
+ }
+
+ public static String $(String s) {
+ return "${" + s +"}";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java
new file mode 100644
index 0000000..7440b11
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+
+
+public class ServiceRegistryUtils {
+
+ /**
+ * Base path for services
+ */
+ public static final String ZK_SERVICES = "services";
+
+ /**
+ * Base path for all Slider references
+ */
+ public static final String ZK_SLIDER = "slider";
+ public static final String ZK_USERS = "users";
+ public static final String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
+ public static final String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
+
+ /**
+ * Get the registry path for an instance under the user's home node
+ * @param instanceName application instance
+ * @return a path to the registry location for this application instance.
+ */
+ public static String registryPathForInstance(String instanceName) {
+ return RegistryUtils.servicePath(
+ RegistryUtils.currentUser(), YarnServiceConstants.APP_TYPE, instanceName
+ );
+ }
+
+ /**
+ * Build the path to a cluster; exists once the cluster has come up.
+ * Even before that, a ZK watcher could wait for it.
+ * @param username user
+ * @param clustername name of the cluster
+ * @return a strin
+ */
+ public static String mkClusterPath(String username, String clustername) {
+ return mkSliderUserPath(username) + "/" + clustername;
+ }
+
+ /**
+ * Build the path to a cluster; exists once the cluster has come up.
+ * Even before that, a ZK watcher could wait for it.
+ * @param username user
+ * @return a string
+ */
+ public static String mkSliderUserPath(String username) {
+ return SVC_SLIDER_USERS + "/" + username;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
new file mode 100644
index 0000000..d6d664e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+
+/**
+ * Extends Core Filesystem with operations to manipulate ClusterDescription
+ * persistent state
+ */
+public class SliderFileSystem extends CoreFileSystem {
+
+ Path appDir = null;
+
+ public SliderFileSystem(FileSystem fileSystem,
+ Configuration configuration) {
+ super(fileSystem, configuration);
+ }
+
+ public SliderFileSystem(Configuration configuration) throws IOException {
+ super(configuration);
+ }
+
+ public void setAppDir(Path appDir) {
+ this.appDir = appDir;
+ }
+
+ public Path getAppDir() {
+ return this.appDir;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java
deleted file mode 100644
index 9c5708f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.junit.Assert;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.assertNotNull;
-
-/**
- * This is an evolving engine to mock YARN operations.
- */
-public class MockYarnEngine {
- private static final Logger LOG =
- LoggerFactory.getLogger(MockYarnEngine.class);
-
- private MockYarnCluster cluster;
- private Allocator allocator;
- private List<ContainerRequestOperation> pending = new ArrayList<>();
-
- private ApplicationId appId = new MockApplicationId(0, 0);
-
- private ApplicationAttemptId attemptId = new MockApplicationAttemptId(appId,
- 1);
-
- @Override
- public String toString() {
- return "MockYarnEngine " + cluster + " + pending=" + pending.size();
- }
-
- public int containerCount() {
- return cluster.containersInUse();
- }
-
- public MockYarnEngine(int clusterSize, int containersPerNode) {
- cluster = new MockYarnCluster(clusterSize, containersPerNode);
- allocator = new Allocator(cluster);
- }
-
- public MockYarnCluster getCluster() {
- return cluster;
- }
-
- public Allocator getAllocator() {
- return allocator;
- }
-
- /**
- * Allocate a container from a request. The containerID will be
- * unique, nodeId and other fields chosen internally with
- * no such guarantees; resource and priority copied over
- * @param request request
- * @return container
- */
- public Container allocateContainer(AMRMClient.ContainerRequest request) {
- MockContainer allocated = allocator.allocate(request);
- if (allocated != null) {
- MockContainerId id = (MockContainerId)allocated.getId();
- id.setApplicationAttemptId(attemptId);
- }
- return allocated;
- }
-
- MockYarnCluster.MockYarnClusterContainer releaseContainer(ContainerId
- containerId) {
- return cluster.release(containerId);
- }
-
- /**
- * Process a list of operations -release containers to be released,
- * allocate those for which there is space (but don't rescan the list after
- * the scan).
- * @param ops
- * @return
- */
- public List<Container> execute(List<AbstractRMOperation> ops) {
- return execute(ops, new ArrayList<>());
- }
-
- /**
- * Process a list of operations -release containers to be released,
- * allocate those for which there is space (but don't rescan the list after
- * the scan). Unsatisifed entries are appended to the "pending" list
- * @param ops operations
- * @return the list of all satisfied operations
- */
- public List<Container> execute(List<AbstractRMOperation> ops,
- List<ContainerId> released) {
- validateRequests(ops);
- List<Container> allocation = new ArrayList<>();
- for (AbstractRMOperation op : ops) {
- if (op instanceof ContainerReleaseOperation) {
- ContainerReleaseOperation cro = (ContainerReleaseOperation) op;
- ContainerId cid = cro.getContainerId();
- assertNotNull(releaseContainer(cid));
- released.add(cid);
- } else if (op instanceof CancelSingleRequest) {
- // no-op
- LOG.debug("cancel request {}", op);
- } else if (op instanceof ContainerRequestOperation) {
- ContainerRequestOperation req = (ContainerRequestOperation) op;
- Container container = allocateContainer(req.getRequest());
- if (container != null) {
- LOG.info("allocated container {} for {}", container, req);
- allocation.add(container);
- } else {
- LOG.debug("Unsatisfied allocation {}", req);
- pending.add(req);
- }
- } else {
- LOG.warn("Unsupported operation {}", op);
- }
- }
- return allocation;
- }
-
- /**
- * Try and mimic some of the logic of <code>AMRMClientImpl
- * .checkLocalityRelaxationConflict</code>.
- * @param ops operations list
- */
- void validateRequests(List<AbstractRMOperation> ops) {
- // run through the requests and verify that they are all consistent.
- List<ContainerRequestOperation> outstandingRequests = new ArrayList<>();
- for (AbstractRMOperation operation : ops) {
- if (operation instanceof ContainerRequestOperation) {
- ContainerRequestOperation containerRequest =
- (ContainerRequestOperation) operation;
- ContainerRequest amRequest = containerRequest.getRequest();
- Priority priority = amRequest.getPriority();
- boolean relax = amRequest.getRelaxLocality();
-
- for (ContainerRequestOperation req : outstandingRequests) {
- if (req.getPriority() == priority && req.getRelaxLocality() !=
- relax) {
- // mismatch in values
- Assert.fail("operation " + operation + " has incompatible request" +
- " priority from outsanding request");
- }
- outstandingRequests.add(containerRequest);
-
- }
-
- }
- }
- }
-
- /**
- * Get the list of node reports. These are not cloned; updates will persist
- * in the nodemap.
- * @return current node report list
- */
- List<NodeReport> getNodeReports() {
- return cluster.getNodeReports();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.java
deleted file mode 100644
index 16bd195..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.monkey;
-
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.server.appmaster.actions.ActionHalt;
-import org.apache.slider.server.appmaster.actions.ActionKillContainer;
-import org.apache.slider.server.appmaster.actions.AsyncAction;
-import org.apache.slider.server.appmaster.actions.QueueService;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler;
-import org.apache.slider.server.appmaster.monkey.ChaosKillAM;
-import org.apache.slider.server.appmaster.monkey.ChaosKillContainer;
-import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService;
-import org.apache.slider.server.appmaster.monkey.ChaosTarget;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Test chaos monkey.
- */
-public class TestMockMonkey extends BaseMockAppStateTest {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestMockMonkey.class);
-
- /**
- * This queue service is NOT started; tests need to poll the queue
- * rather than expect them to execute.
- */
- private QueueService queues;
- private ChaosMonkeyService monkey;
-
- @Before
- public void init() {
- YarnConfiguration configuration = new YarnConfiguration();
- queues = new QueueService();
- queues.init(configuration);
- monkey = new ChaosMonkeyService(METRICS.getMetrics(), queues);
- monkey.init(configuration);
- }
-
- //@Test
- public void testMonkeyStart() throws Throwable {
- monkey.start();
- monkey.stop();
- }
-
- //@Test
- public void testMonkeyPlay() throws Throwable {
- ChaosCounter counter = new ChaosCounter();
- monkey.addTarget("target", counter, InternalKeys.PROBABILITY_PERCENT_100);
- assertEquals(1, monkey.getTargetCount());
- monkey.play();
- assertEquals(1, counter.count);
- }
-
- //@Test
- public void testMonkeySchedule() throws Throwable {
- ChaosCounter counter = new ChaosCounter();
- assertEquals(0, monkey.getTargetCount());
- monkey.addTarget("target", counter, InternalKeys.PROBABILITY_PERCENT_100);
- assertEquals(1, monkey.getTargetCount());
- assertTrue(monkey.schedule(0, 1, TimeUnit.SECONDS));
- assertEquals(1, queues.scheduledActions.size());
- }
-
- //@Test
- public void testMonkeyDoesntAddProb0Actions() throws Throwable {
- ChaosCounter counter = new ChaosCounter();
- monkey.addTarget("target", counter, 0);
- assertEquals(0, monkey.getTargetCount());
- monkey.play();
- assertEquals(0, counter.count);
- }
-
- //@Test
- public void testMonkeyScheduleProb0Actions() throws Throwable {
- ChaosCounter counter = new ChaosCounter();
- monkey.addTarget("target", counter, 0);
- assertFalse(monkey.schedule(0, 1, TimeUnit.SECONDS));
- assertEquals(0, queues.scheduledActions.size());
- }
-
- //@Test
- public void testMonkeyPlaySometimes() throws Throwable {
- ChaosCounter counter = new ChaosCounter();
- ChaosCounter counter2 = new ChaosCounter();
- monkey.addTarget("target1", counter, InternalKeys.PROBABILITY_PERCENT_1
- * 50);
- monkey.addTarget("target2", counter2, InternalKeys
- .PROBABILITY_PERCENT_1 * 25);
-
- for (int i = 0; i < 100; i++) {
- monkey.play();
- }
- LOG.info("Counter1 = {} counter2 = {}", counter.count, counter2.count);
- /*
- * Relying on probability here to give approximate answers
- */
- assertTrue(counter.count > 25);
- assertTrue(counter.count < 75);
- assertTrue(counter2.count < counter.count);
- }
-
- //@Test
- public void testAMKiller() throws Throwable {
-
- ChaosKillAM chaos = new ChaosKillAM(queues, -1);
- chaos.chaosAction();
- assertEquals(1, queues.scheduledActions.size());
- AsyncAction action = queues.scheduledActions.take();
- assertTrue(action instanceof ActionHalt);
- }
-
- //@Test
- public void testContainerKillerEmptyApp() throws Throwable {
-
-
- ChaosKillContainer chaos = new ChaosKillContainer(appState,
- queues,
- new MockRMOperationHandler());
- chaos.chaosAction();
- assertEquals(0, queues.scheduledActions.size());
- }
-
- @Ignore
- //@Test
- public void testContainerKillerIgnoresAM() throws Throwable {
- // TODO: AM needed in live container list?
- addAppMastertoAppState();
- assertEquals(1, appState.getLiveContainers().size());
-
- ChaosKillContainer chaos = new ChaosKillContainer(appState,
- queues,
- new MockRMOperationHandler());
- chaos.chaosAction();
- assertEquals(0, queues.scheduledActions.size());
- }
-
- //@Test
- public void testContainerKiller() throws Throwable {
- MockRMOperationHandler ops = new MockRMOperationHandler();
- getRole0Status().setDesired(1);
- List<RoleInstance> instances = createAndStartNodes();
- assertEquals(1, instances.size());
- RoleInstance instance = instances.get(0);
-
- ChaosKillContainer chaos = new ChaosKillContainer(appState, queues, ops);
- chaos.chaosAction();
- assertEquals(1, queues.scheduledActions.size());
- AsyncAction action = queues.scheduledActions.take();
- ActionKillContainer killer = (ActionKillContainer) action;
- assertEquals(killer.getContainerId(), instance.getContainerId());
- killer.execute(null, queues, appState);
- assertEquals(1, ops.getNumReleases());
-
- ContainerReleaseOperation operation = (ContainerReleaseOperation) ops
- .getFirstOp();
- assertEquals(operation.getContainerId(), instance.getContainerId());
- }
-
- /**
- * Chaos target that just implements a counter.
- */
- private static class ChaosCounter implements ChaosTarget {
- private int count;
-
- @Override
- public void chaosAction() {
- count++;
- }
-
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder(
- "ChaosCounter{");
- sb.append("count=").append(count);
- sb.append('}');
- return sb.toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/security/TestSecurityConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/security/TestSecurityConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/security/TestSecurityConfiguration.java
deleted file mode 100644
index 6e77806..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/security/TestSecurityConfiguration.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.security;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.slider.api.resource.Application;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.core.exceptions.SliderException;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Test security configuration.
- */
-public class TestSecurityConfiguration {
-
- //@Test
- public void testValidLocalConfiguration() throws Throwable {
- Configuration config = new Configuration();
- config.set(CommonConfigurationKeysPublic
- .HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- Map<String, String> compOps = new HashMap<>();
- compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test");
- compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH,
- "/some/local/path");
- Application application = new Application().configuration(new org.apache
- .slider.api.resource.Configuration().properties(compOps));
-
- SecurityConfiguration securityConfiguration =
- new SecurityConfiguration(config, application, "testCluster");
- }
-
- //@Test
- public void testValidDistributedConfiguration() throws Throwable {
- Configuration config = new Configuration();
- config.set(CommonConfigurationKeysPublic
- .HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- Map<String, String> compOps = new HashMap<>();
- compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test");
- compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab");
- Application application = new Application().configuration(new org.apache
- .slider.api.resource.Configuration().properties(compOps));
-
- SecurityConfiguration securityConfiguration =
- new SecurityConfiguration(config, application, "testCluster");
- }
-
- //@Test
- public void testMissingPrincipalNoLoginWithDistributedConfig() throws
- Throwable {
- Configuration config = new Configuration();
- config.set(CommonConfigurationKeysPublic
- .HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- Map<String, String> compOps = new HashMap<>();
- compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab");
- Application application = new Application().configuration(new org.apache
- .slider.api.resource.Configuration().properties(compOps));
-
- try {
- SecurityConfiguration securityConfiguration =
- new SecurityConfiguration(config, application, "testCluster") {
- @Override
- protected UserGroupInformation getLoginUser() throws
- IOException {
- return null;
- }
- };
- fail("expected SliderException");
- } catch (SliderException e) {
- // expected
- }
- }
-
- //@Test
- public void testMissingPrincipalNoLoginWithLocalConfig() throws Throwable {
- Configuration config = new Configuration();
- config.set(CommonConfigurationKeysPublic
- .HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- Map<String, String> compOps = new HashMap<>();
- compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH,
- "/some/local/path");
- Application application = new Application().configuration(new org.apache
- .slider.api.resource.Configuration().properties(compOps));
-
- try {
- SecurityConfiguration securityConfiguration =
- new SecurityConfiguration(config, application, "testCluster") {
- @Override
- protected UserGroupInformation getLoginUser() throws IOException {
- return null;
- }
- };
- fail("expected SliderException");
- } catch (SliderException e) {
- // expected
- }
- }
-
- //@Test
- public void testBothKeytabMechanismsConfigured() throws Throwable {
- Configuration config = new Configuration();
- config.set(CommonConfigurationKeysPublic
- .HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- Map<String, String> compOps = new HashMap<>();
- compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test");
- compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH,
- "/some/local/path");
- compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab");
- Application application = new Application().configuration(new org.apache
- .slider.api.resource.Configuration().properties(compOps));
-
- try {
- SecurityConfiguration securityConfiguration =
- new SecurityConfiguration(config, application,
- "testCluster");
- fail("expected SliderException");
- } catch (SliderException e) {
- // expected
- }
- }
-
- //@Test
- public void testMissingPrincipalButLoginWithDistributedConfig() throws
- Throwable {
- Configuration config = new Configuration();
- config.set(CommonConfigurationKeysPublic
- .HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- Map<String, String> compOps = new HashMap<>();
- compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab");
- Application application = new Application().configuration(new org.apache
- .slider.api.resource.Configuration().properties(compOps));
-
- SecurityConfiguration securityConfiguration =
- new SecurityConfiguration(config, application, "testCluster");
- }
-
- //@Test
- public void testMissingPrincipalButLoginWithLocalConfig() throws Throwable {
- Configuration config = new Configuration();
- config.set(CommonConfigurationKeysPublic
- .HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- Map<String, String> compOps = new HashMap<>();
- compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH,
- "/some/local/path");
- Application application = new Application().configuration(new org.apache
- .slider.api.resource.Configuration().properties(compOps));
-
- SecurityConfiguration securityConfiguration =
- new SecurityConfiguration(config, application, "testCluster");
- }
-
- //@Test
- public void testKeypathLocationOnceLocalized() throws Throwable {
- Configuration config = new Configuration();
- config.set(CommonConfigurationKeysPublic
- .HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- Map<String, String> compOps = new HashMap<>();
- compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab");
- Application application = new Application().configuration(new org.apache
- .slider.api.resource.Configuration().properties(compOps));
-
- SecurityConfiguration securityConfiguration =
- new SecurityConfiguration(config, application, "testCluster");
-
- assertEquals(new File(SliderKeys.KEYTAB_DIR, "some.keytab")
- .getAbsolutePath(),
- securityConfiguration.getKeytabFile().getAbsolutePath());
- }
-
- //@Test
- public void testAMKeytabProvided() throws Throwable {
- Configuration config = new Configuration();
- Map<String, String> compOps = new HashMap<>();
- compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, " ");
- Application application = new Application().configuration(new org.apache
- .slider.api.resource.Configuration().properties(compOps));
-
- SecurityConfiguration securityConfiguration =
- new SecurityConfiguration(config, application, "testCluster");
- assertFalse(securityConfiguration.isKeytabProvided());
-
- compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "");
- assertFalse(securityConfiguration.isKeytabProvided());
-
- compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab");
- assertTrue(securityConfiguration.isKeytabProvided());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.java
deleted file mode 100644
index b887f28..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.registry;
-
-import org.apache.hadoop.registry.client.binding.JsonSerDeser;
-
-class PathEntryMarshalling
- extends JsonSerDeser<PathEntryResource> {
- public PathEntryMarshalling() {
- super(PathEntryResource.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.java
deleted file mode 100644
index b0b0e31..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.registry;
-
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.junit.Test;
-
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-
-/**
- * This test exists because Jersey appears to behave "oddly"
- * when it comes to marshalling JSON, and some of the REST resources
- * appear to have trouble.
- *
- * This test tries to isolate it
- */
-public class TestRegistryRestMarshalling {
-
- //@Test
- public void testDeser() throws Throwable {
- PathEntryMarshalling pem = new PathEntryMarshalling();
- PathEntryResource unmarshalled = pem.fromResource(
- "/org/apache/slider/server/appmaster/web/rest/registry/sample.json");
-
- ServiceRecord serviceRecord = unmarshalled.service;
- assertNotNull(serviceRecord);
- assertNotNull(serviceRecord.get(YarnRegistryAttributes.YARN_ID));
- assertNotEquals("", serviceRecord.get(YarnRegistryAttributes
- .YARN_PERSISTENCE));
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.java
deleted file mode 100644
index 43e4f39..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.view;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.state.ProviderAppState;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.WebAppApiImpl;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.PrintWriter;
-import java.io.StringWriter;
-
-/**
- * Test cluster specification block.
- */
-public class TestClusterSpecificationBlock extends BaseMockAppStateTest {
-
- private ClusterSpecificationBlock clusterSpecBlock;
-
- @Before
- public void setup() throws Exception {
- super.setup();
- ProviderAppState providerAppState = new ProviderAppState(
- "undefined",
- appState);
-
- WebAppApiImpl inst = new WebAppApiImpl(
- providerAppState,
- null,
- null, null);
-
- Injector injector = Guice.createInjector(new AbstractModule() {
- @Override
- protected void configure() {
- bind(WebAppApi.class).toInstance(inst);
- }
- });
-
- clusterSpecBlock = injector.getInstance(ClusterSpecificationBlock.class);
- }
-
- //@Test
- public void testJsonGeneration() {
- StringWriter sw = new StringWriter(64);
- PrintWriter pw = new PrintWriter(sw);
-
- Hamlet hamlet = new Hamlet(pw, 0, false);
-
- int level = hamlet.nestLevel();
- clusterSpecBlock.doRender(hamlet);
-
- assertEquals(level, hamlet.nestLevel());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.java
deleted file mode 100644
index 56f209c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.view;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
-import org.apache.hadoop.yarn.webapp.hamlet.HamletImpl.EImp;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockContainer;
-import org.apache.slider.server.appmaster.model.mock.MockContainerId;
-import org.apache.slider.server.appmaster.model.mock.MockNodeId;
-import org.apache.slider.server.appmaster.model.mock.MockResource;
-import org.apache.slider.server.appmaster.state.ProviderAppState;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.WebAppApiImpl;
-import org.apache.slider.server.appmaster.web.view.ContainerStatsBlock.ClusterNodeNameComparator;
-import org.apache.slider.server.appmaster.web.view.ContainerStatsBlock.TableAnchorContent;
-import org.apache.slider.server.appmaster.web.view.ContainerStatsBlock.TableContent;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Test container stats block.
- */
-public class TestContainerStatsBlock extends BaseMockAppStateTest {
-
- private ContainerStatsBlock statsBlock;
-
- private Container cont1, cont2;
-
- @Before
- public void setup() throws Exception {
- super.setup();
- ProviderAppState providerAppState = new ProviderAppState(
- "undefined",
- appState);
-
- WebAppApiImpl inst = new WebAppApiImpl(
- providerAppState,
- null,
- METRICS, null);
-
- Injector injector = Guice.createInjector(new WebappModule(inst));
- statsBlock = injector.getInstance(ContainerStatsBlock.class);
-
- cont1 = new MockContainer();
-
- cont1.setId(mockContainerId(0));
- cont1.setNodeId(new MockNodeId());
- cont1.setPriority(Priority.newInstance(1));
- cont1.setResource(new MockResource(0, 0));
-
- cont2 = new MockContainer();
- cont2.setId(mockContainerId(1));
- cont2.setNodeId(new MockNodeId());
- cont2.setPriority(Priority.newInstance(1));
- cont2.setResource(new MockResource(0, 0));
- }
-
- private static class WebappModule extends AbstractModule {
- private final WebAppApiImpl instance;
-
- WebappModule(WebAppApiImpl instance) {
- this.instance = instance;
- }
-
- @Override
- protected void configure() {
- bind(WebAppApi.class).toInstance(instance);
- }
- }
-
-
- public MockContainerId mockContainerId(int count) {
- return new MockContainerId(applicationAttemptId, count);
- }
-
- //@Test
- public void testGetContainerInstances() {
- List<RoleInstance> roles = Arrays.asList(
- new RoleInstance(cont1),
- new RoleInstance(cont2)
- );
- Map<String, RoleInstance> map = statsBlock.getContainerInstances(roles);
-
- assertEquals(2, map.size());
-
- assertTrue(map.containsKey("mockcontainer_0"));
- assertEquals(map.get("mockcontainer_0"), roles.get(0));
-
- assertTrue(map.containsKey("mockcontainer_1"));
- assertEquals(map.get("mockcontainer_1"), roles.get(1));
- }
-
- //@Test
- public void testGenerateRoleDetailsWithTwoColumns() {
- StringWriter sw = new StringWriter(64);
- PrintWriter pw = new PrintWriter(sw);
-
- Hamlet hamlet = new Hamlet(pw, 0, false);
-
- // Make a div to put the content into
- DIV<Hamlet> div = hamlet.div();
-
- String detailsName = "testing";
- String selector = "selector";
- Map<TableContent, String> data = new HashMap<>();
- data.put(new ContainerStatsBlock.TableContent("Foo"), "bar");
-
- int levelPrior = hamlet.nestLevel();
- statsBlock.generateRoleDetails(div, selector, detailsName, data.entrySet());
-
- // Close out the div we made
- // DIV<Hamlet>._() will actually invoke the wrong method (creating <p>),
- // explicit cast to make sure we're closing out the <div>
- ((EImp) div)._();
-
- assertEquals(levelPrior, hamlet.nestLevel());
- }
-
- //@Test
- public void testGenerateRoleDetailsWithOneColumn() {
- StringWriter sw = new StringWriter(64);
- PrintWriter pw = new PrintWriter(sw);
-
- Hamlet hamlet = new Hamlet(pw, 0, false);
- DIV<Hamlet> div = hamlet.div();
-
- String detailsName = "testing";
- String selector = "selector";
- Map<TableContent, String> data = new HashMap<>();
- data.put(new ContainerStatsBlock.TableContent("Bar"), null);
-
- int levelPrior = hamlet.nestLevel();
- statsBlock.generateRoleDetails(div, selector, detailsName, data.entrySet());
-
- // Close out the div we made
- // DIV<Hamlet>._() will actually invoke the wrong method (creating <p>),
- // explicit cast to make sure we're closing out the <div>
- ((EImp) div)._();
-
- assertEquals(levelPrior, hamlet.nestLevel());
- }
-
- //@Test
- public void testGenerateRoleDetailsWithNoData() {
- StringWriter sw = new StringWriter(64);
- PrintWriter pw = new PrintWriter(sw);
-
- Hamlet hamlet = new Hamlet(pw, 0, false);
- DIV<Hamlet> div = hamlet.div();
-
- String detailsName = "testing";
- String selector = "selector";
- Map<TableContent, String> data = new HashMap<>();
-
- int levelPrior = hamlet.nestLevel();
- statsBlock.generateRoleDetails(div, selector, detailsName, data.entrySet());
-
- // Close out the div we made
- // DIV<Hamlet>._() will actually invoke the wrong method (creating <p>),
- // explicit cast to make sure we're closing out the <div>
- ((EImp) div)._();
-
- assertEquals(levelPrior, hamlet.nestLevel());
- }
-
- //@Test
- public void testClusterNodeNameComparator() {
- ClusterNode n1 = new ClusterNode(mockContainerId(1)),
- n2 = new ClusterNode(mockContainerId(2)),
- n3 = new ClusterNode(mockContainerId(3));
-
- List<ClusterNode> nodes = new ArrayList<ClusterNode>();
- nodes.add(n2);
- nodes.add(n3);
- nodes.add(n1);
-
- Collections.sort(nodes, new ClusterNodeNameComparator());
-
- String prevName = "";
- for (ClusterNode node : nodes) {
- assertTrue(prevName.compareTo(node.name) <= 0);
- prevName = node.name;
- }
- }
-
- //@Test
- public void testTableContent() {
- StringWriter sw = new StringWriter(64);
- PrintWriter pw = new PrintWriter(sw);
- TableContent tc = new TableContent("foo");
-
- Hamlet hamlet = new Hamlet(pw, 0, false);
- TR<TABLE<Hamlet>> tr = hamlet.table().tr();
-
- int prevLevel = hamlet.nestLevel();
- // printCell should not end the tr
- tc.printCell(tr);
- tr._();
- assertEquals(prevLevel, hamlet.nestLevel());
- }
-
- //@Test
- public void testTableAnchorContent() {
- StringWriter sw = new StringWriter(64);
- PrintWriter pw = new PrintWriter(sw);
- TableContent tc = new TableAnchorContent("foo", "http://bar.com");
-
- Hamlet hamlet = new Hamlet(pw, 0, false);
- TR<TABLE<Hamlet>> tr = hamlet.table().tr();
-
- int prevLevel = hamlet.nestLevel();
- // printCell should not end the tr
- tc.printCell(tr);
- tr._();
- assertEquals(prevLevel, hamlet.nestLevel());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestIndexBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestIndexBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestIndexBlock.java
deleted file mode 100644
index eecf213..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestIndexBlock.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.view;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.slider.server.appmaster.model.appstate.BaseMockAppStateAATest;
-import org.apache.slider.server.appmaster.model.mock.MockContainer;
-import org.apache.slider.server.appmaster.model.mock.MockContainerId;
-import org.apache.slider.server.appmaster.model.mock.MockNodeId;
-import org.apache.slider.server.appmaster.model.mock.MockResource;
-import org.apache.slider.server.appmaster.state.ContainerOutcome;
-import org.apache.slider.server.appmaster.state.OutstandingRequest;
-import org.apache.slider.server.appmaster.state.ProviderAppState;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.WebAppApiImpl;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.PrintWriter;
-import java.io.StringWriter;
-
-/**
- * Test index block.
- */
-public class TestIndexBlock extends BaseMockAppStateAATest {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestIndexBlock.class);
-
- private IndexBlock indexBlock;
-
- private Container cont1, cont2;
-
- @Before
- public void setup() throws Exception {
- super.setup();
- assertNotNull(appState);
- ProviderAppState providerAppState = new ProviderAppState(
- "undefined",
- appState);
-
- WebAppApiImpl inst = new WebAppApiImpl(
- providerAppState,
- null,
- METRICS, null);
-
- Injector injector = Guice.createInjector(new AbstractModule() {
- @Override
- protected void configure() {
- bind(WebAppApi.class).toInstance(inst);
- }
- });
-
- indexBlock = injector.getInstance(IndexBlock.class);
-
- cont1 = new MockContainer();
- cont1.setId(new MockContainerId(applicationAttemptId, 0));
- cont1.setNodeId(new MockNodeId());
- cont1.setPriority(Priority.newInstance(1));
- cont1.setResource(new MockResource(0, 0));
-
- cont2 = new MockContainer();
- cont2.setId(new MockContainerId(applicationAttemptId, 1));
- cont2.setNodeId(new MockNodeId());
- cont2.setPriority(Priority.newInstance(1));
- cont2.setResource(new MockResource(0, 0));
- }
-
- //@Test
- public void testIndex() {
- RoleStatus role0 = getRole0Status();
- RoleStatus role1 = getRole1Status();
- RoleStatus role2 = getRole2Status();
-
- int role0Desired = 8;
-
- role0.setDesired(role0Desired);
- int role0Actual = 5;
- int role0Requested = role0Desired - role0Actual;
- for (int i = 0; i < role0Actual; i++) {
- appState.incRunningContainers(role0);
- }
- assertEquals(role0.getRunning(), role0Actual);
- for (int i = 0; i < role0Requested; i++) {
- appState.incRequestedContainers(role0);
- }
- assertEquals(role0.getRequested(), role0Requested);
-
- int role0Failures = 2;
-
- appState.incFailedContainers(role0, ContainerOutcome.Failed);
- appState.incFailedContainers(role0, ContainerOutcome.Failed);
-
- RoleStatus aaRole = getAaRole();
- // all aa roles fields are in the
- int aaroleDesired = 200;
- aaRole.setDesired(aaroleDesired);
- int aaroleActual = 90;
- int aaroleActive = 1;
- int aaroleRequested = aaroleDesired - aaroleActual;
- int aarolePending = aaroleRequested - 1;
- int aaroleFailures = 0;
- for (int i = 0; i < aaroleActual; i++) {
- appState.incRunningContainers(aaRole);
- }
- assertEquals(aaRole.getRunning(), aaroleActual);
- aaRole.setOutstandingAArequest(new OutstandingRequest(2, ""));
- // add a requested
- appState.incRequestedContainers(aaRole);
- aaRole.getComponentMetrics().pendingAAContainers.set(aarolePending);
- assertEquals(aaRole.getAAPending(), aarolePending);
-
- assertEquals(aaRole.getActualAndRequested(), aaroleActual + 1);
- StringWriter sw = new StringWriter(64);
- PrintWriter pw = new PrintWriter(sw);
-
- Hamlet hamlet = new Hamlet(pw, 0, false);
-
- indexBlock.doIndex(hamlet, "accumulo");
-
- String body = sw.toString();
- LOG.info(body);
- // verify role data came out
- assertTrue(body.contains("role0"));
- assertContains(role0Desired, body);
- assertContains(role0Actual, body);
- assertContains(role0Requested, body);
- assertContains(role0Failures, body);
-
- assertTrue(body.contains("role1"));
- assertTrue(body.contains("role2"));
-
- assertContains(aaroleDesired, body);
- assertContains(aaroleActual, body);
-// assertContains(aaroleRequested, body)
- assertContains(aaroleFailures, body);
- assertTrue(body.contains(indexBlock.buildAADetails(true, aarolePending)));
-
- // verify that the sorting took place
- assertTrue(body.indexOf("role0") < body.indexOf("role1"));
- assertTrue(body.indexOf("role1") < body.indexOf("role2"));
-
- assertFalse(body.contains(IndexBlock.ALL_CONTAINERS_ALLOCATED));
- // role
- }
-
- void assertContains(int ex, String html) {
- assertStringContains(Integer.toString(ex), html);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/management/TestGauges.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/management/TestGauges.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/management/TestGauges.java
deleted file mode 100644
index 11ebabe..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/management/TestGauges.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.management;
-
-import org.apache.slider.server.appmaster.management.LongGauge;
-import org.apache.slider.utils.SliderTestBase;
-import org.junit.Test;
-
-/**
- * Test gauges.
- */
-public class TestGauges extends SliderTestBase {
-
- //@Test
- public void testLongGaugeOperations() throws Throwable {
- LongGauge gauge = new LongGauge();
- assertEquals(0, gauge.get());
- gauge.inc();
- assertEquals(1, gauge.get());
- gauge.inc();
- assertEquals(2, gauge.get());
- gauge.inc();
- assertEquals(3, gauge.get());
- assertEquals(gauge.getValue().longValue(), gauge.get());
- assertEquals(gauge.getCount().longValue(), gauge.get());
-
- gauge.dec();
- assertEquals(2, gauge.get());
- assertEquals(1, gauge.decToFloor(1));
- assertEquals(1, gauge.get());
- assertEquals(0, gauge.decToFloor(1));
- assertEquals(0, gauge.decToFloor(1));
- assertEquals(0, gauge.decToFloor(0));
-
- gauge.set(4);
- assertEquals(0, gauge.decToFloor(8));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/MockService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/MockService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/MockService.java
deleted file mode 100644
index 588f621..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/MockService.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.service.ServiceStateException;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-public class MockService extends AbstractService {
- private final boolean fail;
- private final int lifespan;
- private final ExecutorService executorService =
- Executors.newSingleThreadExecutor();
-
- MockService() {
- this("mock", false, -1);
- }
-
- MockService(String name, boolean fail, int lifespan) {
- super(name);
- this.fail = fail;
- this.lifespan = lifespan;
- }
-
- @Override
- protected void serviceStart() throws Exception {
- //act on the lifespan here
- if (lifespan > 0) {
- executorService.submit(new Runnable() {
- @Override
- public void run() {
- try {
- Thread.sleep(lifespan);
- } catch (InterruptedException ignored) {
-
- }
- finish();
- }
- });
- } else {
- if (lifespan == 0) {
- finish();
- } else {
- //continue until told not to
- }
- }
- }
-
- void finish() {
- if (fail) {
- ServiceStateException e =
- new ServiceStateException(getName() + " failed");
-
- noteFailure(e);
- stop();
- throw e;
- } else {
- stop();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ParentWorkflowTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ParentWorkflowTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ParentWorkflowTestBase.java
deleted file mode 100644
index a11a1cf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ParentWorkflowTestBase.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.service.Service;
-
-/**
- * Extends {@link WorkflowServiceTestBase} with parent-specific operations
- * and logic to build up and run the parent service
- */
-public abstract class ParentWorkflowTestBase extends WorkflowServiceTestBase {
-
- /**
- * Wait a second for the service parent to stop
- * @param parent the service to wait for
- */
- protected void waitForParentToStop(ServiceParent parent) {
- waitForParentToStop(parent, 1000);
- }
-
- /**
- * Wait for the service parent to stop
- * @param parent the service to wait for
- * @param timeout time in milliseconds
- */
- protected void waitForParentToStop(ServiceParent parent, int timeout) {
- boolean stop = parent.waitForServiceToStop(timeout);
- if (!stop) {
- logState(parent);
- fail("Service failed to stop : after " + timeout + " millis " + parent);
- }
- }
-
- /**
- * Subclasses are require to implement this and return an instance of a
- * ServiceParent
- * @param services a possibly empty list of services
- * @return an inited -but -not-started- service parent instance
- */
- protected abstract ServiceParent buildService(Service... services);
-
- /**
- * Use {@link #buildService(Service...)} to create service and then start it
- * @param services
- * @return
- */
- protected ServiceParent startService(Service... services) {
- ServiceParent parent = buildService(services);
- //expect service to start and stay started
- parent.start();
- return parent;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
deleted file mode 100644
index 4a19417..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.util.Shell;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * A source of commands, with the goal being to allow for adding different
- * implementations for different platforms
- */
-public class ProcessCommandFactory {
-
- protected ProcessCommandFactory() {
- }
-
- /**
- * The command to list a directory
- * @param dir directory
- * @return commands
- */
- public List<String> ls(File dir) {
- List<String> commands;
- if (!Shell.WINDOWS) {
- commands = Arrays.asList("ls","-1", dir.getAbsolutePath());
- } else {
- commands = Arrays.asList("cmd", "/c", "dir", dir.getAbsolutePath());
- }
- return commands;
- }
-
- /**
- * Echo some text to stdout
- * @param text text
- * @return commands
- */
- public List<String> echo(String text) {
- List<String> commands = new ArrayList<String>(5);
- commands.add("echo");
- commands.add(text);
- return commands;
- }
-
- /**
- * print env variables
- * @return commands
- */
- public List<String> env() {
- List<String> commands;
- if (!Shell.WINDOWS) {
- commands = Arrays.asList("env");
- } else {
- commands = Arrays.asList("cmd", "/c", "set");
- }
- return commands;
- }
-
- /**
- * execute a command that returns with an error code that will
- * be converted into a number
- * @return commands
- */
- public List<String> exitFalse() {
- List<String> commands = new ArrayList<String>(2);
- commands.add("false");
- return commands;
- }
-
- /**
- * Create a process command factory for this OS
- * @return
- */
- public static ProcessCommandFactory createProcessCommandFactory() {
- return new ProcessCommandFactory();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/SimpleRunnable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/SimpleRunnable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/SimpleRunnable.java
deleted file mode 100644
index 1f330f4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/SimpleRunnable.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-/**
- * Test runnable that can be made to exit, or throw an exception
- * during its run
- */
-class SimpleRunnable implements Runnable {
- boolean throwException = false;
-
-
- SimpleRunnable() {
- }
-
- SimpleRunnable(boolean throwException) {
- this.throwException = throwException;
- }
-
- @Override
- public synchronized void run() {
- try {
- if (throwException) {
- throw new RuntimeException("SimpleRunnable");
- }
- } finally {
- this.notify();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java
deleted file mode 100644
index 19f40e9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-public class TestWorkflowClosingService extends WorkflowServiceTestBase {
-
- //@Test
- public void testSimpleClose() throws Throwable {
- ClosingService<OpenClose> svc = instance(false);
- OpenClose openClose = svc.getCloseable();
- assertFalse(openClose.closed);
- svc.stop();
- assertTrue(openClose.closed);
- }
-
- //@Test
- public void testNullClose() throws Throwable {
- ClosingService<OpenClose> svc = new ClosingService<OpenClose>("", null);
- svc.init(new Configuration());
- svc.start();
- assertNull(svc.getCloseable());
- svc.stop();
- }
-
- //@Test
- public void testFailingClose() throws Throwable {
- ClosingService<OpenClose> svc = instance(false);
- OpenClose openClose = svc.getCloseable();
- openClose.raiseExceptionOnClose = true;
- svc.stop();
- assertTrue(openClose.closed);
- Throwable cause = svc.getFailureCause();
- assertNotNull(cause);
-
- //retry should be a no-op
- svc.close();
- }
-
- //@Test
- public void testDoubleClose() throws Throwable {
- ClosingService<OpenClose> svc = instance(false);
- OpenClose openClose = svc.getCloseable();
- openClose.raiseExceptionOnClose = true;
- svc.stop();
- assertTrue(openClose.closed);
- Throwable cause = svc.getFailureCause();
- assertNotNull(cause);
- openClose.closed = false;
- svc.stop();
- assertEquals(cause, svc.getFailureCause());
- }
-
- /**
- * This does not recurse forever, as the service has already entered the
- * STOPPED state before the inner close tries to stop it -that operation
- * is a no-op
- * @throws Throwable
- */
- //@Test
- public void testCloseSelf() throws Throwable {
- ClosingService<ClosingService> svc =
- new ClosingService<ClosingService>("");
- svc.setCloseable(svc);
- svc.stop();
- }
-
-
- private ClosingService<OpenClose> instance(boolean raiseExceptionOnClose) {
- ClosingService<OpenClose> svc = new ClosingService<OpenClose>(new OpenClose(
- raiseExceptionOnClose));
- svc.init(new Configuration());
- svc.start();
- return svc;
- }
-
- private static class OpenClose implements Closeable {
- public boolean closed = false;
- public boolean raiseExceptionOnClose;
-
- private OpenClose(boolean raiseExceptionOnClose) {
- this.raiseExceptionOnClose = raiseExceptionOnClose;
- }
-
- @Override
- public void close() throws IOException {
- if (!closed) {
- closed = true;
- if (raiseExceptionOnClose) {
- throw new IOException("OpenClose");
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowCompositeService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowCompositeService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowCompositeService.java
deleted file mode 100644
index 0cd1ac9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowCompositeService.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.Service;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestWorkflowCompositeService extends ParentWorkflowTestBase {
- private static final Logger
- log = LoggerFactory.getLogger(TestWorkflowCompositeService.class);
-
- //@Test
- public void testSingleChild() throws Throwable {
- Service parent = startService(new MockService());
- parent.stop();
- }
-
- //@Test
- public void testSingleChildTerminating() throws Throwable {
- ServiceParent parent =
- startService(new MockService("1", false, 100));
- waitForParentToStop(parent);
- }
-
- //@Test
- public void testSingleChildFailing() throws Throwable {
- ServiceParent parent =
- startService(new MockService("1", true, 100));
- waitForParentToStop(parent);
- assert parent.getFailureCause() != null;
- }
-
- //@Test
- public void testTwoChildren() throws Throwable {
- MockService one = new MockService("one", false, 100);
- MockService two = new MockService("two", false, 100);
- ServiceParent parent = startService(one, two);
- waitForParentToStop(parent);
- assertStopped(one);
- assertStopped(two);
- }
-
- //@Test
- public void testCallableChild() throws Throwable {
-
- MockService one = new MockService("one", false, 100);
- CallableHandler handler = new CallableHandler("hello");
- WorkflowCallbackService<String> ens =
- new WorkflowCallbackService<String>("handler", handler, 100, true);
- MockService two = new MockService("two", false, 100);
- ServiceParent parent = startService(one, ens, two);
- waitForParentToStop(parent);
- assertStopped(one);
- assertStopped(ens);
- assertStopped(two);
- assertTrue(handler.notified);
- String s = ens.getScheduledFuture().get();
- assertEquals("hello", s);
- }
-
- //@Test
- public void testNestedComposite() throws Throwable {
- MockService one = new MockService("one", false, 100);
- MockService two = new MockService("two", false, 100);
- ServiceParent parent = buildService(one, two);
- ServiceParent outer = startService(parent);
- assertTrue(outer.waitForServiceToStop(1000));
- assertStopped(one);
- assertStopped(two);
- }
-
- //@Test
- public void testFailingComposite() throws Throwable {
- MockService one = new MockService("one", true, 10);
- MockService two = new MockService("two", false, 1000);
- ServiceParent parent = startService(one, two);
- waitForParentToStop(parent);
- assertStopped(one);
- assertStopped(two);
- assertNotNull(one.getFailureCause());
- assertNotNull(parent.getFailureCause());
- assertEquals(one.getFailureCause(), parent.getFailureCause());
- }
-
- @Override
- public ServiceParent buildService(Service... services) {
- ServiceParent parent =
- new WorkflowCompositeService("test", services);
- parent.init(new Configuration());
- return parent;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java
deleted file mode 100644
index 38cc886..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.junit.Test;
-
-import java.util.concurrent.ExecutorService;
-
-
-/**
- * Basic tests for executor service
- */
-public class TestWorkflowExecutorService extends WorkflowServiceTestBase {
-
- //@Test
- public void testAsyncRun() throws Throwable {
-
- ExecutorSvc svc = run(new ExecutorSvc());
- ServiceTerminatingRunnable runnable = new ServiceTerminatingRunnable(svc,
- new SimpleRunnable());
-
- // synchronous in-thread execution
- svc.execute(runnable);
- Thread.sleep(1000);
- assertStopped(svc);
- }
-
- //@Test
- public void testFailureRun() throws Throwable {
-
- ExecutorSvc svc = run(new ExecutorSvc());
- ServiceTerminatingRunnable runnable = new ServiceTerminatingRunnable(svc,
- new SimpleRunnable(true));
-
- // synchronous in-thread execution
- svc.execute(runnable);
- Thread.sleep(1000);
- assertStopped(svc);
- assertNotNull(runnable.getException());
- }
-
- private static class ExecutorSvc
- extends WorkflowExecutorService<ExecutorService> {
- private ExecutorSvc() {
- super("ExecutorService",
- ServiceThreadFactory.singleThreadExecutor("test", true));
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowRpcService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowRpcService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowRpcService.java
deleted file mode 100644
index 758c64f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowRpcService.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.Server;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-public class TestWorkflowRpcService extends WorkflowServiceTestBase {
-
- //@Test
- public void testCreateMockRPCService() throws Throwable {
- MockRPC rpc = new MockRPC();
- rpc.start();
- assertTrue(rpc.started);
- rpc.getListenerAddress();
- rpc.stop();
- assertTrue(rpc.stopped);
- }
-
- //@Test
- public void testLifecycle() throws Throwable {
- MockRPC rpc = new MockRPC();
- WorkflowRpcService svc = new WorkflowRpcService("test", rpc);
- run(svc);
- assertTrue(rpc.started);
- svc.getConnectAddress();
- svc.stop();
- assertTrue(rpc.stopped);
- }
-
- //@Test
- public void testStartFailure() throws Throwable {
- MockRPC rpc = new MockRPC();
- rpc.failOnStart = true;
- WorkflowRpcService svc = new WorkflowRpcService("test", rpc);
- svc.init(new Configuration());
- try {
- svc.start();
- fail("expected an exception");
- } catch (RuntimeException e) {
- assertEquals("failOnStart", e.getMessage());
- }
- svc.stop();
- assertTrue(rpc.stopped);
- }
-
- private static class MockRPC extends Server {
-
- public boolean stopped;
- public boolean started;
- public boolean failOnStart;
-
- private MockRPC() throws IOException {
- super("localhost", 0, null, 1, new Configuration());
- }
-
- @Override
- public synchronized void start() {
- if (failOnStart) {
- throw new RuntimeException("failOnStart");
- }
- started = true;
- super.start();
- }
-
- @Override
- public synchronized void stop() {
- stopped = true;
- super.stop();
- }
-
- @Override
- public synchronized InetSocketAddress getListenerAddress() {
- return super.getListenerAddress();
- }
-
- @Override
- public Writable call(RPC.RpcKind rpcKind,
- String protocol,
- Writable param,
- long receiveTime) throws Exception {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowSequenceService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowSequenceService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowSequenceService.java
deleted file mode 100644
index b683641..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowSequenceService.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.Service;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestWorkflowSequenceService extends ParentWorkflowTestBase {
- private static final Logger
- log = LoggerFactory.getLogger(TestWorkflowSequenceService.class);
-
- //@Test
- public void testSingleSequence() throws Throwable {
- ServiceParent parent = startService(new MockService());
- parent.stop();
- }
-
- //@Test
- public void testEmptySequence() throws Throwable {
- ServiceParent parent = startService();
- waitForParentToStop(parent);
- }
-
- //@Test
- public void testSequence() throws Throwable {
- MockService one = new MockService("one", false, 100);
- MockService two = new MockService("two", false, 100);
- ServiceParent parent = startService(one, two);
- waitForParentToStop(parent);
- assertStopped(one);
- assertStopped(two);
- assert ((WorkflowSequenceService) parent).getPreviousService().equals(two);
- }
-
- //@Test
- public void testCallableChild() throws Throwable {
-
- MockService one = new MockService("one", false, 100);
- CallableHandler handler = new CallableHandler("hello");
- WorkflowCallbackService<String> ens =
- new WorkflowCallbackService<String>("handler", handler, 100, true);
- MockService two = new MockService("two", false, 100);
- ServiceParent parent = startService(one, ens, two);
- waitForParentToStop(parent);
- assertStopped(one);
- assertStopped(ens);
- assertStopped(two);
- assertTrue(handler.notified);
- String s = ens.getScheduledFuture().get();
- assertEquals("hello", s);
- }
-
-
- //@Test
- public void testFailingSequence() throws Throwable {
- MockService one = new MockService("one", true, 100);
- MockService two = new MockService("two", false, 100);
- WorkflowSequenceService parent =
- (WorkflowSequenceService) startService(one, two);
- waitForParentToStop(parent);
- assertStopped(one);
- assertInState(two, Service.STATE.NOTINITED);
- assertEquals(one, parent.getPreviousService());
- }
-
-
- //@Test
- public void testFailInStartNext() throws Throwable {
- MockService one = new MockService("one", false, 100);
- MockService two = new MockService("two", true, 0);
- MockService three = new MockService("3", false, 0);
- ServiceParent parent = startService(one, two, three);
- waitForParentToStop(parent);
- assertStopped(one);
- assertStopped(two);
- Throwable failureCause = two.getFailureCause();
- assertNotNull(failureCause);
- Throwable parentFailureCause = parent.getFailureCause();
- assertNotNull(parentFailureCause);
- assertEquals(parentFailureCause, failureCause);
- assertInState(three, Service.STATE.NOTINITED);
- }
-
- //@Test
- public void testSequenceInSequence() throws Throwable {
- MockService one = new MockService("one", false, 100);
- MockService two = new MockService("two", false, 100);
- ServiceParent parent = buildService(one, two);
- ServiceParent outer = startService(parent);
- waitForParentToStop(parent);
- assertStopped(one);
- assertStopped(two);
- }
-
- //@Test
- public void testVarargsConstructor() throws Throwable {
- MockService one = new MockService("one", false, 100);
- MockService two = new MockService("two", false, 100);
- ServiceParent parent = new WorkflowSequenceService("test", one, two);
- parent.init(new Configuration());
- parent.start();
- waitForParentToStop(parent);
- assertStopped(one);
- assertStopped(two);
- }
-
-
- //@Test
- public void testAddChild() throws Throwable {
- MockService one = new MockService("one", false, 5000);
- MockService two = new MockService("two", false, 100);
- ServiceParent parent = startService(one, two);
- CallableHandler handler = new CallableHandler("hello");
- WorkflowCallbackService<String> ens =
- new WorkflowCallbackService<String>("handler", handler, 100, true);
- parent.addService(ens);
- waitForParentToStop(parent, 10000);
- assertStopped(one);
- assertStopped(two);
- assertStopped(ens);
- assertStopped(two);
- assertEquals("hello", ens.getScheduledFuture().get());
- }
-
- public WorkflowSequenceService buildService(Service... services) {
- WorkflowSequenceService parent =
- new WorkflowSequenceService("test", services);
- parent.init(new Configuration());
- return parent;
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[56/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
deleted file mode 100644
index e10305a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import java.util.Locale;
-
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public enum ConfigFormat {
-
- JSON("json"),
- PROPERTIES("properties"),
- XML("xml"),
- HADOOP_XML("hadoop_xml"),
- ENV("env"),
- TEMPLATE("template"),
- YAML("yaml"),
- ;
- ConfigFormat(String suffix) {
- this.suffix = suffix;
- }
-
- private final String suffix;
-
- public String getSuffix() {
- return suffix;
- }
-
-
- @Override
- public String toString() {
- return suffix;
- }
-
- /**
- * Get a matching format or null
- * @param type
- * @return the format
- */
- public static ConfigFormat resolve(String type) {
- for (ConfigFormat format: values()) {
- if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) {
- return format;
- }
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
deleted file mode 100644
index 0ac508b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * Set of configuration properties that can be injected into the application
- * components via envs, files and custom pluggable helper docker containers.
- * Files of several standard formats like xml, properties, json, yaml and
- * templates will be supported.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "Set of configuration properties that can be injected into the application components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Configuration implements Serializable {
- private static final long serialVersionUID = -4330788704981074466L;
-
- private Map<String, String> properties = new HashMap<String, String>();
- private Map<String, String> env = new HashMap<String, String>();
- private List<ConfigFile> files = new ArrayList<ConfigFile>();
-
- /**
- * A blob of key-value pairs of common application properties.
- **/
- public Configuration properties(Map<String, String> properties) {
- this.properties = properties;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A blob of key-value pairs of common application properties.")
- @JsonProperty("properties")
- public Map<String, String> getProperties() {
- return properties;
- }
-
- public void setProperties(Map<String, String> properties) {
- this.properties = properties;
- }
-
- /**
- * A blob of key-value pairs which will be appended to the default system
- * properties and handed off to the application at start time. All placeholder
- * references to properties will be substituted before injection.
- **/
- public Configuration env(Map<String, String> env) {
- this.env = env;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A blob of key-value pairs which will be appended to the default system properties and handed off to the application at start time. All placeholder references to properties will be substituted before injection.")
- @JsonProperty("env")
- public Map<String, String> getEnv() {
- return env;
- }
-
- public void setEnv(Map<String, String> env) {
- this.env = env;
- }
-
- /**
- * Array of list of files that needs to be created and made available as
- * volumes in the application component containers.
- **/
- public Configuration files(List<ConfigFile> files) {
- this.files = files;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Array of list of files that needs to be created and made available as volumes in the application component containers.")
- @JsonProperty("files")
- public List<ConfigFile> getFiles() {
- return files;
- }
-
- public void setFiles(List<ConfigFile> files) {
- this.files = files;
- }
-
- public long getPropertyLong(String name, long defaultValue) {
- String value = getProperty(name);
- if (StringUtils.isEmpty(value)) {
- return defaultValue;
- }
- return Long.parseLong(value);
- }
-
- public int getPropertyInt(String name, int defaultValue) {
- String value = getProperty(name);
- if (StringUtils.isEmpty(value)) {
- return defaultValue;
- }
- return Integer.parseInt(value);
- }
-
- public boolean getPropertyBool(String name, boolean defaultValue) {
- String value = getProperty(name);
- if (StringUtils.isEmpty(value)) {
- return defaultValue;
- }
- return Boolean.parseBoolean(value);
- }
-
- public String getProperty(String name, String defaultValue) {
- String value = getProperty(name);
- if (StringUtils.isEmpty(value)) {
- return defaultValue;
- }
- return value;
- }
-
- public void setProperty(String name, String value) {
- properties.put(name, value);
- }
-
- public String getProperty(String name) {
- return properties.get(name.trim());
- }
-
- public String getEnv(String name) {
- return env.get(name.trim());
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Configuration configuration = (Configuration) o;
- return Objects.equals(this.properties, configuration.properties)
- && Objects.equals(this.env, configuration.env)
- && Objects.equals(this.files, configuration.files);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(properties, env, files);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Configuration {\n");
-
- sb.append(" properties: ").append(toIndentedString(properties))
- .append("\n");
- sb.append(" env: ").append(toIndentedString(env)).append("\n");
- sb.append(" files: ").append(toIndentedString(files)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-
- /**
- * Merge all properties and envs from that configuration to this configration.
- * For ConfigFiles, all properties and envs of that ConfigFile are merged into
- * this ConfigFile.
- */
- public synchronized void mergeFrom(Configuration that) {
- SliderUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
- .getProperties());
- SliderUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
-
- Map<String, ConfigFile> thatMap = new HashMap<>();
- for (ConfigFile file : that.getFiles()) {
- thatMap.put(file.getDestFile(), file.copy());
- }
- for (ConfigFile thisFile : files) {
- if(thatMap.containsKey(thisFile.getDestFile())) {
- ConfigFile thatFile = thatMap.get(thisFile.getDestFile());
- SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProps(),
- thatFile.getProps());
- thatMap.remove(thisFile.getDestFile());
- }
- }
- // add remaining new files from that Configration
- for (ConfigFile thatFile : thatMap.values()) {
- files.add(thatFile.copy());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
deleted file mode 100644
index 8b687bb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.util.Date;
-import java.util.Objects;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * An instance of a running application container.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "An instance of a running application container")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@XmlRootElement
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Container extends BaseResource {
- private static final long serialVersionUID = -8955788064529288L;
-
- private String id = null;
- private Date launchTime = null;
- private String ip = null;
- private String hostname = null;
- private String bareHost = null;
- private ContainerState state = null;
- private String componentName = null;
- private Resource resource = null;
- private Artifact artifact = null;
- private Boolean privilegedContainer = null;
-
- /**
- * Unique container id of a running application, e.g.
- * container_e3751_1458061340047_0008_01_000002.
- **/
- public Container id(String id) {
- this.id = id;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Unique container id of a running application, e.g. container_e3751_1458061340047_0008_01_000002.")
- @JsonProperty("id")
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- /**
- * The time when the container was created, e.g. 2016-03-16T01:01:49.000Z.
- * This will most likely be different from cluster launch time.
- **/
- public Container launchTime(Date launchTime) {
- this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.")
- @JsonProperty("launch_time")
- public Date getLaunchTime() {
- return launchTime == null ? null : (Date) launchTime.clone();
- }
-
- @XmlElement(name = "launch_time")
- public void setLaunchTime(Date launchTime) {
- this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
- }
-
- /**
- * IP address of a running container, e.g. 172.31.42.141. The IP address and
- * hostname attribute values are dependent on the cluster/docker network setup
- * as per YARN-4007.
- **/
- public Container ip(String ip) {
- this.ip = ip;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.")
- @JsonProperty("ip")
- public String getIp() {
- return ip;
- }
-
- public void setIp(String ip) {
- this.ip = ip;
- }
-
- /**
- * Fully qualified hostname of a running container, e.g.
- * ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and
- * hostname attribute values are dependent on the cluster/docker network setup
- * as per YARN-4007.
- **/
- public Container hostname(String hostname) {
- this.hostname = hostname;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.")
- @JsonProperty("hostname")
- public String getHostname() {
- return hostname;
- }
-
- public void setHostname(String hostname) {
- this.hostname = hostname;
- }
-
- /**
- * The bare node or host in which the container is running, e.g.
- * cn008.example.com.
- **/
- public Container bareHost(String bareHost) {
- this.bareHost = bareHost;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The bare node or host in which the container is running, e.g. cn008.example.com.")
- @JsonProperty("bare_host")
- public String getBareHost() {
- return bareHost;
- }
-
- @XmlElement(name = "bare_host")
- public void setBareHost(String bareHost) {
- this.bareHost = bareHost;
- }
-
- /**
- * State of the container of an application.
- **/
- public Container state(ContainerState state) {
- this.state = state;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "State of the container of an application.")
- @JsonProperty("state")
- public ContainerState getState() {
- return state;
- }
-
- public void setState(ContainerState state) {
- this.state = state;
- }
-
- /**
- * Name of the component that this container instance belongs to.
- **/
- public Container componentName(String componentName) {
- this.componentName = componentName;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Name of the component that this container instance belongs to.")
- @JsonProperty("component_name")
- public String getComponentName() {
- return componentName;
- }
-
- @XmlElement(name = "component_name")
- public void setComponentName(String componentName) {
- this.componentName = componentName;
- }
-
- /**
- * Resource used for this container.
- **/
- public Container resource(Resource resource) {
- this.resource = resource;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Resource used for this container.")
- @JsonProperty("resource")
- public Resource getResource() {
- return resource;
- }
-
- public void setResource(Resource resource) {
- this.resource = resource;
- }
-
- /**
- * Artifact used for this container.
- **/
- public Container artifact(Artifact artifact) {
- this.artifact = artifact;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact used for this container.")
- @JsonProperty("artifact")
- public Artifact getArtifact() {
- return artifact;
- }
-
- public void setArtifact(Artifact artifact) {
- this.artifact = artifact;
- }
-
- /**
- * Container running in privileged mode or not.
- **/
- public Container privilegedContainer(Boolean privilegedContainer) {
- this.privilegedContainer = privilegedContainer;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Container running in privileged mode or not.")
- @JsonProperty("privileged_container")
- public Boolean getPrivilegedContainer() {
- return privilegedContainer;
- }
-
- public void setPrivilegedContainer(Boolean privilegedContainer) {
- this.privilegedContainer = privilegedContainer;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Container container = (Container) o;
- return Objects.equals(this.id, container.id);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(id);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Container {\n");
-
- sb.append(" id: ").append(toIndentedString(id)).append("\n");
- sb.append(" launchTime: ").append(toIndentedString(launchTime))
- .append("\n");
- sb.append(" ip: ").append(toIndentedString(ip)).append("\n");
- sb.append(" hostname: ").append(toIndentedString(hostname)).append("\n");
- sb.append(" bareHost: ").append(toIndentedString(bareHost)).append("\n");
- sb.append(" state: ").append(toIndentedString(state)).append("\n");
- sb.append(" componentName: ").append(toIndentedString(componentName))
- .append("\n");
- sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
- sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
- sb.append(" privilegedContainer: ")
- .append(toIndentedString(privilegedContainer)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
deleted file mode 100644
index bf09ff2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * The current state of the container of an application.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public enum ContainerState {
- RUNNING_BUT_UNREADY, READY, STOPPED
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
deleted file mode 100644
index c64b1b5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModelProperty;
-
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public class Error {
-
- private Integer code = null;
- private String message = null;
- private String fields = null;
-
- /**
- **/
- public Error code(Integer code) {
- this.code = code;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "")
- @JsonProperty("code")
- public Integer getCode() {
- return code;
- }
-
- public void setCode(Integer code) {
- this.code = code;
- }
-
- /**
- **/
- public Error message(String message) {
- this.message = message;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "")
- @JsonProperty("message")
- public String getMessage() {
- return message;
- }
-
- public void setMessage(String message) {
- this.message = message;
- }
-
- /**
- **/
- public Error fields(String fields) {
- this.fields = fields;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "")
- @JsonProperty("fields")
- public String getFields() {
- return fields;
- }
-
- public void setFields(String fields) {
- this.fields = fields;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Error error = (Error) o;
- return Objects.equals(this.code, error.code)
- && Objects.equals(this.message, error.message)
- && Objects.equals(this.fields, error.fields);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(code, message, fields);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Error {\n");
-
- sb.append(" code: ").append(toIndentedString(code)).append("\n");
- sb.append(" message: ").append(toIndentedString(message)).append("\n");
- sb.append(" fields: ").append(toIndentedString(fields)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
deleted file mode 100644
index 7d1b889..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Placement policy of an instance of an application. This feature is in the
- * works in YARN-4902.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "Placement policy of an instance of an application. This feature is in the works in YARN-4902.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public class PlacementPolicy implements Serializable {
- private static final long serialVersionUID = 4341110649551172231L;
-
- private String label = null;
-
- /**
- * Assigns an app to a named partition of the cluster where the application
- * desires to run (optional). If not specified all apps are submitted to a
- * default label of the app owner. One or more labels can be setup for each
- * application owner account with required constraints like no-preemption,
- * sla-99999, preemption-ok, etc.
- **/
- public PlacementPolicy label(String label) {
- this.label = label;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Assigns an app to a named partition of the cluster where the application desires to run (optional). If not specified all apps are submitted to a default label of the app owner. One or more labels can be setup for each application owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.")
- @JsonProperty("label")
- public String getLabel() {
- return label;
- }
-
- public void setLabel(String label) {
- this.label = label;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- PlacementPolicy placementPolicy = (PlacementPolicy) o;
- return Objects.equals(this.label, placementPolicy.label);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(label);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class PlacementPolicy {\n");
-
- sb.append(" label: ").append(toIndentedString(label)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
deleted file mode 100644
index eadbb48..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonValue;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * A custom command or a pluggable helper container to determine the readiness
- * of a container of a component. Readiness for every application is different.
- * Hence the need for a simple interface, with scope to support advanced
- * usecases.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every application is different. Hence the need for a simple interface, with scope to support advanced usecases.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public class ReadinessCheck implements Serializable {
- private static final long serialVersionUID = -3836839816887186801L;
-
- public enum TypeEnum {
- HTTP("HTTP"),
- PORT("PORT");
-
- private String value;
-
- TypeEnum(String value) {
- this.value = value;
- }
-
- @Override
- @JsonValue
- public String toString() {
- return value;
- }
- }
-
- private TypeEnum type = null;
- private Map<String, String> props = new HashMap<String, String>();
- private Artifact artifact = null;
-
- /**
- * E.g. HTTP (YARN will perform a simple REST call at a regular interval and
- * expect a 204 No content).
- **/
- public ReadinessCheck type(TypeEnum type) {
- this.type = type;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).")
- @JsonProperty("type")
- public TypeEnum getType() {
- return type;
- }
-
- public void setType(TypeEnum type) {
- this.type = type;
- }
-
- public ReadinessCheck props(Map<String, String> props) {
- this.props = props;
- return this;
- }
-
- public ReadinessCheck putPropsItem(String key, String propsItem) {
- this.props.put(key, propsItem);
- return this;
- }
-
- /**
- * A blob of key value pairs that will be used to configure the check.
- * @return props
- **/
- @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be used to configure the check.")
- public Map<String, String> getProps() {
- return props;
- }
-
- public void setProps(Map<String, String> props) {
- this.props = props;
- }
-
- /**
- * Artifact of the pluggable readiness check helper container (optional). If
- * specified, this helper container typically hosts the http uri and
- * encapsulates the complex scripts required to perform actual container
- * readiness check. At the end it is expected to respond a 204 No content just
- * like the simplified use case. This pluggable framework benefits application
- * owners who can run applications without any packaging modifications. Note,
- * artifacts of type docker only is supported for now.
- **/
- public ReadinessCheck artifact(Artifact artifact) {
- this.artifact = artifact;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits application owners who can run applications without any packaging modifications. Note, artifacts of type docker only is supported for now.")
- @JsonProperty("artifact")
- public Artifact getArtifact() {
- return artifact;
- }
-
- public void setArtifact(Artifact artifact) {
- this.artifact = artifact;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- ReadinessCheck readinessCheck = (ReadinessCheck) o;
- return Objects.equals(this.type, readinessCheck.type) &&
- Objects.equals(this.props, readinessCheck.props) &&
- Objects.equals(this.artifact, readinessCheck.artifact);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(type, props, artifact);
- }
-
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class ReadinessCheck {\n");
-
- sb.append(" type: ").append(toIndentedString(type)).append("\n");
- sb.append(" props: ").append(toIndentedString(props)).append("\n");
- sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
deleted file mode 100644
index bda79c9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Resource determines the amount of resources (vcores, memory, network, etc.)
- * usable by a container. This field determines the resource to be applied for
- * all the containers of a component or application. The resource specified at
- * the app (or global) level can be overriden at the component level. Only one
- * of profile OR cpu & memory are exepected. It raises a validation
- * exception otherwise.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public class Resource extends BaseResource implements Cloneable {
- private static final long serialVersionUID = -6431667797380250037L;
-
- private String profile = null;
- private Integer cpus = 1;
- private String memory = null;
-
- /**
- * Each resource profile has a unique id which is associated with a
- * cluster-level predefined memory, cpus, etc.
- **/
- public Resource profile(String profile) {
- this.profile = profile;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.")
- @JsonProperty("profile")
- public String getProfile() {
- return profile;
- }
-
- public void setProfile(String profile) {
- this.profile = profile;
- }
-
- /**
- * Amount of vcores allocated to each container (optional but overrides cpus
- * in profile if specified).
- **/
- public Resource cpus(Integer cpus) {
- this.cpus = cpus;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).")
- @JsonProperty("cpus")
- public Integer getCpus() {
- return cpus;
- }
-
- public void setCpus(Integer cpus) {
- this.cpus = cpus;
- }
-
- /**
- * Amount of memory allocated to each container (optional but overrides memory
- * in profile if specified). Currently accepts only an integer value and
- * default unit is in MB.
- **/
- public Resource memory(String memory) {
- this.memory = memory;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.")
- @JsonProperty("memory")
- public String getMemory() {
- return memory;
- }
-
- public void setMemory(String memory) {
- this.memory = memory;
- }
-
- public long getMemoryMB() {
- if (this.memory == null) {
- return 0;
- }
- return Long.valueOf(memory);
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Resource resource = (Resource) o;
- return Objects.equals(this.profile, resource.profile)
- && Objects.equals(this.cpus, resource.cpus)
- && Objects.equals(this.memory, resource.memory);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(profile, cpus, memory);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Resource {\n");
-
- sb.append(" profile: ").append(toIndentedString(profile)).append("\n");
- sb.append(" cpus: ").append(toIndentedString(cpus)).append("\n");
- sb.append(" memory: ").append(toIndentedString(memory)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException {
- return super.clone();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java
deleted file mode 100644
index 0749077..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.client.ServerProxy;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-
-import java.net.InetSocketAddress;
-
-public class ClientAMProxy extends ServerProxy{
-
- public static <T> T createProxy(final Configuration conf,
- final Class<T> protocol, final UserGroupInformation ugi,
- final YarnRPC rpc, final InetSocketAddress serverAddress) {
-
- RetryPolicy retryPolicy =
- createRetryPolicy(conf, YarnServiceConf.CLIENT_AM_RETRY_MAX_WAIT_MS,
- 15 * 60 * 1000, YarnServiceConf.CLIENT_AM_RETRY_MAX_INTERVAL_MS,
- 2 * 1000);
- Configuration confClone = new Configuration(conf);
- confClone.setInt(
- CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
- confClone.setInt(CommonConfigurationKeysPublic.
- IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
- return createRetriableProxy(confClone, protocol, ugi, rpc, serverAddress,
- retryPolicy);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
deleted file mode 100644
index c7421ff..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.*;
-
-public class ServiceCLI {
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceClient.class);
- protected ServiceClient client;
-
- int exec(ClientArgs args) throws Throwable {
- if (StringUtils.isEmpty(args.getAction())) {
- System.out.println(args.usage());
- return -1;
- }
- switch (args.getAction()) {
- case ACTION_BUILD: // Upload app json onto hdfs
- client.actionBuild(args.getActionBuildArgs());
- break;
- case ACTION_START: // start the app with the pre-uploaded app json on hdfs
- client.actionStart(args.getClusterName());
- break;
- case ACTION_CREATE: // create == build + start
- client.actionCreate(args.getActionCreateArgs());
- break;
- case ACTION_STATUS:
- Application app = client.getStatus(args.getClusterName());
- System.out.println(app);
- break;
- case ACTION_FLEX:
- client.actionFlexByCLI(args);
- break;
- case ACTION_STOP:
- client.actionStop(args.getClusterName(), false);
- break;
- case ACTION_DESTROY: // Destroy can happen only if app is already stopped
- client.actionDestroy(args.getClusterName());
- break;
- case ACTION_DEPENDENCY: // upload dependency jars
- client.actionDependency(args.getActionDependencyArgs());
- break;
- case ACTION_UPDATE:
- client.updateLifetime(args.getClusterName(),
- args.getActionUpdateArgs().lifetime);
- break;
- case ACTION_HELP:
- LOG.info(args.usage());
- break;
- default:
- LOG.info("NOT IMPLEMENTED: " + args.getAction());
- LOG.info(args.usage());
- return -1;
- }
- return 0;
- }
-
- public ServiceCLI() {
- createServiceClient();
- }
-
- protected void createServiceClient() {
- client = new ServiceClient();
- client.init(new YarnConfiguration());
- client.start();
- }
-
- public static void main(String[] args) throws Throwable {
- ClientArgs clientArgs = new ClientArgs(args);
- try {
- clientArgs.parse();
- } catch (BadCommandArgumentsException e) {
- System.err.println(e.getMessage());
- System.exit(-1);
- }
- ServiceCLI cli = new ServiceCLI();
- int res = cli.exec(clientArgs);
- System.exit(res);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org