You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by go...@apache.org on 2016/08/25 20:20:01 UTC
[32/46] incubator-slider git commit: SLIDER-1165 Create
yarn-native-services branch on Slider corresponding to the
yarn-native-services branch on Hadoop
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
deleted file mode 100644
index efec676..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import org.apache.hadoop.fs.InvalidRequestException;
-import org.apache.hadoop.fs.PathAccessDeniedException;
-import org.apache.hadoop.fs.PathIOException;
-import org.apache.hadoop.yarn.webapp.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletResponse;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
-/**
- * static methods to convert exceptions into different types, including
- * extraction of details and finer-grained conversions.
- */
-public class ExceptionConverter {
- private static final Logger
- log = LoggerFactory.getLogger(ExceptionConverter.class);
-
- /**
- * Uprate error codes 400 and up into faults;
- * 404 is converted to a {@link FileNotFoundException},
- * 401 to {@link ForbiddenException}
- * FileNotFoundException for an unknown resource
- * PathAccessDeniedException for access denied
- * PathIOException for anything else
- * @param verb HTTP Verb used
- * @param targetURL URL being targeted
- * @param exception original exception
- * @return a new exception, the original one nested as a cause
- */
- public static IOException convertJerseyException(String verb,
- String targetURL,
- UniformInterfaceException exception) {
-
- IOException ioe = null;
- ClientResponse response = exception.getResponse();
- if (response != null) {
- int status = response.getStatus();
- String body = "";
- try {
- if (response.hasEntity()) {
- body = response.getEntity(String.class);
- log.error("{} {} returned status {} and body\n{}",
- verb, targetURL, status, body);
- } else {
- log.error("{} {} returned status {} and empty body",
- verb, targetURL, status);
- }
- } catch (Exception e) {
- log.warn("Failed to extract body from client response", e);
- }
-
- if (status == HttpServletResponse.SC_UNAUTHORIZED
- || status == HttpServletResponse.SC_FORBIDDEN) {
- ioe = new PathAccessDeniedException(targetURL);
- } else if (status == HttpServletResponse.SC_BAD_REQUEST
- || status == HttpServletResponse.SC_NOT_ACCEPTABLE
- || status == HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE) {
- // bad request
- ioe = new InvalidRequestException(
- String.format("Bad %s request: status code %d against %s",
- verb, status, targetURL));
- } else if (status > 400 && status < 500) {
- ioe = new FileNotFoundException(targetURL);
- }
- if (ioe == null) {
- ioe = new PathIOException(targetURL,
- verb + " " + targetURL
- + " failed with status code : " + status
- + ":" + exception);
- }
- } else {
- ioe = new PathIOException(targetURL,
- verb + " " + targetURL + " failed: " + exception);
- }
- ioe.initCause(exception);
- return ioe;
- }
-
- /**
- * Handle a client-side Jersey exception.
- * <p>
- * If there's an inner IOException, return that.
- * <p>
- * Otherwise: create a new wrapper IOE including verb and target details
- * @param verb HTTP Verb used
- * @param targetURL URL being targeted
- * @param exception original exception
- * @return an exception to throw
- */
- public static IOException convertJerseyException(String verb,
- String targetURL,
- ClientHandlerException exception) {
- if (exception.getCause() instanceof IOException) {
- return (IOException)exception.getCause();
- } else {
- IOException ioe = new IOException(
- verb + " " + targetURL + " failed: " + exception);
- ioe.initCause(exception);
- return ioe;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/NoSuchNodeException.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/NoSuchNodeException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/NoSuchNodeException.java
deleted file mode 100644
index ad2f1a4..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/NoSuchNodeException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-import java.io.IOException;
-
-/**
- * Exception raised when a node cannot be found in the structure
- * that is being examined.
- */
-public class NoSuchNodeException extends IOException {
-
- public NoSuchNodeException(String uuid) {
- super(uuid);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/NotFoundException.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/NotFoundException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/NotFoundException.java
deleted file mode 100644
index 40cb94d..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/NotFoundException.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-
-/**
- * Whatever was being resolved: it was not found
- */
-public class NotFoundException extends SliderException {
- public NotFoundException(String message,
- Object... args) {
- super(EXIT_NOT_FOUND, message, args);
- }
-
- public NotFoundException(Throwable throwable,
- String message, Object... args) {
- super(EXIT_NOT_FOUND, throwable, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/ServiceNotReadyException.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/ServiceNotReadyException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/ServiceNotReadyException.java
deleted file mode 100644
index 435bc1a..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/ServiceNotReadyException.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-import java.io.IOException;
-
-/**
- * This is an exception raised when the service does not consider itself
- * live (yet)
- */
-public class ServiceNotReadyException extends IOException {
-
- public static final String E_NOT_READY =
- "Service not ready for access: please retry";
-
- public ServiceNotReadyException(String message) {
- super(message);
- }
-
- public ServiceNotReadyException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public ServiceNotReadyException(Throwable cause) {
- super(cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
deleted file mode 100644
index 7f3134a..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-import org.apache.slider.common.SliderExitCodes;
-import org.apache.slider.core.main.ServiceLaunchException;
-
-public class SliderException extends ServiceLaunchException implements
- SliderExitCodes {
- public SliderException() {
- super(EXIT_EXCEPTION_THROWN, "SliderException");
- }
-
- public SliderException(int code, String message) {
- super(code, message);
- }
-
- public SliderException(String s) {
- super(EXIT_EXCEPTION_THROWN, s);
- }
-
- public SliderException(String s, Throwable throwable) {
- super(EXIT_EXCEPTION_THROWN, s, throwable);
- }
-
- /**
- * Format the exception as you create it
- * @param code exit code
- * @param message exception message -sprintf formatted
- * @param args arguments for the formatting
- */
- public SliderException(int code, String message, Object... args) {
- super(code, String.format(message, args));
- }
-
- /**
- * Format the exception, include a throwable.
- * The throwable comes before the message so that it is out of the varargs
- * @param code exit code
- * @param throwable thrown
- * @param message message
- * @param args arguments
- */
- public SliderException(int code,
- Throwable throwable,
- String message,
- Object... args) {
- super(code, String.format(message, args), throwable);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderInternalStateException.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderInternalStateException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderInternalStateException.java
deleted file mode 100644
index deddbbc..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderInternalStateException.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-public class SliderInternalStateException extends SliderException {
- public SliderInternalStateException(String s) {
- super(EXIT_INTERNAL_ERROR, s);
- }
-
- public SliderInternalStateException(String s, Throwable throwable) {
- super(EXIT_INTERNAL_ERROR, throwable, s);
- }
-
- public SliderInternalStateException(String message,
- Object... args) {
- super(EXIT_INTERNAL_ERROR, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java
deleted file mode 100644
index bb9f430..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-
-/**
- * An Exception to be thrown for an explicit "shut down the cluster" operation
- * raised by the application state or other parts of the AM
- */
-public class TriggerClusterTeardownException extends SliderException {
-
- private final FinalApplicationStatus finalApplicationStatus;
-
- public TriggerClusterTeardownException(int code,
- FinalApplicationStatus finalApplicationStatus, String message,
- Object... args) {
- super(code, message, args);
- this.finalApplicationStatus = finalApplicationStatus;
- }
-
- public FinalApplicationStatus getFinalApplicationStatus() {
- return finalApplicationStatus;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/UnknownApplicationInstanceException.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/UnknownApplicationInstanceException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/UnknownApplicationInstanceException.java
deleted file mode 100644
index a1f8ae9..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/UnknownApplicationInstanceException.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-public class UnknownApplicationInstanceException extends SliderException {
- public UnknownApplicationInstanceException(String s) {
- super(EXIT_UNKNOWN_INSTANCE, s);
- }
-
- public UnknownApplicationInstanceException(String s, Throwable throwable) {
- super(EXIT_UNKNOWN_INSTANCE, throwable, s);
- }
-
- public UnknownApplicationInstanceException(String message,
- Object... args) {
- super(EXIT_UNKNOWN_INSTANCE, message, args);
- }
-
- /**
- * Create an instance with the standard exception name
- * @param name name
- * @return an instance to throw
- */
- public static UnknownApplicationInstanceException unknownInstance(String name) {
- return new UnknownApplicationInstanceException(ErrorStrings.E_UNKNOWN_INSTANCE
- + ": " + name);
- }
- public static UnknownApplicationInstanceException unknownInstance(String name,
- Throwable throwable) {
- UnknownApplicationInstanceException exception =
- unknownInstance(name);
- exception.initCause(throwable);
- return exception;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java
deleted file mode 100644
index 8684294..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-/**
- * Used to raise a usage exception ... this has the exit code
- * {@link #EXIT_USAGE}
- */
-public class UsageException extends SliderException {
- public UsageException(String s, Object... args) {
- super(EXIT_USAGE, s, args);
- }
-
- public UsageException(Throwable throwable, String message,
- Object... args) {
- super(EXIT_USAGE, throwable, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/WaitTimeoutException.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/WaitTimeoutException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/WaitTimeoutException.java
deleted file mode 100644
index 5ad3fdc..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/WaitTimeoutException.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.exceptions;
-
-import java.io.IOException;
-
-/**
- * Called when some spinning operation timed out
- */
-public class WaitTimeoutException extends IOException {
- public WaitTimeoutException(String message) {
- super(message);
- }
-
- public WaitTimeoutException(String message, Throwable cause) {
- super(message, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
deleted file mode 100644
index 5a3eb3d..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ /dev/null
@@ -1,528 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LogAggregationContext;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.RoleKeys;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.tools.CoreFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.MapOperations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.Method;
-import java.lang.reflect.InvocationTargetException;
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-/**
- * Launcher of applications: base class
- */
-public abstract class AbstractLauncher extends Configured {
- private static final Logger log =
- LoggerFactory.getLogger(AbstractLauncher.class);
- public static final String CLASSPATH = "CLASSPATH";
- /**
- * Filesystem to use for the launch
- */
- protected final CoreFileSystem coreFileSystem;
- /**
- * Env vars; set up at final launch stage
- */
- protected final Map<String, String> envVars = new HashMap<>();
- protected final MapOperations env = new MapOperations("env", envVars);
- protected final ContainerLaunchContext containerLaunchContext =
- Records.newRecord(ContainerLaunchContext.class);
- protected final List<String> commands = new ArrayList<>(20);
- protected final Map<String, LocalResource> localResources = new HashMap<>();
- protected final Map<String, String> mountPaths = new HashMap<>();
- private final Map<String, ByteBuffer> serviceData = new HashMap<>();
- // security
- protected final Credentials credentials;
- protected LogAggregationContext logAggregationContext;
- protected boolean yarnDockerMode = false;
- protected String dockerImage;
- protected String yarnContainerMountPoints;
- protected String runPrivilegedContainer;
-
-
- /**
- * Create instance.
- * @param conf configuration
- * @param coreFileSystem filesystem
- * @param credentials initial set of credentials -null is permitted
- */
- protected AbstractLauncher(Configuration conf,
- CoreFileSystem coreFileSystem,
- Credentials credentials) {
- super(conf);
- this.coreFileSystem = coreFileSystem;
- this.credentials = credentials != null ? credentials: new Credentials();
- }
-
- /**
- * Get the container. Until "completed", this isn't valid to launch.
- * @return the container to launch
- */
- public ContainerLaunchContext getContainerLaunchContext() {
- return containerLaunchContext;
- }
-
- public void setYarnDockerMode(boolean yarnDockerMode){
- this.yarnDockerMode = yarnDockerMode;
- }
-
- /**
- * Get the env vars to work on
- * @return env vars
- */
- public MapOperations getEnv() {
- return env;
- }
-
- /**
- * Get the launch commands.
- * @return the live list of commands
- */
- public List<String> getCommands() {
- return commands;
- }
-
- /**
- * Get the map of local resources.
- * @return the live map of local resources.
- */
- public Map<String, LocalResource> getLocalResources() {
- return localResources;
- }
-
- public void addLocalResource(String subPath, LocalResource resource) {
- localResources.put(subPath, resource);
- }
-
- public void addLocalResource(String subPath, LocalResource resource, String mountPath) {
- localResources.put(subPath, resource);
- mountPaths.put(subPath, mountPath);
- }
-
- /**
- * Add a set of local resources
- * @param resourceMap map of name:resource to add
- */
- public void addLocalResources(Map<String, LocalResource> resourceMap) {
- localResources.putAll(resourceMap);
- }
-
- public Map<String, ByteBuffer> getServiceData() {
- return serviceData;
- }
-
- /**
- * Accessor to the credentials
- * @return the credentials associated with this launcher
- */
- public Credentials getCredentials() {
- return credentials;
- }
-
- /**
- * Add a command line. It is converted to a single command before being
- * added.
- * @param cmd
- */
- public void addCommandLine(CommandLineBuilder cmd) {
- commands.add(cmd.build());
- }
-
- public void addCommand(String cmd) {
- commands.add(cmd);
- }
-
- /**
- * Add a list of commands. Each element in the list becomes a single command
- * @param commandList list of commands
- */
- public void addCommands(List<String> commandList) {
- commands.addAll(commandList);
- }
-
- /**
- * Get all commands as a string, separated by ";". This is for diagnostics
- * @return a string description of the commands
- */
- public String getCommandsAsString() {
- return SliderUtils.join(getCommands(), "; ");
- }
-
- /**
- * Complete the launch context (copy in env vars, etc).
- * @return the container to launch
- */
- public ContainerLaunchContext completeContainerLaunch() throws IOException {
-
- String cmdStr = SliderUtils.join(commands, " ", false);
- log.debug("Completed setting up container command {}", cmdStr);
- containerLaunchContext.setCommands(commands);
-
- //env variables
- if (log.isDebugEnabled()) {
- log.debug("Environment variables");
- for (Map.Entry<String, String> envPair : envVars.entrySet()) {
- log.debug(" \"{}\"=\"{}\"", envPair.getKey(), envPair.getValue());
- }
- }
- containerLaunchContext.setEnvironment(env);
-
- //service data
- if (log.isDebugEnabled()) {
- log.debug("Service Data size");
- for (Map.Entry<String, ByteBuffer> entry : serviceData.entrySet()) {
- log.debug("\"{}\"=> {} bytes of data", entry.getKey(),
- entry.getValue().array().length);
- }
- }
- containerLaunchContext.setServiceData(serviceData);
-
- // resources
- dumpLocalResources();
- containerLaunchContext.setLocalResources(localResources);
-
- //tokens
- log.debug("{} tokens", credentials.numberOfTokens());
- containerLaunchContext.setTokens(CredentialUtils.marshallCredentials(
- credentials));
-
- if(yarnDockerMode){
- Map<String, String> env = containerLaunchContext.getEnvironment();
- env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker");
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage);//if yarnDockerMode, then dockerImage is set
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer);
- StringBuilder sb = new StringBuilder();
- for (Entry<String,String> mount : mountPaths.entrySet()) {
- if (sb.length() > 0) {
- sb.append(",");
- }
- sb.append(mount.getKey());
- sb.append(":");
- sb.append(mount.getValue());
- }
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS", sb.toString());
- log.info("yarn docker env var has been set {}", containerLaunchContext.getEnvironment().toString());
- }
-
- return containerLaunchContext;
- }
-
- /**
- * Dump local resources at debug level
- */
- private void dumpLocalResources() {
- if (log.isDebugEnabled()) {
- log.debug("{} resources: ", localResources.size());
- for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) {
-
- String key = entry.getKey();
- LocalResource val = entry.getValue();
- log.debug(key + "=" + SliderUtils.stringify(val.getResource()));
- }
- }
- }
-
- /**
- * This is critical for an insecure cluster -it passes
- * down the username to YARN, and so gives the code running
- * in containers the rights it needs to work with
- * data.
- * @throws IOException problems working with current user
- */
- protected void propagateUsernameInInsecureCluster() throws IOException {
- //insecure cluster: propagate user name via env variable
- String userName = UserGroupInformation.getCurrentUser().getUserName();
- env.put(SliderKeys.HADOOP_USER_NAME, userName);
- }
-
- /**
- * Extract any resource requirements from this component's settings.
- * All fields that are set will override the existing values -if
- * unset that resource field will be left unchanged.
- *
- * Important: the configuration must already be fully resolved
- * in order to pick up global options.
- * @param resource resource to configure
- * @param map map of options
- */
- public void extractResourceRequirements(Resource resource,
- Map<String, String> map) {
-
- if (map != null) {
- MapOperations options = new MapOperations("", map);
- resource.setMemory(options.getOptionInt(ResourceKeys.YARN_MEMORY,
- resource.getMemory()));
- resource.setVirtualCores(options.getOptionInt(ResourceKeys.YARN_CORES,
- resource.getVirtualCores()));
- }
- }
-
- /**
- * Extract the value for option
- * {@code yarn.resourcemanager.am.retry-count-window-ms}
- * and set it on the ApplicationSubmissionContext. Use the default value
- * if option is not set.
- *
- * @param submissionContext
- * @param map
- */
- public void extractAmRetryCount(ApplicationSubmissionContext submissionContext,
- Map<String, String> map) {
-
- if (map != null) {
- MapOperations options = new MapOperations("", map);
- long amRetryCountWindow = options.getOptionLong(ResourceKeys
- .YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS,
- ResourceKeys.DEFAULT_AM_RETRY_COUNT_WINDOW_MS);
- log.info("Setting {} to {}",
- ResourceKeys.YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS,
- amRetryCountWindow);
- submissionContext.setAttemptFailuresValidityInterval(amRetryCountWindow);
- }
- }
-
- public void extractLogAggregationContext(Map<String, String> map) {
- if (map != null) {
- String logPatternSepStr = "\\|";
- String logPatternJoinStr = "|";
- MapOperations options = new MapOperations("", map);
-
- List<String> logIncludePatterns = new ArrayList<>();
- String includePatternExpression = options.getOption(
- ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, "").trim();
- if (!includePatternExpression.isEmpty()) {
- String[] includePatterns = includePatternExpression
- .split(logPatternSepStr);
- for (String includePattern : includePatterns) {
- String trimmedIncludePattern = includePattern.trim();
- if (!trimmedIncludePattern.isEmpty()) {
- logIncludePatterns.add(trimmedIncludePattern);
- }
- }
- }
- String logIncludePattern = StringUtils.join(logIncludePatterns,
- logPatternJoinStr);
- log.info("Log include patterns: {}", logIncludePattern);
-
- List<String> logExcludePatterns = new ArrayList<>();
- String excludePatternExpression = options.getOption(
- ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, "").trim();
- if (!excludePatternExpression.isEmpty()) {
- String[] excludePatterns = excludePatternExpression
- .split(logPatternSepStr);
- for (String excludePattern : excludePatterns) {
- String trimmedExcludePattern = excludePattern.trim();
- if (!trimmedExcludePattern.isEmpty()) {
- logExcludePatterns.add(trimmedExcludePattern);
- }
- }
- }
- String logExcludePattern = StringUtils.join(logExcludePatterns,
- logPatternJoinStr);
- log.info("Log exclude patterns: {}", logExcludePattern);
-
- // SLIDER-810/YARN-3154 - hadoop 2.7.0 onwards a new instance method has
- // been added for log aggregation for LRS. Existing newInstance method's
- // behavior has changed and is used for log aggregation only after the
- // application has finished. This forces Slider users to move to hadoop
- // 2.7.0+ just for log aggregation, which is not very desirable. So we
- // decided to use reflection here to find out if the new 2.7.0 newInstance
- // method is available. If yes, then we use it, so log aggregation will
- // work in hadoop 2.7.0+ env. If no, then we fallback to the pre-2.7.0
- // newInstance method, which means log aggregation will work as expected
- // in hadoop 2.6 as well.
- // TODO: At some point, say 2-3 Slider releases down, when most users are
- // running hadoop 2.7.0, we should get rid of the reflection code here.
- try {
- Method logAggregationContextMethod = LogAggregationContext.class
- .getMethod("newInstance", String.class, String.class, String.class,
- String.class);
- // Need to set include/exclude patterns appropriately since by default
- // rolled log aggregation is not done for any files, so defaults are
- // - include pattern set to ""
- // - exclude pattern set to "*"
- // For Slider we want all logs to be uploaded if include/exclude
- // patterns are left empty by the app owner in resources file
- if (StringUtils.isEmpty(logIncludePattern)
- && StringUtils.isEmpty(logExcludePattern)) {
- logIncludePattern = ".*";
- logExcludePattern = "";
- } else if (StringUtils.isEmpty(logIncludePattern)
- && StringUtils.isNotEmpty(logExcludePattern)) {
- logIncludePattern = ".*";
- } else if (StringUtils.isNotEmpty(logIncludePattern)
- && StringUtils.isEmpty(logExcludePattern)) {
- logExcludePattern = "";
- }
- log.debug("LogAggregationContext newInstance method for rolled logs "
- + "include/exclude patterns is available");
- log.info("Modified log include patterns: {}", logIncludePattern);
- log.info("Modified log exclude patterns: {}", logExcludePattern);
- logAggregationContext = (LogAggregationContext) logAggregationContextMethod
- .invoke(null, null, null, logIncludePattern, logExcludePattern);
- } catch (NoSuchMethodException | SecurityException
- | IllegalAccessException | IllegalArgumentException
- | InvocationTargetException e) {
- log.debug("LogAggregationContext newInstance method for rolled logs "
- + "include/exclude patterns is not available - fallback to old one");
- log.debug(e.toString());
- logAggregationContext = LogAggregationContext.newInstance(
- logIncludePattern, logExcludePattern);
- }
- }
- }
-
- /**
- * Utility method to set up the classpath
- * @param classpath classpath to use
- */
- public void setClasspath(ClasspathConstructor classpath) {
- setEnv(CLASSPATH, classpath.buildClasspath());
- }
-
- /**
- * Set an environment variable in the launch context
- * @param var variable name
- * @param value value (must be non null)
- */
- public void setEnv(String var, String value) {
- Preconditions.checkArgument(var != null, "null variable name");
- Preconditions.checkArgument(value != null, "null value");
- env.put(var, value);
- }
-
- /**
- * Set an environment variable if its value is non-null.
- * @param var variable name
- * @param value value (may be null)
- */
- public void maybeSetEnv(String var, String value) {
- if (value != null) {
- setEnv(var, value);
- }
- }
-
- public void putEnv(Map<String, String> map) {
- env.putAll(map);
- }
-
- /**
- * Important: the configuration must already be fully resolved
- * in order to pick up global options
- * Copy env vars into the launch context.
- */
- public boolean copyEnvVars(MapOperations options) {
- if (options == null) {
- return false;
- }
- for (Map.Entry<String, String> entry : options.entrySet()) {
- String key = entry.getKey();
- if (key.startsWith(RoleKeys.ENV_PREFIX)) {
- key = key.substring(RoleKeys.ENV_PREFIX.length());
- env.put(key, entry.getValue());
- }
- }
- return true;
- }
-
- public String[] dumpEnvToString() {
-
- List<String> nodeEnv = new ArrayList<>();
-
- for (Map.Entry<String, String> entry : env.entrySet()) {
- String envElt = String.format("%s=\"%s\"",
- entry.getKey(),
- entry.getValue());
- log.debug(envElt);
- nodeEnv.add(envElt);
- }
- String[] envDescription = nodeEnv.toArray(new String[nodeEnv.size()]);
-
- return envDescription;
- }
-
- /**
- * Submit an entire directory
- * @param srcDir src path in filesystem
- * @param destRelativeDir relative path under destination local dir
- * @throws IOException IO problems
- */
- public void submitDirectory(Path srcDir, String destRelativeDir)
- throws IOException {
- //add the configuration resources
- Map<String, LocalResource> confResources;
- confResources = coreFileSystem.submitDirectory(
- srcDir,
- destRelativeDir);
- addLocalResources(confResources);
- }
-
- /**
- * Return the label expression and if not set null
- * @param map map to look up
- * @return extracted label or null
- */
- public String extractLabelExpression(Map<String, String> map) {
- if (map != null) {
- MapOperations options = new MapOperations("", map);
- return options.getOption(ResourceKeys.YARN_LABEL_EXPRESSION, null);
- }
- return null;
- }
-
- public void setDockerImage(String dockerImage) {
- this.dockerImage = dockerImage;
- }
-
- public void setYarnContainerMountPoints(String yarnContainerMountPoints) {
- this.yarnContainerMountPoints = yarnContainerMountPoints;
- }
-
- public void setRunPrivilegedContainer(String runPrivilegedContainer) {
- this.runPrivilegedContainer = runPrivilegedContainer;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
deleted file mode 100644
index 7190c3a..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.slider.client.SliderYarnClientImpl;
-import org.apache.slider.common.tools.CoreFileSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-
-public class AppMasterLauncher extends AbstractLauncher {
-
-
- private static final Logger log =
- LoggerFactory.getLogger(AppMasterLauncher.class);
-
- public final YarnClientApplication application;
- public final String name;
- public final String type;
- public final ApplicationSubmissionContext submissionContext;
- public final ApplicationId appId;
- public final boolean secureCluster;
- private int maxAppAttempts = 0;
- private boolean keepContainersOverRestarts = true;
- private String queue = YarnConfiguration.DEFAULT_QUEUE_NAME;
- private int priority = 1;
- private final Resource resource = Records.newRecord(Resource.class);
- private final SliderYarnClientImpl yarnClient;
- private Long submitTime;
-
- /**
- * Build the AM Launcher
- * @param name app name
- * @param type application type
- * @param conf hadoop config
- * @param fs filesystem binding
- * @param yarnClient yarn client
- * @param secureCluster flag to indicate secure cluster
- * @param options map of options. All values are extracted in this constructor only
- * @param resourceGlobalOptions global options
- * @param applicationTags any app tags
- * @param credentials initial set of credentials
- * @throws IOException
- * @throws YarnException
- */
- public AppMasterLauncher(String name,
- String type,
- Configuration conf,
- CoreFileSystem fs,
- SliderYarnClientImpl yarnClient,
- boolean secureCluster,
- Map<String, String> options,
- Map<String, String> resourceGlobalOptions,
- Set<String> applicationTags,
- Credentials credentials) throws IOException, YarnException {
- super(conf, fs, credentials);
- this.yarnClient = yarnClient;
- this.application = yarnClient.createApplication();
- this.name = name;
- this.type = type;
- this.secureCluster = secureCluster;
-
- submissionContext = application.getApplicationSubmissionContext();
- appId = submissionContext.getApplicationId();
- // set the application name;
- submissionContext.setApplicationName(name);
- // app type used in service enum;
- submissionContext.setApplicationType(type);
- if (!applicationTags.isEmpty()) {
- submissionContext.setApplicationTags(applicationTags);
- }
- submissionContext.setNodeLabelExpression(extractLabelExpression(options));
-
- extractAmRetryCount(submissionContext, resourceGlobalOptions);
- extractResourceRequirements(resource, options);
- extractLogAggregationContext(resourceGlobalOptions);
- }
-
- public void setMaxAppAttempts(int maxAppAttempts) {
- this.maxAppAttempts = maxAppAttempts;
- }
-
- public void setKeepContainersOverRestarts(boolean keepContainersOverRestarts) {
- this.keepContainersOverRestarts = keepContainersOverRestarts;
- }
-
-
- public Resource getResource() {
- return resource;
- }
-
- public void setMemory(int memory) {
- resource.setMemory(memory);
- }
-
- public void setVirtualCores(int cores) {
- resource.setVirtualCores(cores);
- }
-
- public ApplicationId getApplicationId() {
- return appId;
- }
-
- public int getMaxAppAttempts() {
- return maxAppAttempts;
- }
-
- public boolean isKeepContainersOverRestarts() {
- return keepContainersOverRestarts;
- }
-
- public String getQueue() {
- return queue;
- }
-
- public int getPriority() {
- return priority;
- }
-
- public void setQueue(String queue) {
- this.queue = queue;
- }
-
- public void setPriority(int priority) {
- this.priority = priority;
- }
-
- /**
- * Complete the launch context (copy in env vars, etc).
- * @return the container to launch
- */
- public ApplicationSubmissionContext completeAppMasterLaunch()
- throws IOException {
-
- //queue priority
- Priority pri = Records.newRecord(Priority.class);
- pri.setPriority(priority);
- submissionContext.setPriority(pri);
-
- // Set the queue to which this application is to be submitted in the RM
- // Queue for App master
-
- submissionContext.setQueue(queue);
-
-
- //container requirements
- submissionContext.setResource(resource);
- submissionContext.setLogAggregationContext(logAggregationContext);
-
- if (keepContainersOverRestarts) {
- log.debug("Requesting cluster stays running over AM failure");
- submissionContext.setKeepContainersAcrossApplicationAttempts(true);
- }
-
- if (maxAppAttempts > 0) {
- log.debug("Setting max AM attempts to {}", maxAppAttempts);
- submissionContext.setMaxAppAttempts(maxAppAttempts);
- }
-
- if (secureCluster) {
- //tokens
- log.debug("Credentials: {}",
- CredentialUtils.dumpTokens(getCredentials(), "\n"));
-
- } else {
- propagateUsernameInInsecureCluster();
- }
- completeContainerLaunch();
- submissionContext.setAMContainerSpec(containerLaunchContext);
- return submissionContext;
- }
-
- /**
- * Submit the application.
- * @return a launched application representing the submitted application
- * @throws IOException
- * @throws YarnException
- */
- public LaunchedApplication submitApplication() throws IOException, YarnException {
- completeAppMasterLaunch();
- log.info("Submitting application to Resource Manager");
- ApplicationId applicationId =
- yarnClient.submitApplication(submissionContext);
- // implicit success; record the time
- submitTime = System.currentTimeMillis();
- return new LaunchedApplication(applicationId, yarnClient);
- }
-
- /**
- * Build a serializable application report. This is a very minimal
- * report that contains the application Id, name and type \u2014the information
- * available
- * @return a data structure which can be persisted
- */
- public SerializedApplicationReport createSerializedApplicationReport() {
- SerializedApplicationReport sar = new SerializedApplicationReport();
- sar.applicationId = appId.toString();
- sar.name = name;
- sar.applicationType = type;
- sar.queue = queue;
- sar.submitTime = submitTime;
- return sar;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
deleted file mode 100644
index 6eb4058..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.common.tools.SliderUtils;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * build a classpath -allows for entries to be injected in front of
- * YARN classpath as well as behind, adds appropriate separators,
- * extraction of local classpath, etc.
- */
-public class ClasspathConstructor {
-
- public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR;
- private final List<String> pathElements = new ArrayList<>();
-
- public ClasspathConstructor() {
- }
-
-
- /**
- * Get the list of JARs from the YARN settings
- * @param config configuration
- */
- public List<String> yarnApplicationClasspath(Configuration config) {
- String[] cp = config.getTrimmedStrings(
- YarnConfiguration.YARN_APPLICATION_CLASSPATH,
- YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH);
- return cp != null ? Arrays.asList(cp) : new ArrayList<String>(0);
-
- }
-
-
- @Override
- public String toString() {
- return buildClasspath();
- }
-
- public String buildClasspath() {
- return SliderUtils.join(pathElements,
- CLASS_PATH_SEPARATOR,
- false);
- }
-
- /**
- * Get a copy of the path list
- * @return the JARs
- */
- public List<String> getPathElements() {
- return Collections.unmodifiableList(pathElements);
- }
-
- /**
- * Append an entry
- * @param path path
- */
- public void append(String path) {
- pathElements.add(path);
- }
-
- /**
- * Insert a path at the front of the list. This places it ahead of
- * the standard YARN artifacts
- * @param path path to the JAR. Absolute or relative -on the target
- * system
- */
- public void insert(String path) {
- pathElements.add(0, path);
- }
-
- public void appendAll(Collection<String> paths) {
- pathElements.addAll(paths);
- }
-
- public void insertAll(Collection<String> paths) {
- pathElements.addAll(0, paths);
- }
-
-
- public void addLibDir(String pathToLibDir) {
- append(buildLibDir(pathToLibDir));
- }
-
- public void insertLibDir(String pathToLibDir) {
- insert(buildLibDir(pathToLibDir));
- }
-
- public void addClassDirectory(String pathToDir) {
- append(appendDirectoryTerminator(pathToDir));
- }
-
- public void insertClassDirectory(String pathToDir) {
- insert(buildLibDir(appendDirectoryTerminator(pathToDir)));
- }
-
-
- public void addRemoteClasspathEnvVar() {
- append(ApplicationConstants.Environment.CLASSPATH.$$());
- }
-
-
- public void insertRemoteClasspathEnvVar() {
- append(ApplicationConstants.Environment.CLASSPATH.$$());
- }
-
-
- /**
- * Build a lib dir path
- * @param pathToLibDir path to the directory; may or may not end with a
- * trailing space
- * @return a path to a lib dir that is compatible with the java classpath
- */
- public String buildLibDir(String pathToLibDir) {
- String dir = appendDirectoryTerminator(pathToLibDir);
- dir += "*";
- return dir;
- }
-
- private String appendDirectoryTerminator(String pathToLibDir) {
- String dir = pathToLibDir.trim();
- if (!dir.endsWith("/")) {
- dir += "/";
- }
- return dir;
- }
-
- /**
- * Split a classpath. This uses the local path separator so MUST NOT
- * be used to work with remote classpaths
- * @param localpath local path
- * @return a splite
- */
- public Collection<String> splitClasspath(String localpath) {
- String separator = System.getProperty("path.separator");
- return StringUtils.getStringCollection(localpath, separator);
- }
-
- /**
- * Get the local JVM classpath split up
- * @return the list of entries on the JVM classpath env var
- */
- public Collection<String> localJVMClasspath() {
- return splitClasspath(System.getProperty("java.class.path"));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
deleted file mode 100644
index dbaa981..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.slider.common.tools.SliderUtils;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Build a single command line to include in the container commands;
- * Special support for JVM command buildup.
- */
-public class CommandLineBuilder {
- protected final List<String> argumentList = new ArrayList<>(20);
-
-
- /**
- * Add an entry to the command list
- * @param args arguments -these will be converted strings
- */
- public void add(Object... args) {
- for (Object arg : args) {
- argumentList.add(arg.toString());
- }
- }
-
- /**
- * Get the value at an offset
- * @param offset offset
- * @return the value at that point
- */
- public String elt(int offset) {
- return argumentList.get(offset);
- }
-
- /**
- * Get the number of arguments
- * @return an integer >= 0
- */
- public int size() {
- return argumentList.size();
- }
-
- /**
- * Append the output and error files to the tail of the command
- * @param stdout out
- * @param stderr error. Set this to null to append into stdout
- */
- public void addOutAndErrFiles(String stdout, String stderr) {
- Preconditions.checkNotNull(stdout, "Null output file");
- Preconditions.checkState(!stdout.isEmpty(), "output filename invalid");
- // write out the path output
- argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
- stdout);
- if (stderr != null) {
- argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
- stderr);
- } else {
- argumentList.add("2>&1");
- }
- }
-
- /**
- * This just returns the command line
- * @see #build()
- * @return the command line
- */
- @Override
- public String toString() {
- return build();
- }
-
- /**
- * Build the command line
- * @return the command line
- */
- public String build() {
- return SliderUtils.join(argumentList, " ");
- }
-
- public List<String> getArgumentList() {
- return argumentList;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
deleted file mode 100644
index d220928..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.slider.common.tools.CoreFileSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.InetSocketAddress;
-
-/**
- * Code to ease launching of any container
- */
-public class ContainerLauncher extends AbstractLauncher {
- private static final Logger log =
- LoggerFactory.getLogger(ContainerLauncher.class);
- // Allocated container
- public final Container container;
-
- public ContainerLauncher(Configuration conf,
- CoreFileSystem coreFileSystem,
- Container container,
- Credentials credentials) {
- super(conf, coreFileSystem, credentials);
- this.container = container;
- }
-
- /**
- * This code is in the dist shell examples -it's been moved here
- * so that if it is needed, it's still here
- * @return a remote user with a token to access the container.
- */
- public UserGroupInformation setupUGI() {
- UserGroupInformation user =
- UserGroupInformation.createRemoteUser(container.getId().toString());
- String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
- final InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr);
-
- org.apache.hadoop.yarn.api.records.Token containerToken = container.getContainerToken();
- if (containerToken != null) {
- Token<ContainerTokenIdentifier> token =
- ConverterUtils.convertFromYarn(containerToken, cmAddress);
- user.addToken(token);
- }
- return user;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
deleted file mode 100644
index 5357cc4..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.client.ClientRMProxy;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.conf.HAUtil;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.text.DateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hadoop.security.UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
-
-/**
- * Utils to work with credentials and tokens.
- *
- * Designed to be movable to Hadoop core
- */
-public final class CredentialUtils {
-
- public static final String JOB_CREDENTIALS_BINARY
- = SliderXmlConfKeys.MAPREDUCE_JOB_CREDENTIALS_BINARY;
-
- private CredentialUtils() {
- }
-
- private static final Logger LOG =
- LoggerFactory.getLogger(CredentialUtils.class);
-
- /**
- * Save credentials to a byte buffer. Returns null if there were no
- * credentials to save
- * @param credentials credential set
- * @return a byte buffer of serialized tokens
- * @throws IOException if the credentials could not be written to the stream
- */
- public static ByteBuffer marshallCredentials(Credentials credentials) throws IOException {
- ByteBuffer buffer = null;
- if (!credentials.getAllTokens().isEmpty()) {
- DataOutputBuffer dob = new DataOutputBuffer();
- try {
- credentials.writeTokenStorageToStream(dob);
- } finally {
- dob.close();
- }
- buffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
- }
- return buffer;
- }
-
- public static File locateEnvCredentials(Map<String, String> env,
- Configuration conf,
- StringBuffer sourceTextOut) throws FileNotFoundException {
- String tokenFilename = env.get(HADOOP_TOKEN_FILE_LOCATION);
- String source = "environment variable " + HADOOP_TOKEN_FILE_LOCATION;
- if (tokenFilename == null) {
- tokenFilename = conf.get(JOB_CREDENTIALS_BINARY);
- source = "configuration option " + JOB_CREDENTIALS_BINARY;
- }
- if (tokenFilename != null) {
- // use delegation tokens, i.e. from Oozie
- File file = new File(tokenFilename.trim());
- String details = String.format(
- "Token File %s from %s",
- file,
- source);
- if (!file.exists()) {
- throw new FileNotFoundException("No " + details);
- }
- if (!file.isFile() && !file.canRead()) {
- throw new FileNotFoundException("Cannot read " + details);
- }
- sourceTextOut.append(details);
- return file;
- } else {
- return null;
- }
- }
-
- /**
- * Load the credentials from the environment. This looks at
- * the value of {@link UserGroupInformation#HADOOP_TOKEN_FILE_LOCATION}
- * and attempts to read in the value
- * @param env environment to resolve the variable from
- * @param conf configuration use when reading the tokens
- * @return a set of credentials, or null if the environment did not
- * specify any
- * @throws IOException if a location for credentials was defined, but
- * the credentials could not be loaded.
- */
- public static Credentials loadTokensFromEnvironment(Map<String, String> env,
- Configuration conf)
- throws IOException {
- StringBuffer origin = new StringBuffer();
- File file = locateEnvCredentials(env, conf, origin);
- if (file != null) {
- LOG.debug("Using {}", origin);
- return Credentials.readTokenStorageFile(file, conf);
- } else {
- return null;
- }
- }
-
- /**
- * Save credentials to a file
- * @param file file to save to (will be overwritten)
- * @param credentials credentials to write
- * @throws IOException
- */
- public static void saveTokens(File file,
- Credentials credentials) throws IOException {
- try(DataOutputStream daos = new DataOutputStream(
- new FileOutputStream(file))) {
- credentials.writeTokenStorageToStream(daos);
- }
- }
-
- /**
- * Look up and return the resource manager's principal. This method
- * automatically does the <code>_HOST</code> replacement in the principal and
- * correctly handles HA resource manager configurations.
- *
- * From: YARN-4629
- * @param conf the {@link Configuration} file from which to read the
- * principal
- * @return the resource manager's principal string
- * @throws IOException thrown if there's an error replacing the host name
- */
- public static String getRMPrincipal(Configuration conf) throws IOException {
- String principal = conf.get(RM_PRINCIPAL, "");
- String hostname;
- Preconditions.checkState(!principal.isEmpty(), "Not set: " + RM_PRINCIPAL);
-
- if (HAUtil.isHAEnabled(conf)) {
- YarnConfiguration yarnConf = new YarnConfiguration(conf);
- if (yarnConf.get(RM_HA_ID) == null) {
- // If RM_HA_ID is not configured, use the first of RM_HA_IDS.
- // Any valid RM HA ID should work.
- String[] rmIds = yarnConf.getStrings(RM_HA_IDS);
- Preconditions.checkState((rmIds != null) && (rmIds.length > 0),
- "Not set " + RM_HA_IDS);
- yarnConf.set(RM_HA_ID, rmIds[0]);
- }
-
- hostname = yarnConf.getSocketAddr(
- RM_ADDRESS,
- DEFAULT_RM_ADDRESS,
- DEFAULT_RM_PORT).getHostName();
- } else {
- hostname = conf.getSocketAddr(
- RM_ADDRESS,
- DEFAULT_RM_ADDRESS,
- DEFAULT_RM_PORT).getHostName();
- }
- return SecurityUtil.getServerPrincipal(principal, hostname);
- }
-
- /**
- * Create and add any filesystem delegation tokens with
- * the RM(s) configured to be able to renew them. Returns null
- * on an insecure cluster (i.e. harmless)
- * @param conf configuration
- * @param fs filesystem
- * @param credentials credentials to update
- * @return a list of all added tokens.
- * @throws IOException
- */
- public static Token<?>[] addRMRenewableFSDelegationTokens(Configuration conf,
- FileSystem fs,
- Credentials credentials) throws IOException {
- Preconditions.checkArgument(conf != null);
- Preconditions.checkArgument(credentials != null);
- if (UserGroupInformation.isSecurityEnabled()) {
- return fs.addDelegationTokens(CredentialUtils.getRMPrincipal(conf),
- credentials);
- }
- return null;
- }
-
- /**
- * Add an FS delegation token which can be renewed by the current user
- * @param fs filesystem
- * @param credentials credentials to update
- * @throws IOException problems.
- */
- public static void addSelfRenewableFSDelegationTokens(
- FileSystem fs,
- Credentials credentials) throws IOException {
- Preconditions.checkArgument(fs != null);
- Preconditions.checkArgument(credentials != null);
- fs.addDelegationTokens(
- getSelfRenewer(),
- credentials);
- }
-
- public static String getSelfRenewer() throws IOException {
- return UserGroupInformation.getLoginUser().getShortUserName();
- }
-
- /**
- * Create and add an RM delegation token to the credentials
- * @param yarnClient Yarn Client
- * @param credentials to add token to
- * @return the token which was added
- * @throws IOException
- * @throws YarnException
- */
- public static Token<TokenIdentifier> addRMDelegationToken(YarnClient yarnClient,
- Credentials credentials)
- throws IOException, YarnException {
- Configuration conf = yarnClient.getConfig();
- Text rmPrincipal = new Text(CredentialUtils.getRMPrincipal(conf));
- Text rmDTService = ClientRMProxy.getRMDelegationTokenService(conf);
- Token<TokenIdentifier> rmDelegationToken =
- ConverterUtils.convertFromYarn(
- yarnClient.getRMDelegationToken(rmPrincipal),
- rmDTService);
- credentials.addToken(rmDelegationToken.getService(), rmDelegationToken);
- return rmDelegationToken;
- }
-
- public static Token<TimelineDelegationTokenIdentifier> maybeAddTimelineToken(
- Configuration conf,
- Credentials credentials)
- throws IOException, YarnException {
- if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
- LOG.debug("Timeline service enabled -fetching token");
-
- try(TimelineClient timelineClient = TimelineClient.createTimelineClient()) {
- timelineClient.init(conf);
- timelineClient.start();
- Token<TimelineDelegationTokenIdentifier> token =
- timelineClient.getDelegationToken(
- CredentialUtils.getRMPrincipal(conf));
- credentials.addToken(token.getService(), token);
- return token;
- }
- } else {
- LOG.debug("Timeline service is disabled");
- return null;
- }
- }
-
- /**
- * Filter a list of tokens from a set of credentials
- * @param credentials credential source (a new credential set os re
- * @param filter List of tokens to strip out
- * @return a new, filtered, set of credentials
- */
- public static Credentials filterTokens(Credentials credentials,
- List<Text> filter) {
- Credentials result = new Credentials(credentials);
- Iterator<Token<? extends TokenIdentifier>> iter =
- result.getAllTokens().iterator();
- while (iter.hasNext()) {
- Token<? extends TokenIdentifier> token = iter.next();
- LOG.debug("Token {}", token.getKind());
- if (filter.contains(token.getKind())) {
- LOG.debug("Filtering token {}", token.getKind());
- iter.remove();
- }
- }
- return result;
- }
-
- public static String dumpTokens(Credentials credentials, String separator) {
- ArrayList<Token<? extends TokenIdentifier>> sorted =
- new ArrayList<>(credentials.getAllTokens());
- Collections.sort(sorted, new TokenComparator());
- StringBuilder buffer = new StringBuilder(sorted.size()* 128);
- for (Token<? extends TokenIdentifier> token : sorted) {
- buffer.append(tokenToString(token)).append(separator);
- }
- return buffer.toString();
- }
-
- /**
- * Create a string for people to look at
- * @param token token to convert to a string form
- * @return a printable view of the token
- */
- public static String tokenToString(Token<? extends TokenIdentifier> token) {
- DateFormat df = DateFormat.getDateTimeInstance(
- DateFormat.SHORT, DateFormat.SHORT);
- StringBuilder buffer = new StringBuilder(128);
- buffer.append(token.toString());
- try {
- TokenIdentifier ti = token.decodeIdentifier();
- buffer.append("; ").append(ti);
- if (ti instanceof AbstractDelegationTokenIdentifier) {
- // details in human readable form, and compensate for information HDFS DT omits
- AbstractDelegationTokenIdentifier dt = (AbstractDelegationTokenIdentifier) ti;
- buffer.append("; Renewer: ").append(dt.getRenewer());
- buffer.append("; Issued: ")
- .append(df.format(new Date(dt.getIssueDate())));
- buffer.append("; Max Date: ")
- .append(df.format(new Date(dt.getMaxDate())));
- }
- } catch (IOException e) {
- //marshall problem; not ours
- LOG.debug("Failed to decode {}: {}", token, e, e);
- }
- return buffer.toString();
- }
-
- /**
- * Get the expiry time of a token.
- * @param token token to examine
- * @return the time in milliseconds after which the token is invalid.
- * @throws IOException
- */
- public static long getTokenExpiryTime(Token token) throws IOException {
- TokenIdentifier identifier = token.decodeIdentifier();
- Preconditions.checkState(identifier instanceof AbstractDelegationTokenIdentifier,
- "Token %s of type: %s has an identifier which cannot be examined: %s",
- token, token.getClass(), identifier);
- AbstractDelegationTokenIdentifier id =
- (AbstractDelegationTokenIdentifier) identifier;
- return id.getMaxDate();
- }
-
- private static class TokenComparator
- implements Comparator<Token<? extends TokenIdentifier>>, Serializable {
- @Override
- public int compare(Token<? extends TokenIdentifier> left,
- Token<? extends TokenIdentifier> right) {
- return left.getKind().toString().compareTo(right.getKind().toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
deleted file mode 100644
index b8aa4c6..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
-
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- * Command line builder purely for the Java CLI.
- * Some of the <code>define</code> methods are designed to work with Hadoop tool and
- * Slider launcher applications.
- */
-public class JavaCommandLineBuilder extends CommandLineBuilder {
-
- public JavaCommandLineBuilder() {
- add(getJavaBinary());
- }
-
- /**
- * Get the java binary. This is called in the constructor so don't try and
- * do anything other than return a constant.
- * @return the path to the Java binary
- */
- protected String getJavaBinary() {
- return ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java";
- }
-
- /**
- * Set the size of the heap if a non-empty heap is passed in.
- * @param heap empty string or something like "128M" ,"1G" etc. The value is
- * trimmed.
- */
- public void setJVMHeap(String heap) {
- if (SliderUtils.isSet(heap)) {
- add("-Xmx" + heap.trim());
- }
- }
-
- /**
- * Turn Java assertions on
- */
- public void enableJavaAssertions() {
- add("-ea");
- add("-esa");
- }
-
- /**
- * Add a system property definition -must be used before setting the main entry point
- * @param property
- * @param value
- */
- public void sysprop(String property, String value) {
- Preconditions.checkArgument(property != null, "null property name");
- Preconditions.checkArgument(value != null, "null value");
- add("-D" + property + "=" + value);
- }
-
- public JavaCommandLineBuilder forceIPv4() {
- sysprop("java.net.preferIPv4Stack", "true");
- return this;
- }
-
- public JavaCommandLineBuilder headless() {
- sysprop("java.awt.headless", "true");
- return this;
- }
-
- public boolean addConfOption(Configuration conf, String key) {
- return defineIfSet(key, conf.get(key));
- }
-
- /**
- * Add a varargs list of configuration parameters \u2014if they are present
- * @param conf configuration source
- * @param keys keys
- */
- public void addConfOptions(Configuration conf, String... keys) {
- for (String key : keys) {
- addConfOption(conf, key);
- }
- }
-
- /**
- * Add all configuration options which match the prefix
- * @param conf configuration
- * @param prefix prefix, e.g {@code "slider."}
- * @return the number of entries copied
- */
- public int addPrefixedConfOptions(Configuration conf, String prefix) {
- int copied = 0;
- for (Map.Entry<String, String> entry : conf) {
- if (entry.getKey().startsWith(prefix)) {
- define(entry.getKey(), entry.getValue());
- copied++;
- }
- }
- return copied;
- }
-
- /**
- * Ass a configuration option to the command line of the application
- * @param conf configuration
- * @param key key
- * @param defVal default value
- * @return the resolved configuration option
- * @throws IllegalArgumentException if key is null or the looked up value
- * is null (that is: the argument is missing and devVal was null.
- */
- public String addConfOptionToCLI(Configuration conf,
- String key,
- String defVal) {
- Preconditions.checkArgument(key != null, "null key");
- String val = conf.get(key, defVal);
- define(key, val);
- return val;
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI. This is very Hadoop API
- * @param key key
- * @param val value
- * @throws IllegalArgumentException if either argument is null
- */
- public void define(String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- Preconditions.checkArgument(val != null, "null value");
- add("-D", key + "=" + val);
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
- * is not null
- * @param key key
- * @param val value
- */
- public boolean defineIfSet(String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- if (val != null) {
- define(key, val);
- return true;
- } else {
- return false;
- }
- }
-
- /**
- * Add a mandatory config option
- * @param conf configuration
- * @param key key
- * @throws BadConfigException if the key is missing
- */
- public void addMandatoryConfOption(Configuration conf,
- String key) throws BadConfigException {
- if (!addConfOption(conf, key)) {
- throw new BadConfigException("Missing configuration option: " + key);
- }
- }
-
-}