You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/06/21 18:34:03 UTC
[41/50] [abbrv] hadoop git commit: YARN-6160. Create an agent-less
docker-less provider in the native services framework. Contributed by Billie
Rinaldi
YARN-6160. Create an agent-less docker-less provider in the native services framework. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34c34e8b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34c34e8b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34c34e8b
Branch: refs/heads/yarn-native-services
Commit: 34c34e8b84b3d50b1acb5b65d6ba5e5c2dcf7ee1
Parents: 4d3e875
Author: Jian He <ji...@apache.org>
Authored: Wed May 17 09:58:32 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Wed Jun 21 11:32:54 2017 -0700
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 34 ++--
.../apache/slider/client/SliderClientAPI.java | 10 --
.../org/apache/slider/common/SliderKeys.java | 18 +-
.../slider/common/tools/ConfigHelper.java | 39 -----
.../providers/AbstractClientProvider.java | 42 +++--
.../providers/AbstractProviderService.java | 169 +++++++++++++++++++
.../slider/providers/DefaultClientProvider.java | 45 +++++
.../providers/DefaultProviderFactory.java | 47 ++++++
.../providers/DefaultProviderService.java | 37 ++++
.../slider/providers/SliderProviderFactory.java | 105 ++++--------
.../slider/providers/agent/AgentKeys.java | 102 -----------
.../providers/docker/DockerClientProvider.java | 38 ++---
.../slider/providers/docker/DockerKeys.java | 3 -
.../providers/docker/DockerProviderFactory.java | 21 ++-
.../providers/docker/DockerProviderService.java | 140 +--------------
.../tarball/TarballClientProvider.java | 65 +++++++
.../tarball/TarballProviderFactory.java | 52 ++++++
.../tarball/TarballProviderService.java | 50 ++++++
.../server/appmaster/RoleLaunchService.java | 19 +--
.../server/appmaster/SliderAppMaster.java | 64 +++----
.../slider/server/appmaster/web/WebAppApi.java | 6 -
.../server/appmaster/web/WebAppApiImpl.java | 11 +-
.../appmaster/web/view/SliderHamletBlock.java | 3 -
.../main/resources/org/apache/slider/slider.xml | 30 ----
.../slider/providers/TestProviderFactory.java | 40 +++--
.../web/view/TestClusterSpecificationBlock.java | 4 -
.../web/view/TestContainerStatsBlock.java | 4 -
.../appmaster/web/view/TestIndexBlock.java | 4 -
28 files changed, 664 insertions(+), 538 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 83b4841..32d78b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -133,7 +133,6 @@ import org.apache.slider.core.zk.BlockingZKWatcher;
import org.apache.slider.core.zk.ZKIntegration;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.ProviderUtils;
-import org.apache.slider.providers.SliderProviderFactory;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.rpc.RpcBinder;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
@@ -257,8 +256,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
config = super.bindArgs(config, args);
serviceArgs = new ClientArgs(args);
serviceArgs.parse();
- // add the slider XML config
- ConfigHelper.injectSliderXMLResource();
// yarn-ify
YarnConfiguration yarnConfiguration = new YarnConfiguration(config);
return patchConfiguration(yarnConfiguration);
@@ -622,14 +619,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- @Override
- public AbstractClientProvider createClientProvider(String provider)
- throws SliderException {
- SliderProviderFactory factory =
- SliderProviderFactory.createSliderProviderFactory(provider);
- return factory.createClientProvider();
- }
-
private Application getApplicationFromArgs(String clusterName,
AbstractClusterBuildingActionArgs args) throws IOException {
File file = args.getAppDef();
@@ -893,7 +882,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private void persistApp(Path appDir, Application application)
throws IOException, SliderException {
- FsPermission appDirPermission = new FsPermission("777");
+ FsPermission appDirPermission = new FsPermission("750");
sliderFileSystem.createWithPermissions(appDir, appDirPermission);
Path appJson = new Path(appDir, application.getName() + ".json");
jsonSerDeser
@@ -1190,17 +1179,18 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
+ // TODO handle client install
// Only INSTALL is supported
- AbstractClientProvider
- provider = createClientProvider(SliderProviderFactory.DEFAULT_CLUSTER_TYPE);
- provider.processClientOperation(sliderFileSystem,
- getRegistryOperations(),
- getConfig(),
- "INSTALL",
- clientInfo.installLocation,
- pkgFile,
- config,
- clientInfo.name);
+ // ClientProvider
+ // provider = createClientProvider(SliderProviderFactory.DEFAULT_CLUSTER_TYPE);
+ // provider.processClientOperation(sliderFileSystem,
+ // getRegistryOperations(),
+ // getConfig(),
+ // "INSTALL",
+ // clientInfo.installLocation,
+ // pkgFile,
+ // config,
+ // clientInfo.name);
return EXIT_SUCCESS;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
index 197a564..7477c05 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -42,7 +42,6 @@ import org.apache.slider.common.params.ActionThawArgs;
import org.apache.slider.common.params.ActionUpgradeArgs;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.providers.AbstractClientProvider;
import java.io.IOException;
@@ -63,15 +62,6 @@ public interface SliderClientAPI extends Service {
ActionAMSuicideArgs args) throws YarnException, IOException;
/**
- * Get the provider for this cluster
- * @param provider the name of the provider
- * @return the provider instance
- * @throws SliderException problems building the provider
- */
- AbstractClientProvider createClientProvider(String provider)
- throws SliderException;
-
- /**
* Manage keytabs leveraged by slider
*
* @param keytabInfo the arguments needed to manage the keytab
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 968a90b..734fec5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -18,8 +18,6 @@
package org.apache.slider.common;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -158,12 +156,7 @@ public interface SliderKeys extends SliderXmlConfKeys {
*/
String HISTORY_FILENAME_GLOB_PATTERN = HISTORY_FILENAME_PREFIX +"*."+
HISTORY_FILENAME_SUFFIX;
- /**
- * XML resource listing the standard Slider providers
- * {@value}
- */
- String SLIDER_XML = "org/apache/slider/slider.xml";
-
+
String CLUSTER_DIRECTORY = "cluster";
String PACKAGE_DIRECTORY = "package";
@@ -311,5 +304,12 @@ public interface SliderKeys extends SliderXmlConfKeys {
String APP_RESOURCES = "application.resources";
String APP_RESOURCES_DIR = "app/resources";
- String APP_PACKAGES_DIR = "app/packages";
+ String APP_INSTALL_DIR = "app/install";
+
+ String OUT_FILE = "stdout.txt";
+ String ERR_FILE = "stderr.txt";
+
+ String QUICK_LINKS = "quicklinks";
+
+ String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
index 0e94a29..25debdc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
@@ -50,7 +50,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
-import java.util.concurrent.atomic.AtomicBoolean;
/**
* Methods to aid in config, both in the Configuration class and
@@ -62,11 +61,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
public class ConfigHelper {
private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);
- private static AtomicBoolean sliderResourceInjected =
- new AtomicBoolean(false);
- private static AtomicBoolean sliderResourceInjectionAttempted =
- new AtomicBoolean(false);
-
/**
* Dump the (sorted) configuration
* @param conf config
@@ -614,37 +608,4 @@ public class ConfigHelper {
public static void registerDeprecatedConfigItems() {
}
- /**
- * Load a configuration with the {@link SliderKeys#SLIDER_XML} resource
- * included
- * @return a configuration instance
- */
- public static Configuration loadSliderConfiguration() {
- Configuration conf = new Configuration();
- conf.addResource(SliderKeys.SLIDER_XML);
- return conf;
- }
-
- /**
- * Inject the {@link SliderKeys#SLIDER_XML} resource
- * into the configuration resources <i>of all configurations</i>.
- * <p>
- * This operation is idempotent.
- * <p>
- * If the resource is not on the classpath, downgrades, rather than
- * fails.
- * @return true if the resource was found and loaded.
- */
- public static synchronized boolean injectSliderXMLResource() {
- if (sliderResourceInjectionAttempted.getAndSet(true)) {
- return sliderResourceInjected.get();
- }
- URL resourceUrl = getResourceUrl(SliderKeys.SLIDER_XML);
- if (resourceUrl != null) {
- Configuration.addDefaultResource(SliderKeys.SLIDER_XML);
- sliderResourceInjected.set(true);
- }
- return sliderResourceInjected.get();
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
index df174f5..185dcd4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
@@ -19,32 +19,26 @@
package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.SliderException;
import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.File;
+import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-public abstract class AbstractClientProvider extends Configured {
- private static final Logger log =
- LoggerFactory.getLogger(AbstractClientProvider.class);
+public abstract class AbstractClientProvider {
- public AbstractClientProvider(Configuration conf) {
- super(conf);
+ public AbstractClientProvider() {
}
- public abstract String getName();
-
- public abstract List<ProviderRole> getRoles();
-
/**
* Generates a fixed format of application tags given one or more of
* application name, version and description. This allows subsequent query for
@@ -70,7 +64,29 @@ public abstract class AbstractClientProvider extends Configured {
}
/**
- * Process client operations for applications such as install, configure
+ * Validate the artifact.
+ * @param artifact
+ */
+ public abstract void validateArtifact(Artifact artifact, FileSystem
+ fileSystem) throws IOException;
+
+ protected abstract void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException;
+
+ /**
+ * Validate the config files.
+ * @param configFiles config file list
+ * @param fileSystem file system
+ */
+ public void validateConfigFiles(List<ConfigFile> configFiles, FileSystem
+ fileSystem) throws IOException {
+ for (ConfigFile configFile : configFiles) {
+ validateConfigFile(configFile, fileSystem);
+ }
+ }
+
+ /**
+ * Process client operations for applications such as install, configure.
* @param fileSystem
* @param registryOperations
* @param configuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
new file mode 100644
index 0000000..b7fa802
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers;
+
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.api.resource.ContainerState;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.CommandLineBuilder;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
+import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static org.apache.slider.util.ServiceApiUtil.$;
+
+public abstract class AbstractProviderService extends AbstractService
+ implements ProviderService, SliderKeys {
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(AbstractProviderService.class);
+ private static final ProviderUtils providerUtils = new ProviderUtils(log);
+ protected StateAccessForProviders amState;
+ protected YarnRegistryViewForProviders yarnRegistry;
+ private ServiceTimelinePublisher serviceTimelinePublisher;
+
+ protected AbstractProviderService(String name) {
+ super(name);
+ }
+
+ public abstract void processArtifact(ContainerLauncher launcher, Component
+ component, SliderFileSystem fileSystem) throws IOException;
+
+ @Override
+ public void setAMState(StateAccessForProviders stateAccessor) {
+ this.amState = stateAccessor;
+ }
+
+ @Override
+ public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) {
+ this.yarnRegistry = yarnRegistry;
+ }
+
+ public void buildContainerLaunchContext(ContainerLauncher launcher,
+ Application application, Container container, ProviderRole providerRole,
+ SliderFileSystem fileSystem, RoleInstance roleInstance)
+ throws IOException, SliderException {
+ Component component = providerRole.component;
+ processArtifact(launcher, component, fileSystem);
+
+ // Generate tokens (key-value pair) for config substitution.
+ // Get pre-defined tokens
+ Map<String, String> globalTokens = amState.getGlobalSubstitutionTokens();
+ Map<String, String> tokensForSubstitution = providerUtils
+ .initCompTokensForSubstitute(roleInstance);
+ tokensForSubstitution.putAll(globalTokens);
+ // Set the environment variables in launcher
+ launcher.putEnv(SliderUtils
+ .buildEnvMap(component.getConfiguration(), tokensForSubstitution));
+ launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
+ launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ if (System.getenv(HADOOP_USER_NAME) != null) {
+ launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
+ }
+ launcher.setEnv("LANG", "en_US.UTF-8");
+ launcher.setEnv("LC_ALL", "en_US.UTF-8");
+ launcher.setEnv("LANGUAGE", "en_US.UTF-8");
+
+ for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
+ tokensForSubstitution.put($(entry.getKey()), entry.getValue());
+ }
+ providerUtils.addComponentHostTokens(tokensForSubstitution, amState);
+
+ // create config file on hdfs and add local resource
+ providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
+ component, tokensForSubstitution, roleInstance, amState);
+
+ // substitute launch command
+ String launchCommand = ProviderUtils
+ .substituteStrWithTokens(component.getLaunchCommand(),
+ tokensForSubstitution);
+ CommandLineBuilder operation = new CommandLineBuilder();
+ operation.add(launchCommand);
+ operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
+ launcher.addCommand(operation.build());
+
+ // publish exports
+ providerUtils
+ .substituteMapWithTokens(application.getQuicklinks(), tokensForSubstitution);
+ PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS,
+ application.getQuicklinks().entrySet());
+ amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf);
+ if (serviceTimelinePublisher != null) {
+ serviceTimelinePublisher.serviceAttemptUpdated(application);
+ }
+ }
+
+ public boolean processContainerStatus(ContainerId containerId,
+ ContainerStatus status) {
+ log.debug("Handling container status: {}", status);
+ if (SliderUtils.isEmpty(status.getIPs()) ||
+ SliderUtils.isUnset(status.getHost())) {
+ return true;
+ }
+ RoleInstance instance = amState.getOwnedContainer(containerId);
+ if (instance == null) {
+ // container is completed?
+ return false;
+ }
+
+ try {
+ providerUtils.updateServiceRecord(amState, yarnRegistry,
+ containerId.toString(), instance.role, status.getIPs(), status.getHost());
+ } catch (IOException e) {
+ // could not write service record to ZK, log and retry
+ log.warn("Error updating container {} service record in registry, " +
+ "retrying", containerId, e);
+ return true;
+ }
+ // TODO publish ip and host
+ org.apache.slider.api.resource.Container container =
+ instance.providerRole.component.getContainer(containerId.toString());
+ if (container != null) {
+ container.setIp(StringUtils.join(",", status.getIPs()));
+ container.setHostname(status.getHost());
+ container.setState(ContainerState.READY);
+ } else {
+ log.warn(containerId + " not found in Application!");
+ }
+ return false;
+ }
+
+ @Override
+ public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) {
+ this.serviceTimelinePublisher = publisher;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java
new file mode 100644
index 0000000..004e81f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.ConfigFile;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+public class DefaultClientProvider extends AbstractClientProvider {
+
+ public DefaultClientProvider() {
+ }
+
+ @Override
+ public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
+ }
+
+ @Override
+ protected void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException {
+ // validate dest_file is not absolute
+ if (Paths.get(configFile.getDestFile()).isAbsolute()) {
+ throw new IllegalArgumentException(
+ "Dest_file must not be absolute path: " + configFile.getDestFile());
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java
new file mode 100644
index 0000000..09a1423
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers;
+
+public final class DefaultProviderFactory extends SliderProviderFactory {
+ private static final SliderProviderFactory FACTORY = new
+ DefaultProviderFactory();
+
+ private DefaultProviderFactory() {}
+
+ private static class Client {
+ static final AbstractClientProvider PROVIDER = new DefaultClientProvider();
+ }
+
+ private static class Server {
+ static final ProviderService PROVIDER = new DefaultProviderService();
+ }
+
+ @Override
+ public AbstractClientProvider createClientProvider() {
+ return Client.PROVIDER;
+ }
+
+ @Override
+ public ProviderService createServerProvider() {
+ return Server.PROVIDER;
+ }
+
+ public static SliderProviderFactory getInstance() {
+ return FACTORY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java
new file mode 100644
index 0000000..7f7d209
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers;
+
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.providers.AbstractProviderService;
+
+import java.io.IOException;
+
+public class DefaultProviderService extends AbstractProviderService {
+
+ protected DefaultProviderService() {
+ super(DefaultProviderService.class.getSimpleName());
+ }
+
+ @Override
+ public void processArtifact(ContainerLauncher launcher, Component
+ component, SliderFileSystem fileSystem) throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
index 5dd4a32..9c52643 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
@@ -18,93 +18,60 @@
package org.apache.slider.providers;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.api.resource.Artifact;
import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.providers.agent.AgentKeys;
+import org.apache.slider.providers.docker.DockerProviderFactory;
+import org.apache.slider.providers.tarball.TarballProviderFactory;
+import org.apache.slider.util.RestApiErrorMessages;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Base class for factories
+ * Base class for factories.
*/
-public abstract class SliderProviderFactory extends Configured {
-
- public static final String DEFAULT_CLUSTER_TYPE = AgentKeys.PROVIDER_AGENT;
-
+public abstract class SliderProviderFactory {
protected static final Logger log =
- LoggerFactory.getLogger(SliderProviderFactory.class);
- public static final String PROVIDER_NOT_FOUND =
- "Unable to find provider of application type %s";
-
- public SliderProviderFactory(Configuration conf) {
- super(conf);
- }
+ LoggerFactory.getLogger(SliderProviderFactory.class);
- protected SliderProviderFactory() {
- }
+ protected SliderProviderFactory() {}
public abstract AbstractClientProvider createClientProvider();
public abstract ProviderService createServerProvider();
+ public static synchronized ProviderService getProviderService(Artifact
+ artifact) {
+ return createSliderProviderFactory(artifact).createServerProvider();
+ }
+
+ public static synchronized AbstractClientProvider getClientProvider(Artifact
+ artifact) {
+ return createSliderProviderFactory(artifact).createClientProvider();
+ }
+
/**
* Create a provider for a specific application
- * @param application app
- * @return app instance
+ * @param artifact artifact
+ * @return provider factory
* @throws SliderException on any instantiation problem
*/
- public static SliderProviderFactory createSliderProviderFactory(String application) throws
- SliderException {
- Configuration conf = loadSliderConfiguration();
- if (application == null) {
- application = DEFAULT_CLUSTER_TYPE;
+ public static synchronized SliderProviderFactory createSliderProviderFactory(
+ Artifact artifact) {
+ if (artifact == null || artifact.getType() == null) {
+ log.info("Loading service provider type default");
+ return DefaultProviderFactory.getInstance();
}
- String providerKey =
- String.format(SliderXmlConfKeys.KEY_PROVIDER, application);
- if (application.contains(".")) {
- log.debug("Treating {} as a classname", application);
- String name = "classname.key";
- conf.set(name, application);
- providerKey = name;
+ log.info("Loading service provider type {}", artifact.getType());
+ switch (artifact.getType()) {
+ // TODO add handling for custom types?
+ // TODO handle application
+ case DOCKER:
+ return DockerProviderFactory.getInstance();
+ case TARBALL:
+ return TarballProviderFactory.getInstance();
+ default:
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
}
-
- Class<? extends SliderProviderFactory> providerClass;
- try {
- providerClass = conf.getClass(providerKey, null, SliderProviderFactory.class);
- } catch (RuntimeException e) {
- throw new BadClusterStateException(e, "Failed to load provider %s: %s", application, e);
- }
- if (providerClass == null) {
- throw new BadClusterStateException(PROVIDER_NOT_FOUND, application);
- }
-
- Exception ex;
- try {
- SliderProviderFactory providerFactory = providerClass.newInstance();
- providerFactory.setConf(conf);
- return providerFactory;
- } catch (Exception e) {
- ex = e;
- }
- //by here the operation failed and ex is set to the value
- throw new BadClusterStateException(ex,
- "Failed to create an instance of %s : %s",
- providerClass,
- ex);
- }
-
- /**
- * Load a configuration with the {@link SliderKeys#SLIDER_XML} resource
- * included
- * @return a configuration instance
- */
- public static Configuration loadSliderConfiguration() {
- Configuration conf = new Configuration();
- conf.addResource(SliderKeys.SLIDER_XML);
- return conf;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
deleted file mode 100644
index c4228e4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-/*
-
- */
-public interface AgentKeys {
-
- String AGENT_TAR = "slider-agent.tar.gz";
- String PROVIDER_AGENT = "agent";
- String ROLE_NODE = "echo";
-
- /**
- * Template stored in the slider classpath -to use if there is
- * no site-specific template
- * {@value}
- */
- String CONF_RESOURCE = "org/apache/slider/providers/agent/conf/";
- /* URL to talk back to Agent Controller*/
- String CONTROLLER_URL = "agent.controller.url";
- /**
- * The location of pre-installed agent path.
- * This can be also be dynamically computed based on Yarn installation of agent.
- */
- String PACKAGE_PATH = "agent.package.root";
- /**
- * The location of the script implementing the command.
- */
- String SCRIPT_PATH = "agent.script";
- /**
- * Execution home for the agent.
- */
- String APP_HOME = "app.home";
- String APP_ROOT = "site.global.app_root";
- String APP_CLIENT_ROOT = "client_root";
- /**
- * Runas user of the application
- */
- String RUNAS_USER = "site.global.app_user";
- /**
- * Name of the service.
- */
- String SERVICE_NAME = "app.name";
- String ARG_LABEL = "--label";
- String ARG_HOST = "--host";
- String ARG_PORT = "--port";
- String ARG_SECURED_PORT = "--secured_port";
- String ARG_ZOOKEEPER_QUORUM = "--zk-quorum";
- String ARG_ZOOKEEPER_REGISTRY_PATH = "--zk-reg-path";
- String ARG_DEBUG = "--debug";
- String AGENT_MAIN_SCRIPT_ROOT = "./infra/agent/slider-agent/";
- String AGENT_JINJA2_ROOT = "./infra/agent/slider-agent/jinja2";
- String AGENT_MAIN_SCRIPT = "agent/main.py";
-
- String APP_DEF = "application.def";
- String APP_DEF_ORIGINAL = "application.def.original";
- String ADDON_PREFIX = "application.addon.";
- String ADDONS = "application.addons";
- String AGENT_VERSION = "agent.version";
- String AGENT_CONF = "agent.conf";
- String ADDON_FOR_ALL_COMPONENTS = "ALL";
-
- String AGENT_INSTALL_DIR = "infra/agent";
- String APP_DEFINITION_DIR = "app/definition";
- String ADDON_DEFINITION_DIR = "addon/definition";
- String AGENT_CONFIG_FILE = "infra/conf/agent.ini";
- String AGENT_VERSION_FILE = "infra/version";
-
- String PACKAGE_LIST = "package_list";
- String WAIT_HEARTBEAT = "wait.heartbeat";
- String PYTHON_EXE = "python";
- String CREATE_DEF_ZK_NODE = "create.default.zookeeper.node";
- String HEARTBEAT_MONITOR_INTERVAL = "heartbeat.monitor.interval";
- String AGENT_INSTANCE_DEBUG_DATA = "agent.instance.debug.data";
- String AGENT_OUT_FILE = "slider-agent.out";
- String KEY_AGENT_TWO_WAY_SSL_ENABLED = "ssl.server.client.auth";
- String INFRA_RUN_SECURITY_DIR = "infra/run/security/";
- String CERT_FILE_LOCALIZATION_PATH = INFRA_RUN_SECURITY_DIR + "ca.crt";
- String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
- String TEST_RELAX_VERIFICATION = "test.relax.validation";
-
- String DEFAULT_METAINFO_MAP_KEY = "DEFAULT_KEY";
-}
-
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
index 1d5d8a0..4773327 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
@@ -17,37 +17,37 @@
*/
package org.apache.slider.providers.docker;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.providers.AbstractClientProvider;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.providers.ProviderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.slider.util.RestApiErrorMessages;
-import java.util.Collections;
-import java.util.List;
+import java.io.IOException;
public class DockerClientProvider extends AbstractClientProvider
implements SliderKeys {
- protected static final Logger log =
- LoggerFactory.getLogger(DockerClientProvider.class);
- private static final ProviderUtils providerUtils = new ProviderUtils(log);
- protected static final String NAME = "docker";
-
- public DockerClientProvider(Configuration conf) {
- super(conf);
+ public DockerClientProvider() {
+ super();
}
@Override
- public String getName() {
- return NAME;
+ public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
+ if (artifact == null) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
+ }
+ if (StringUtils.isEmpty(artifact.getId())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+ }
}
@Override
- public List<ProviderRole> getRoles() {
- return Collections.emptyList();
+ protected void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException {
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
index 0e1d288..7413ebb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
@@ -27,7 +27,4 @@ public interface DockerKeys {
String DEFAULT_DOCKER_NETWORK = "bridge";
Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false;
-
- String OUT_FILE = "stdout.txt";
- String ERR_FILE = "stderr.txt";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java
index 5d2592f..6977e41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java
@@ -17,27 +17,36 @@
*/
package org.apache.slider.providers.docker;
-import org.apache.hadoop.conf.Configuration;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.SliderProviderFactory;
public class DockerProviderFactory extends SliderProviderFactory {
+ private static final SliderProviderFactory FACTORY = new
+ DockerProviderFactory();
- public DockerProviderFactory() {
+ private DockerProviderFactory() {
}
- public DockerProviderFactory(Configuration conf) {
- super(conf);
+ private static class Client {
+ static final AbstractClientProvider PROVIDER = new DockerClientProvider();
+ }
+
+ private static class Server {
+ static final ProviderService PROVIDER = new DockerProviderService();
}
@Override
public AbstractClientProvider createClientProvider() {
- return new DockerClientProvider(getConf());
+ return Client.PROVIDER;
}
@Override
public ProviderService createServerProvider() {
- return new DockerProviderService();
+ return Server.PROVIDER;
+ }
+
+ public static SliderProviderFactory getInstance() {
+ return FACTORY;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index a48bf83..f35d4d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -17,159 +17,31 @@
*/
package org.apache.slider.providers.docker;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.ContainerState;
-import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.CommandLineBuilder;
import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.providers.ProviderService;
-import org.apache.slider.providers.ProviderUtils;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
-import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
+import org.apache.slider.providers.AbstractProviderService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.Map;
-import java.util.Map.Entry;
-import static org.apache.slider.util.ServiceApiUtil.$;
-
-public class DockerProviderService extends AbstractService
- implements ProviderService, DockerKeys, SliderKeys {
+public class DockerProviderService extends AbstractProviderService
+ implements DockerKeys {
protected static final Logger log =
LoggerFactory.getLogger(DockerProviderService.class);
- private static final ProviderUtils providerUtils = new ProviderUtils(log);
- private static final String QUICK_LINKS = "quicklinks";
- protected StateAccessForProviders amState;
- protected YarnRegistryViewForProviders yarnRegistry;
- private ServiceTimelinePublisher serviceTimelinePublisher;
protected DockerProviderService() {
- super("DockerProviderService");
- }
-
- @Override
- public void setAMState(StateAccessForProviders stateAccessor) {
- this.amState = stateAccessor;
- }
-
- @Override
- public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) {
- this.yarnRegistry = yarnRegistry;
+ super(DockerProviderService.class.getSimpleName());
}
-
- public void buildContainerLaunchContext(ContainerLauncher launcher,
- Application application, Container container, ProviderRole providerRole,
- SliderFileSystem fileSystem, RoleInstance roleInstance)
- throws IOException, SliderException {
- Component component = providerRole.component;
+ public void processArtifact(ContainerLauncher launcher, Component
+ component, SliderFileSystem fileSystem) throws IOException {
launcher.setYarnDockerMode(true);
launcher.setDockerImage(component.getArtifact().getId());
launcher.setDockerNetwork(component.getConfiguration()
.getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK));
launcher.setRunPrivilegedContainer(component.getRunPrivilegedContainer());
-
- // Generate tokens (key-value pair) for config substitution.
- // Get pre-defined tokens
- Map<String, String> globalTokens = amState.getGlobalSubstitutionTokens();
- Map<String, String> tokensForSubstitution = providerUtils
- .initCompTokensForSubstitute(roleInstance);
- tokensForSubstitution.putAll(globalTokens);
- // Set the environment variables in launcher
- launcher.putEnv(SliderUtils
- .buildEnvMap(component.getConfiguration(), tokensForSubstitution));
- launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
- launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
- if (System.getenv(HADOOP_USER_NAME) != null) {
- launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
- }
- launcher.setEnv("LANG", "en_US.UTF-8");
- launcher.setEnv("LC_ALL", "en_US.UTF-8");
- launcher.setEnv("LANGUAGE", "en_US.UTF-8");
-
- for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
- tokensForSubstitution.put($(entry.getKey()), entry.getValue());
- }
- providerUtils.addComponentHostTokens(tokensForSubstitution, amState);
-
- // create config file on hdfs and add local resource
- providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
- component, tokensForSubstitution, roleInstance, amState);
-
- // substitute launch command
- String launchCommand = ProviderUtils
- .substituteStrWithTokens(component.getLaunchCommand(),
- tokensForSubstitution);
- CommandLineBuilder operation = new CommandLineBuilder();
- operation.add(launchCommand);
- operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
- launcher.addCommand(operation.build());
-
- // publish exports
- providerUtils
- .substituteMapWithTokens(application.getQuicklinks(), tokensForSubstitution);
- PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS,
- application.getQuicklinks().entrySet());
- amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf);
- if (serviceTimelinePublisher != null) {
- serviceTimelinePublisher.serviceAttemptUpdated(application);
- }
- }
-
- public boolean processContainerStatus(ContainerId containerId,
- ContainerStatus status) {
- log.debug("Handling container status: {}", status);
- if (SliderUtils.isEmpty(status.getIPs()) ||
- SliderUtils.isUnset(status.getHost())) {
- return true;
- }
- RoleInstance instance = amState.getOwnedContainer(containerId);
- if (instance == null) {
- // container is completed?
- return false;
- }
-
- try {
- providerUtils.updateServiceRecord(amState, yarnRegistry,
- containerId.toString(), instance.role, status.getIPs(), status.getHost());
- } catch (IOException e) {
- // could not write service record to ZK, log and retry
- log.warn("Error updating container {} service record in registry, " +
- "retrying", containerId, e);
- return true;
- }
- // TODO publish ip and host
- org.apache.slider.api.resource.Container container =
- instance.providerRole.component.getContainer(containerId.toString());
- if (container != null) {
- container.setIp(StringUtils.join(",", status.getIPs()));
- container.setHostname(status.getHost());
- container.setState(ContainerState.READY);
- } else {
- log.warn(containerId + " not found in Application!");
- }
- return false;
- }
-
- @Override
- public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) {
- this.serviceTimelinePublisher = publisher;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java
new file mode 100644
index 0000000..3e020cf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.tarball;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.ConfigFile;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.util.RestApiErrorMessages;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+public class TarballClientProvider extends AbstractClientProvider
+ implements SliderKeys {
+
+ public TarballClientProvider() {
+ }
+
+ @Override
+ public void validateArtifact(Artifact artifact, FileSystem fs)
+ throws IOException {
+ if (artifact == null) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
+ }
+ if (StringUtils.isEmpty(artifact.getId())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+ }
+ Path p = new Path(artifact.getId());
+ if (!fs.exists(p)) {
+ throw new IllegalArgumentException( "Artifact tarball does not exist "
+ + artifact.getId());
+ }
+ }
+
+ @Override
+ protected void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException {
+ // validate dest_file is not absolute
+ if (Paths.get(configFile.getDestFile()).isAbsolute()) {
+ throw new IllegalArgumentException(
+ "Dest_file must not be absolute path: " + configFile.getDestFile());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java
new file mode 100644
index 0000000..d9b7450
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.tarball;
+
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.ProviderService;
+import org.apache.slider.providers.SliderProviderFactory;
+
+public class TarballProviderFactory extends SliderProviderFactory {
+ private static final SliderProviderFactory FACTORY = new
+ TarballProviderFactory();
+
+ private TarballProviderFactory() {
+ }
+
+ private static class Client {
+ static final AbstractClientProvider PROVIDER = new TarballClientProvider();
+ }
+
+ private static class Server {
+ static final ProviderService PROVIDER = new TarballProviderService();
+ }
+
+ @Override
+ public AbstractClientProvider createClientProvider() {
+ return Client.PROVIDER;
+ }
+
+ @Override
+ public ProviderService createServerProvider() {
+ return Server.PROVIDER;
+ }
+
+ public static SliderProviderFactory getInstance() {
+ return FACTORY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java
new file mode 100644
index 0000000..65a55f0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.tarball;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.providers.AbstractProviderService;
+
+import java.io.IOException;
+
+public class TarballProviderService extends AbstractProviderService {
+
+ protected TarballProviderService() {
+ super(TarballProviderService.class.getSimpleName());
+ }
+
+ @Override
+ public void processArtifact(ContainerLauncher launcher, Component
+ component, SliderFileSystem fileSystem) throws IOException {
+ Path artifact = new Path(component.getArtifact().getId());
+ if (!fileSystem.isFile(artifact)) {
+ throw new IOException("Package doesn't exist as a resource: " +
+ artifact.toString());
+ }
+ log.info("Adding resource {}", artifact.toString());
+ LocalResourceType type = LocalResourceType.ARCHIVE;
+ LocalResource packageResource = fileSystem.createAmResource(
+ artifact, type);
+ launcher.addLocalResource(APP_INSTALL_DIR, packageResource);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index 7c096c7..aa84940 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -27,7 +27,7 @@ import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
-import org.apache.slider.providers.agent.AgentKeys;
+import org.apache.slider.providers.SliderProviderFactory;
import org.apache.slider.server.appmaster.actions.ActionStartContainer;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.ContainerAssignment;
@@ -43,6 +43,8 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import static org.apache.slider.common.SliderKeys.KEY_CONTAINER_LAUNCH_DELAY;
+
/**
* A service for launching containers
*/
@@ -60,11 +62,6 @@ public class RoleLaunchService
private final QueueAccess actionQueue;
/**
- * Provider building up the command
- */
- private final ProviderService provider;
-
- /**
* Filesystem to use for the launch
*/
private final SliderFileSystem fs;
@@ -75,16 +72,14 @@ public class RoleLaunchService
/**
* Construct an instance of the launcher
* @param queueAccess
- * @param provider the provider
* @param fs filesystem
* @param envVars environment variables
*/
- public RoleLaunchService(QueueAccess queueAccess, ProviderService provider,
- SliderFileSystem fs, Map<String, String> envVars) {
+ public RoleLaunchService(QueueAccess queueAccess, SliderFileSystem fs,
+ Map<String, String> envVars) {
super(ROLE_LAUNCH_SERVICE);
this.actionQueue = queueAccess;
this.fs = fs;
- this.provider = provider;
this.envVars = envVars;
}
@@ -167,11 +162,13 @@ public class RoleLaunchService
instance.roleId = role.id;
instance.environment = envDescription;
+ ProviderService provider = SliderProviderFactory.getProviderService(
+ role.component.getArtifact());
provider.buildContainerLaunchContext(containerLauncher, application,
container, role, fs, instance);
long delay = role.component.getConfiguration()
- .getPropertyLong(AgentKeys.KEY_CONTAINER_LAUNCH_DELAY, 0);
+ .getPropertyLong(KEY_CONTAINER_LAUNCH_DELAY, 0);
long maxDelay = getConfig()
.getLong(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 02c9198..84dde08 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.TimelineV2Client;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
@@ -342,7 +341,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* ProviderService of this cluster
*/
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private ProviderService providerService;
+ private List<ProviderService> providers = new ArrayList<>();
/**
* The YARN registry service
@@ -523,8 +522,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public Configuration bindArgs(Configuration config, String... args) throws Exception {
// let the superclass process it
Configuration superConf = super.bindArgs(config, args);
- // add the slider XML config
- ConfigHelper.injectSliderXMLResource();
//yarn-ify
YarnConfiguration yarnConfiguration = new YarnConfiguration(
@@ -603,12 +600,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// obtain security state
// set the global security flag for the instance definition
- //get our provider
- SliderProviderFactory factory =
- SliderProviderFactory.createSliderProviderFactory("docker");
- providerService = factory.createServerProvider();
- // init the provider BUT DO NOT START IT YET
- initAndAddService(providerService);
+ // initialize our providers
+ for (Component component : application.getComponents()) {
+ SliderProviderFactory factory = SliderProviderFactory
+ .createSliderProviderFactory(component.getArtifact());
+ ProviderService providerService = factory.createServerProvider();
+ // init the provider BUT DO NOT START IT YET
+ initAndAddService(providerService);
+ providers.add(providerService);
+ }
InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf);
log.info("RM is at {}", rmSchedulerAddress);
@@ -667,7 +667,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
serviceTimelinePublisher.init(getConfig());
serviceTimelinePublisher.start();
- providerService.setServiceTimelinePublisher(serviceTimelinePublisher);
+ for (ProviderService providerService : providers) {
+ providerService.setServiceTimelinePublisher(serviceTimelinePublisher);
+ }
appState.setServiceTimelinePublisher(serviceTimelinePublisher);
log.info("ServiceTimelinePublisher started.");
}
@@ -707,7 +709,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
WebAppApiImpl webAppApi =
new WebAppApiImpl(
stateForProviders,
- providerService, registryOperations,
+ registryOperations,
metricsAndMonitoring,
actionQueues);
initAMFilterOptions(serviceConf);
@@ -843,13 +845,14 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//launcher service
launchService = new RoleLaunchService(actionQueues,
- providerService,
fs, envVars);
deployChildService(launchService);
//Give the provider access to the state, and AM
- providerService.setAMState(stateForProviders);
+ for (ProviderService providerService : providers) {
+ providerService.setAMState(stateForProviders);
+ }
// chaos monkey
maybeStartMonkey();
@@ -1119,7 +1122,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
SliderKeys.APP_TYPE,
instanceName,
appAttemptID);
- providerService.bindToYarnRegistry(yarnRegistryOperations);
+ for (ProviderService providerService : providers) {
+ providerService.bindToYarnRegistry(yarnRegistryOperations);
+ }
// Yarn registry
ServiceRecord serviceRecord = new ServiceRecord();
@@ -1859,7 +1864,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
protected synchronized void launchProviderService()
throws IOException, SliderException {
// didn't start, so don't register
- providerService.start();
+ for (ProviderService providerService : providers) {
+ providerService.start();
+ }
// and send the started event ourselves
eventCallbackEvent(null);
}
@@ -1959,19 +1966,23 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
ContainerStatus containerStatus) {
LOG_YARN.debug("Container Status: id={}, status={}", containerId,
containerStatus);
+ RoleInstance cinfo = appState.getOwnedContainer(containerId);
+ if (cinfo == null) {
+ LOG_YARN.error("Owned container not found for {}", containerId);
+ return;
+ }
+ ProviderService providerService = SliderProviderFactory
+ .getProviderService(cinfo.providerRole.component.getArtifact());
if (providerService.processContainerStatus(containerId, containerStatus)) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
- RoleInstance cinfo = appState.getOwnedContainer(containerId);
- if (cinfo != null) {
- LOG_YARN.info("Re-requesting status for role {}, {}",
- cinfo.role, containerId);
- //trigger another async container status
- nmClientAsync.getContainerStatusAsync(containerId,
- cinfo.container.getNodeId());
- }
+ LOG_YARN.info("Re-requesting status for role {}, {}",
+ cinfo.role, containerId);
+ //trigger another async container status
+ nmClientAsync.getContainerStatusAsync(containerId,
+ cinfo.container.getNodeId());
} else if (timelineServiceEnabled) {
RoleInstance instance = appState.getOwnedContainer(containerId);
if (instance != null) {
@@ -1997,11 +2008,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
LOG_YARN.warn("Failed to stop Container {}", containerId);
}
-
- public ProviderService getProviderService() {
- return providerService;
- }
-
/**
* Queue an action for immediate execution in the executor thread
* @param action action to execute
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
index 094726d..02f3f0c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
@@ -17,7 +17,6 @@
package org.apache.slider.server.appmaster.web;
import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.state.AppState;
@@ -34,11 +33,6 @@ public interface WebAppApi {
StateAccessForProviders getAppState();
/**
- * The {@link ProviderService} for the current cluster
- */
- ProviderService getProviderService();
-
- /**
* Registry operations accessor
* @return registry access
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
index fd9381c..f88f501 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
@@ -17,7 +17,6 @@
package org.apache.slider.server.appmaster.web;
import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
@@ -33,21 +32,18 @@ public class WebAppApiImpl implements WebAppApi {
private static final Logger log = LoggerFactory.getLogger(WebAppApiImpl.class);
protected final StateAccessForProviders appState;
- protected final ProviderService provider;
private final RegistryOperations registryOperations;
private final MetricsAndMonitoring metricsAndMonitoring;
private final QueueAccess queues;
public WebAppApiImpl(StateAccessForProviders appState,
- ProviderService provider, RegistryOperations registryOperations,
+ RegistryOperations registryOperations,
MetricsAndMonitoring metricsAndMonitoring, QueueAccess queues) {
checkNotNull(appState);
- checkNotNull(provider);
this.queues = queues;
this.registryOperations = registryOperations;
this.appState = appState;
- this.provider = provider;
this.metricsAndMonitoring = metricsAndMonitoring;
}
@@ -57,11 +53,6 @@ public class WebAppApiImpl implements WebAppApi {
}
@Override
- public ProviderService getProviderService() {
- return provider;
- }
-
- @Override
public RegistryOperations getRegistryOperations() {
return registryOperations;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java
index 82d7c8f..5f44bda 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java
@@ -19,7 +19,6 @@
package org.apache.slider.server.appmaster.web.view;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
-import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
@@ -33,12 +32,10 @@ import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_
public abstract class SliderHamletBlock extends HtmlBlock {
protected final StateAccessForProviders appState;
- protected final ProviderService providerService;
protected final RestPaths restPaths = new RestPaths();
public SliderHamletBlock(WebAppApi slider) {
this.appState = slider.getAppState();
- this.providerService = slider.getProviderService();
}
protected String rootPath(String absolutePath) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34c34e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml
deleted file mode 100644
index 96bfe0f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<configuration>
- <property>
- <name>slider.config.loaded</name>
- <value>true</value>
- </property>
- <property>
- <name>slider.provider.docker</name>
- <value>org.apache.slider.providers.docker.DockerProviderFactory</value>
- </property>
-</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org