You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bi...@apache.org on 2016/08/03 23:53:36 UTC
[2/3] incubator-slider git commit: SLIDER-1107 Generate app
configuration files in AM -- squash merge from
feature/SLIDER-1107_AM_config_generation
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/ClientUtils.java b/slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
new file mode 100644
index 0000000..c3ccb1d
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.client;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.exceptions.NoRecordException;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.NotFoundException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.registry.docstore.ConfigFormat;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
+import org.apache.slider.core.registry.retrieve.RegistryRetriever;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
+
+public class ClientUtils {
+ public static ServiceRecord lookupServiceRecord(RegistryOperations rops,
+ String user, String name) throws IOException, SliderException {
+ return lookupServiceRecord(rops, user, null, name);
+ }
+
+ public static ServiceRecord lookupServiceRecord(RegistryOperations rops,
+ String user, String type, String name) throws IOException,
+ SliderException {
+ if (StringUtils.isEmpty(user)) {
+ user = currentUser();
+ } else {
+ user = RegistryPathUtils.encodeForRegistry(user);
+ }
+ if (StringUtils.isEmpty(type)) {
+ type = SliderKeys.APP_TYPE;
+ }
+
+ String path = servicePath(user, type, name);
+ return resolve(rops, path);
+ }
+
+ public static ServiceRecord resolve(RegistryOperations rops, String path)
+ throws IOException, SliderException {
+ try {
+ return rops.resolve(path);
+ } catch (PathNotFoundException | NoRecordException e) {
+ throw new NotFoundException(e.getPath().toString(), e);
+ }
+ }
+
+ public static PublishedConfiguration getConfigFromRegistry(
+ RegistryOperations rops, Configuration configuration,
+ String configName, String appName, String user, boolean external)
+ throws IOException, SliderException {
+ ServiceRecord instance = lookupServiceRecord(rops, user, appName);
+
+ RegistryRetriever retriever = new RegistryRetriever(configuration, instance);
+ PublishedConfigSet configurations = retriever.getConfigurations(external);
+
+ PublishedConfiguration published = retriever.retrieveConfiguration(
+ configurations, configName, external);
+ return published;
+ }
+
+ public static String saveOrReturnConfig(PublishedConfiguration published,
+ String format, File destPath, String fileName)
+ throws BadCommandArgumentsException, IOException {
+ ConfigFormat configFormat = ConfigFormat.resolve(format);
+ if (configFormat == null) {
+ throw new BadCommandArgumentsException(
+ "Unknown/Unsupported format %s ", format);
+ }
+ PublishedConfigurationOutputter outputter =
+ PublishedConfigurationOutputter.createOutputter(configFormat,
+ published);
+ boolean print = destPath == null;
+ if (!print) {
+ if (destPath.isDirectory()) {
+ // creating it under a directory
+ destPath = new File(destPath, fileName);
+ }
+ outputter.save(destPath);
+ return null;
+ } else {
+ return outputter.asString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index f3dcea3..8210f4d 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -101,6 +101,7 @@ import org.apache.slider.common.params.ActionNodesArgs;
import org.apache.slider.common.params.ActionPackageArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionResolveArgs;
+import org.apache.slider.common.params.ActionResourceArgs;
import org.apache.slider.common.params.ActionStatusArgs;
import org.apache.slider.common.params.ActionThawArgs;
import org.apache.slider.common.params.ActionTokensArgs;
@@ -177,7 +178,6 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
-import java.io.FilenameFilter;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -415,7 +415,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
case ACTION_INSTALL_PACKAGE:
exitCode = actionInstallPkg(serviceArgs.getActionInstallPackageArgs());
break;
-
+
case ACTION_KEYTAB:
exitCode = actionKeytab(serviceArgs.getActionKeytabArgs());
break;
@@ -443,7 +443,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
case ACTION_RESOLVE:
exitCode = actionResolve(serviceArgs.getActionResolveArgs());
break;
-
+
+ case ACTION_RESOURCE:
+ exitCode = actionResource(serviceArgs.getActionResourceArgs());
+ break;
+
case ACTION_STATUS:
exitCode = actionStatus(clusterName, serviceArgs.getActionStatusArgs());
break;
@@ -1029,7 +1033,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Path fileInFs = new Path(pkgPath, keytabInfo.keytab );
log.info("Deleting keytab {}", fileInFs);
FileSystem sfs = sliderFileSystem.getFileSystem();
- require(sfs.exists(fileInFs), "No keytab to delete found at %s", fileInFs.toUri());
+ require(sfs.exists(fileInFs), "No keytab to delete found at %s",
+ fileInFs.toUri());
sfs.delete(fileInFs, false);
return EXIT_SUCCESS;
@@ -1105,6 +1110,106 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
@Override
+ public int actionResource(ActionResourceArgs resourceInfo)
+ throws YarnException, IOException {
+ if (resourceInfo.help) {
+ actionHelp(ACTION_RESOURCE);
+ return EXIT_SUCCESS;
+ } else if (resourceInfo.install) {
+ return actionInstallResource(resourceInfo);
+ } else if (resourceInfo.delete) {
+ return actionDeleteResource(resourceInfo);
+ } else if (resourceInfo.list) {
+ return actionListResource(resourceInfo);
+ } else {
+ throw new BadCommandArgumentsException(
+ "Resource option specified not found.\n"
+ + CommonArgs.usage(serviceArgs, ACTION_RESOURCE));
+ }
+ }
+
+ private int actionListResource(ActionResourceArgs resourceInfo) throws IOException {
+ String folder = resourceInfo.folder != null ? resourceInfo.folder : StringUtils.EMPTY;
+ Path path = sliderFileSystem.buildResourcePath(folder);
+ RemoteIterator<LocatedFileStatus> files =
+ sliderFileSystem.getFileSystem().listFiles(path, true);
+ log.info("Resources:");
+ while (files.hasNext()) {
+ log.info("\t" + files.next().getPath().toString());
+ }
+
+ return EXIT_SUCCESS;
+ }
+
+ private int actionDeleteResource(ActionResourceArgs resourceInfo)
+ throws BadCommandArgumentsException, IOException {
+ if (StringUtils.isEmpty(resourceInfo.resource)) {
+ throw new BadCommandArgumentsException("A file name is required.");
+ }
+
+ Path fileInFs;
+ if (resourceInfo.folder == null) {
+ fileInFs = sliderFileSystem.buildResourcePath(resourceInfo.resource);
+ } else {
+ fileInFs = sliderFileSystem.buildResourcePath(resourceInfo.folder,
+ resourceInfo.resource);
+ }
+
+ log.info("Deleting resource {}", fileInFs);
+ FileSystem sfs = sliderFileSystem.getFileSystem();
+ require(sfs.exists(fileInFs), "No resource to delete found at %s", fileInFs.toUri());
+ sfs.delete(fileInFs, true);
+
+ return EXIT_SUCCESS;
+ }
+
+ private int actionInstallResource(ActionResourceArgs resourceInfo)
+ throws BadCommandArgumentsException, IOException {
+ Path srcFile = null;
+ String folder = resourceInfo.folder != null ? resourceInfo.folder : StringUtils.EMPTY;
+
+ requireArgumentSet(Arguments.ARG_RESOURCE, resourceInfo.resource);
+ File file = new File(resourceInfo.resource);
+ require(file.isFile() || file.isDirectory(),
+ "Unable to access supplied file at %s", file.getAbsolutePath());
+
+ File[] files;
+ if (file.isDirectory()) {
+ files = file.listFiles();
+ } else {
+ files = new File[] { file };
+ }
+
+ Path pkgPath = sliderFileSystem.buildResourcePath(folder);
+ FileSystem sfs = sliderFileSystem.getFileSystem();
+
+ if (!sfs.exists(pkgPath)) {
+ sfs.mkdirs(pkgPath);
+ sfs.setPermission(pkgPath, new FsPermission(
+ FsAction.ALL, FsAction.NONE, FsAction.NONE));
+ } else {
+ require(sfs.isDirectory(pkgPath), "Specified folder %s exists and is " +
+ "not a directory", folder);
+ }
+
+ for (File f : files) {
+ srcFile = new Path(f.toURI());
+
+ Path fileInFs = new Path(pkgPath, srcFile.getName());
+ log.info("Installing file {} at {} and overwrite is {}.",
+ srcFile, fileInFs, resourceInfo.overwrite);
+ require(!(sfs.exists(fileInFs) && !resourceInfo.overwrite),
+ "File exists at %s. Use --overwrite to overwrite.", fileInFs.toUri());
+
+ sfs.copyFromLocalFile(false, resourceInfo.overwrite, srcFile, fileInFs);
+ sfs.setPermission(fileInFs,
+ new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE));
+ }
+
+ return EXIT_SUCCESS;
+ }
+
+ @Override
public int actionClient(ActionClientArgs clientInfo) throws
YarnException,
IOException {
@@ -1210,8 +1315,21 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
E_INVALID_INSTALL_PATH + ": " + clientInfo.installLocation.getAbsolutePath());
File pkgFile;
- require(isSet(clientInfo.packageURI), E_INVALID_APPLICATION_PACKAGE_LOCATION);
- pkgFile = new File(clientInfo.packageURI);
+ File tmpDir = null;
+
+ require(isSet(clientInfo.packageURI) || isSet(clientInfo.name),
+ E_INVALID_APPLICATION_PACKAGE_LOCATION);
+ if (isSet(clientInfo.packageURI)) {
+ pkgFile = new File(clientInfo.packageURI);
+ } else {
+ Path appDirPath = sliderFileSystem.buildAppDefDirPath(clientInfo.name);
+ Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG);
+ require(sliderFileSystem.isFile(appDefPath),
+ E_INVALID_APPLICATION_PACKAGE_LOCATION);
+ tmpDir = Files.createTempDir();
+ pkgFile = new File(tmpDir, SliderKeys.DEFAULT_APP_PKG);
+ sliderFileSystem.copyHdfsFileToLocal(appDefPath, pkgFile);
+ }
require(pkgFile.isFile(),
E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + " at %s", pkgFile.getAbsolutePath());
@@ -1233,6 +1351,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
AbstractClientProvider
provider = createClientProvider(SliderProviderFactory.DEFAULT_CLUSTER_TYPE);
provider.processClientOperation(sliderFileSystem,
+ getRegistryOperations(),
+ getConfig(),
"INSTALL",
clientInfo.installLocation,
pkgFile,
@@ -4077,17 +4197,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@VisibleForTesting
public PublishedConfiguration actionRegistryGetConfig(ActionRegistryArgs registryArgs)
throws YarnException, IOException {
- ServiceRecord instance = lookupServiceRecord(registryArgs);
-
- RegistryRetriever retriever = new RegistryRetriever(getConfig(), instance);
- boolean external = !registryArgs.internal;
- PublishedConfigSet configurations =
- retriever.getConfigurations(external);
-
- PublishedConfiguration published = retriever.retrieveConfiguration(configurations,
- registryArgs.getConf,
- external);
- return published;
+ return ClientUtils.getConfigFromRegistry(getRegistryOperations(),
+ getConfig(), registryArgs.getConf, registryArgs.name, registryArgs.user,
+ !registryArgs.internal);
}
/**
@@ -4130,27 +4242,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// decide whether or not to print
String entry = registryArgs.getConf;
String format = registryArgs.format;
- ConfigFormat configFormat = ConfigFormat.resolve(format);
- if (configFormat == null) {
- throw new BadCommandArgumentsException(
- "Unknown/Unsupported format %s ", format);
- }
- PublishedConfigurationOutputter outputter =
- PublishedConfigurationOutputter.createOutputter(configFormat,
- published);
- boolean print = registryArgs.out == null;
- if (!print) {
- File outputPath = registryArgs.out;
- if (outputPath.isDirectory()) {
- // creating it under a directory
- outputPath = new File(outputPath, entry + "." + format);
- }
- log.debug("Destination path: {}", outputPath);
- outputter.save(outputPath);
- } else {
- print(outputter.asString());
+ String output = ClientUtils.saveOrReturnConfig(published,
+ registryArgs.format, registryArgs.out, entry + "." + format);
+ if (output != null) {
+ print(output);
}
-
}
/**
@@ -4200,32 +4296,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private ServiceRecord lookupServiceRecord(ActionRegistryArgs registryArgs) throws
SliderException,
IOException {
- String user;
- if (StringUtils.isNotEmpty(registryArgs.user)) {
- user = RegistryPathUtils.encodeForRegistry(registryArgs.user);
- } else {
- user = currentUser();
- }
-
- String path = servicePath(user, registryArgs.serviceType,
- registryArgs.name);
- return resolve(path);
- }
-
- /**
- * Look up a service record of the current user
- * @param serviceType service type
- * @param id instance ID
- * @return instance data
- * @throws UnknownApplicationInstanceException no path or service record
- * at the end of the path
- * @throws SliderException other failures
- * @throws IOException IO problems or wrapped exceptions
- */
- public ServiceRecord lookupServiceRecord(String serviceType, String id)
- throws IOException, SliderException {
- String path = servicePath(currentUser(), serviceType, id);
- return resolve(path);
+ return ClientUtils.lookupServiceRecord(getRegistryOperations(),
+ registryArgs.user, registryArgs.serviceType, registryArgs.name);
}
/**
@@ -4240,11 +4312,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
*/
public ServiceRecord resolve(String path)
throws IOException, SliderException {
- try {
- return getRegistryOperations().resolve(path);
- } catch (PathNotFoundException | NoRecordException e) {
- throw new NotFoundException(e.getPath().toString(), e);
- }
+ return ClientUtils.resolve(getRegistryOperations(), path);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
index 5c5d96b..30f6ba9 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -43,6 +43,7 @@ import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionListArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionResolveArgs;
+import org.apache.slider.common.params.ActionResourceArgs;
import org.apache.slider.common.params.ActionStatusArgs;
import org.apache.slider.common.params.ActionThawArgs;
import org.apache.slider.common.params.ActionUpgradeArgs;
@@ -117,7 +118,6 @@ public interface SliderClientAPI extends Service {
* @throws YarnException Yarn problems
* @throws IOException other problems
* @throws BadCommandArgumentsException bad arguments.
- * @deprecated use #actionKeytab
*/
int actionKeytab(ActionKeytabArgs keytabInfo)
throws YarnException, IOException;
@@ -134,6 +134,17 @@ public interface SliderClientAPI extends Service {
throws YarnException, IOException;
/**
+ * Manage file resources leveraged by slider
+ *
+ * @param resourceInfo the arguments needed to manage the resource
+ * @throws YarnException Yarn problems
+ * @throws IOException other problems
+ * @throws BadCommandArgumentsException bad arguments.
+ */
+ int actionResource(ActionResourceArgs resourceInfo)
+ throws YarnException, IOException;
+
+ /**
* Perform client operations such as install or configure
*
* @param clientInfo the arguments needed for client operations
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 05c7048..ba3effc 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -116,7 +116,8 @@ public interface SliderKeys extends SliderXmlConfKeys {
String HISTORY_FILENAME_SUFFIX = "json";
String HISTORY_FILENAME_PREFIX = "rolehistory-";
String KEYTAB_DIR = "keytabs";
-
+ String RESOURCE_DIR = "resources";
+
/**
* Filename pattern is required to save in strict temporal order.
* Important: older files must sort less-than newer files when using
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java
new file mode 100644
index 0000000..60fcc87
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = {SliderActions.ACTION_RESOURCE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_RESOURCE)
+
+public class ActionResourceArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_RESOURCE;
+ }
+
+ @Parameter(names = {ARG_INSTALL},
+ description = "Install the resource(s)")
+ public boolean install;
+
+ @Parameter(names = {ARG_DELETE},
+ description = "Delete the file")
+ public boolean delete;
+
+ @Parameter(names = {ARG_LIST},
+ description = "List of installed files")
+ public boolean list;
+
+ @Parameter(names = {ARG_RESOURCE},
+ description = "Name of the file or directory")
+ public String resource;
+
+ @Parameter(names = {ARG_DESTDIR},
+ description = "The name of the folder in which to store the resources")
+ public String folder;
+
+ @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing resource(s)")
+ public boolean overwrite = false;
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 3;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 0a8388d..aec4e26 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -104,6 +104,7 @@ public interface Arguments {
String ARG_PROVIDER = "--provider";
String ARG_QUEUE = "--queue";
String ARG_REPLACE_PKG = "--replacepkg";
+ String ARG_RESOURCE = "--resource";
String ARG_RESOURCES = "--resources";
String ARG_RES_COMP_OPT = "--rescompopt";
String ARG_RES_COMP_OPT_SHORT = "--rco";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index 0a658ea..4016cc9 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -75,6 +75,7 @@ public class ClientArgs extends CommonArgs {
private final ActionPackageArgs actionPackageArgs = new ActionPackageArgs();
private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
+ private final ActionResourceArgs actionResourceArgs = new ActionResourceArgs();
private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
private final ActionThawArgs actionThawArgs = new ActionThawArgs();
private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs();
@@ -116,6 +117,7 @@ public class ClientArgs extends CommonArgs {
actionPackageArgs,
actionRegistryArgs,
actionResolveArgs,
+ actionResourceArgs,
actionStatusArgs,
actionThawArgs,
actionTokenArgs,
@@ -227,6 +229,10 @@ public class ClientArgs extends CommonArgs {
return actionResolveArgs;
}
+ public ActionResourceArgs getActionResourceArgs() {
+ return actionResourceArgs;
+ }
+
public ActionStatusArgs getActionStatusArgs() {
return actionStatusArgs;
}
@@ -346,6 +352,10 @@ public class ClientArgs extends CommonArgs {
bindCoreAction(actionResolveArgs);
break;
+ case ACTION_RESOURCE:
+ bindCoreAction(actionResourceArgs);
+ break;
+
case ACTION_STATUS:
bindCoreAction(actionStatusArgs);
break;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
index aab7c98..204ad9a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -51,6 +51,7 @@ public interface SliderActions {
String ACTION_RECONFIGURE = "reconfigure";
String ACTION_REGISTRY = "registry";
String ACTION_RESOLVE = "resolve";
+ String ACTION_RESOURCE = "resource";
String ACTION_STATUS = "status";
String ACTION_THAW = "start";
String ACTION_TOKENS = "tokens";
@@ -106,6 +107,7 @@ public interface SliderActions {
" Deprecated, use '" + ACTION_KEYTAB + " " + ClientArgs.ARG_INSTALL + "'.";
String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client";
+ String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 6a02367..aa5edf1 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -64,6 +65,8 @@ public class CoreFileSystem {
private static final Logger
log = LoggerFactory.getLogger(CoreFileSystem.class);
+ private static final String UTF_8 = "UTF-8";
+
protected final FileSystem fileSystem;
protected final Configuration configuration;
@@ -209,6 +212,55 @@ public class CoreFileSystem {
}
/**
+ * Build up the path string for resource install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for resource
+ */
+ public Path buildResourcePath(String resourceFolder) {
+ Preconditions.checkNotNull(resourceFolder);
+ Path path = getBaseApplicationPath();
+ return new Path(path, SliderKeys.RESOURCE_DIR + "/" + resourceFolder);
+ }
+
+ /**
+ * Build up the path string for resource install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for resource
+ */
+ public Path buildResourcePath(String dirName, String fileName) {
+ Preconditions.checkNotNull(dirName);
+ Preconditions.checkNotNull(fileName);
+ Path path = getBaseApplicationPath();
+ return new Path(path, SliderKeys.RESOURCE_DIR + "/" + dirName + "/" + fileName);
+ }
+
+ /**
+ * Build up the path string for cluster resource install location -no
+ * attempt to create the directory is made
+ *
+ * @return the path for resource
+ */
+ public Path buildClusterResourcePath(String clusterName, String component) {
+ Preconditions.checkNotNull(clusterName);
+ Path path = buildClusterDirPath(clusterName);
+ return new Path(path, SliderKeys.RESOURCE_DIR + "/" + component);
+ }
+
+ /**
+ * Build up the path string for cluster resource install location -no
+ * attempt to create the directory is made
+ *
+ * @return the path for resource
+ */
+ public Path buildClusterResourcePath(String clusterName) {
+ Preconditions.checkNotNull(clusterName);
+ Path path = buildClusterDirPath(clusterName);
+ return new Path(path, SliderKeys.RESOURCE_DIR);
+ }
+
+ /**
* Create the Slider cluster path for a named cluster and all its subdirs
* This is a directory; a mkdirs() operation is executed
* to ensure that it is there.
@@ -713,6 +765,17 @@ public class CoreFileSystem {
fileSystem.setPermission(destPath, fp);
}
+ public void copyHdfsFileToLocal(Path hdfsPath, File destFile)
+ throws IOException {
+ if (hdfsPath == null || destFile == null) {
+ throw new IOException("Either hdfsPath or destPath is null");
+ }
+ log.info("Copying file {} to {}", hdfsPath.toUri(), destFile.toURI());
+
+ Path destPath = new Path(destFile.getPath());
+ fileSystem.copyToLocalFile(hdfsPath, destPath);
+ }
+
/**
* list entries in a filesystem directory
*
@@ -767,15 +830,36 @@ public class CoreFileSystem {
}
public void touch(Path path, boolean overwrite) throws IOException {
- FSDataOutputStream out = fileSystem.create(path, overwrite);
- out.close();
+ FSDataOutputStream out = null;
+ try {
+ out = fileSystem.create(path, overwrite);
+ } finally {
+ IOUtils.closeStream(out);
+ }
}
public void cat(Path path, boolean overwrite, String data) throws IOException {
- FSDataOutputStream out = fileSystem.create(path, overwrite);
- byte[] bytes = data.getBytes(Charset.forName("UTF-8"));
- out.write(bytes);
- out.close();
+ FSDataOutputStream out = null;
+ try {
+ out = fileSystem.create(path, overwrite);
+ byte[] bytes = data.getBytes(Charset.forName("UTF-8"));
+ out.write(bytes);
+ } finally {
+ IOUtils.closeStream(out);
+ }
+ }
+
+ public String cat(Path path) throws IOException {
+ FileStatus status = fileSystem.getFileStatus(path);
+ byte[] b = new byte[(int) status.getLen()];
+ FSDataInputStream in = null;
+ try {
+ in = fileSystem.open(path);
+ int count = in.read(b);
+ return new String(b, 0, count, UTF_8);
+ } finally {
+ IOUtils.closeStream(in);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
index 9013edb..d24a158 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
@@ -345,7 +345,7 @@ public class ConfTreeOperations {
confTreeSerDeser.fromFile(resource) );
return ops;
}
-
+
/**
* Build from an existing instance -which is cloned via JSON ser/deser
* @param instance the source instance
@@ -431,6 +431,20 @@ public class ConfTreeOperations {
}
/**
+ * Get a component opt as a boolean using {@link Boolean#valueOf(String)}.
+ *
+ * @param name component name
+ * @param option option name
+ * @param defVal default value
+ * @return parsed value
+ * @throws NumberFormatException if the role could not be parsed.
+ */
+ public boolean getComponentOptBool(String name, String option, boolean defVal) {
+ String val = getComponentOpt(name, option, Boolean.toString(defVal));
+ return Boolean.valueOf(val);
+ }
+
+ /**
* Set a component option, creating the component if necessary
* @param component component name
* @param option option name
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 0348828..5a3eb3d 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -50,6 +50,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
/**
* Launcher of applications: base class
@@ -71,6 +72,7 @@ public abstract class AbstractLauncher extends Configured {
Records.newRecord(ContainerLaunchContext.class);
protected final List<String> commands = new ArrayList<>(20);
protected final Map<String, LocalResource> localResources = new HashMap<>();
+ protected final Map<String, String> mountPaths = new HashMap<>();
private final Map<String, ByteBuffer> serviceData = new HashMap<>();
// security
protected final Credentials credentials;
@@ -131,8 +133,13 @@ public abstract class AbstractLauncher extends Configured {
return localResources;
}
- public void addLocalResource(String subpath, LocalResource resource) {
- localResources.put(subpath, resource);
+ public void addLocalResource(String subPath, LocalResource resource) {
+ localResources.put(subPath, resource);
+ }
+
+ public void addLocalResource(String subPath, LocalResource resource, String mountPath) {
+ localResources.put(subPath, resource);
+ mountPaths.put(subPath, mountPath);
}
/**
@@ -227,6 +234,16 @@ public abstract class AbstractLauncher extends Configured {
env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker");
env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage);//if yarnDockerMode, then dockerImage is set
env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer);
+ StringBuilder sb = new StringBuilder();
+ for (Entry<String,String> mount : mountPaths.entrySet()) {
+ if (sb.length() > 0) {
+ sb.append(",");
+ }
+ sb.append(mount.getKey());
+ sb.append(":");
+ sb.append(mount.getValue());
+ }
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS", sb.toString());
log.info("yarn docker env var has been set {}", containerLaunchContext.getEnvironment().toString());
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
index 8efaa5b..7fb3158 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
@@ -150,7 +150,11 @@ public class AppDefinitionPersister {
pkgSrcDir.mkdirs();
File destMetaInfo = new File(pkgSrcDir, "metainfo.json");
if (isFileUsed) {
- Files.copy(buildInfo.appMetaInfo, destMetaInfo);
+ if (buildInfo.appMetaInfo.getName().endsWith(".xml")) {
+ Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.xml"));
+ } else {
+ Files.copy(buildInfo.appMetaInfo, destMetaInfo);
+ }
} else {
Files.write(
buildInfo.appMetaInfoJson.getBytes(Charset.forName("UTF-8")),
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
index 12581d7..ddab606 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
@@ -23,8 +23,10 @@ public enum ConfigFormat {
JSON("json"),
PROPERTIES("properties"),
XML("xml"),
+ HADOOP_XML("hadoop-xml"),
ENV("env"),
-// YAML("yaml");
+ TEMPLATE("template"),
+ YAML("yaml"),
;
ConfigFormat(String suffix) {
this.suffix = suffix;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
new file mode 100644
index 0000000..2e1615b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.core.registry.docstore;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.SliderFileSystem;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class ConfigUtils {
+ public static final String TEMPLATE_FILE = "template.file";
+
+ public static String replaceProps(Map<String, String> config, String content) {
+ Map<String, String> tokens = new HashMap<>();
+ for (Entry<String, String> entry : config.entrySet()) {
+ tokens.put("${" + entry.getKey() + "}", entry.getValue());
+ tokens.put("{{" + entry.getKey() + "}}", entry.getValue());
+ }
+ String value = content;
+ for (Map.Entry<String,String> token : tokens.entrySet()) {
+ value = value.replaceAll(Pattern.quote(token.getKey()),
+ Matcher.quoteReplacement(token.getValue()));
+ }
+ return value;
+ }
+
+ public static Map<String, String> replacePropsInConfig(
+ Map<String, String> config, Map<String, String> env) {
+ Map<String, String> tokens = new HashMap<>();
+ for (Entry<String, String> entry : env.entrySet()) {
+ tokens.put("${" + entry.getKey() + "}", entry.getValue());
+ }
+ Map<String, String> newConfig = new HashMap<>();
+ for (Entry<String, String> entry : config.entrySet()) {
+ String value = entry.getValue();
+ for (Map.Entry<String,String> token : tokens.entrySet()) {
+ value = value.replaceAll(Pattern.quote(token.getKey()),
+ Matcher.quoteReplacement(token.getValue()));
+ }
+ newConfig.put(entry.getKey(), entry.getValue());
+ }
+ return newConfig;
+ }
+
+ public static void prepConfigForTemplateOutputter(ConfigFormat configFormat,
+ Map<String, String> config, SliderFileSystem fileSystem,
+ String clusterName, String fileName) throws IOException {
+ if (!configFormat.equals(ConfigFormat.TEMPLATE)) {
+ return;
+ }
+ Path templateFile = null;
+ if (config.containsKey(TEMPLATE_FILE)) {
+ templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE));
+ if (!fileSystem.isFile(templateFile)) {
+ templateFile = fileSystem.buildResourcePath(clusterName,
+ config.get(TEMPLATE_FILE));
+ }
+ if (!fileSystem.isFile(templateFile)) {
+ throw new IOException("config specified template file " + config
+ .get(TEMPLATE_FILE) + " but " + templateFile + " doesn't exist");
+ }
+ }
+ if (templateFile == null && fileName != null) {
+ templateFile = fileSystem.buildResourcePath(fileName);
+ if (!fileSystem.isFile(templateFile)) {
+ templateFile = fileSystem.buildResourcePath(clusterName,
+ fileName);
+ }
+ }
+ if (fileSystem.isFile(templateFile)) {
+ config.put("content", fileSystem.cat(templateFile));
+ } else {
+ config.put("content", "");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
index 15ac207..9bdcfcb 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
@@ -24,9 +24,11 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.slider.common.tools.ConfigHelper;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.DumperOptions.FlowStyle;
+import org.yaml.snakeyaml.Yaml;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.StringWriter;
@@ -57,14 +59,7 @@ public abstract class PublishedConfigurationOutputter {
}
*/
public void save(File dest) throws IOException {
- FileOutputStream out = null;
- try {
- out = new FileOutputStream(dest);
- save(out);
- out.close();
- } finally {
- org.apache.hadoop.io.IOUtils.closeStream(out);
- }
+ FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
}
/**
@@ -89,12 +84,13 @@ public abstract class PublishedConfigurationOutputter {
* @param owner owning config
* @return the outputter
*/
-
+
public static PublishedConfigurationOutputter createOutputter(ConfigFormat format,
PublishedConfiguration owner) {
Preconditions.checkNotNull(owner);
switch (format) {
case XML:
+ case HADOOP_XML:
return new XmlOutputter(owner);
case PROPERTIES:
return new PropertiesOutputter(owner);
@@ -102,11 +98,15 @@ public abstract class PublishedConfigurationOutputter {
return new JsonOutputter(owner);
case ENV:
return new EnvOutputter(owner);
+ case TEMPLATE:
+ return new TemplateOutputter(owner);
+ case YAML:
+ return new YamlOutputter(owner);
default:
throw new RuntimeException("Unsupported format :" + format);
}
}
-
+
public static class XmlOutputter extends PublishedConfigurationOutputter {
@@ -131,7 +131,7 @@ public abstract class PublishedConfigurationOutputter {
return configuration;
}
}
-
+
public static class PropertiesOutputter extends PublishedConfigurationOutputter {
private final Properties properties;
@@ -146,15 +146,15 @@ public abstract class PublishedConfigurationOutputter {
properties.store(out, "");
}
-
+
public String asString() throws IOException {
StringWriter sw = new StringWriter();
properties.store(sw, "");
return sw.toString();
}
}
-
-
+
+
public static class JsonOutputter extends PublishedConfigurationOutputter {
public JsonOutputter(PublishedConfiguration owner) {
@@ -162,11 +162,6 @@ public abstract class PublishedConfigurationOutputter {
}
@Override
- public void save(File dest) throws IOException {
- FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
- }
-
- @Override
public String asString() throws IOException {
return owner.asJson();
}
@@ -180,19 +175,36 @@ public abstract class PublishedConfigurationOutputter {
}
@Override
- public void save(File dest) throws IOException {
- FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
- }
-
- @Override
public String asString() throws IOException {
if (!owner.entries.containsKey("content")) {
throw new IOException("Configuration has no content field and cannot " +
"be retrieved as type 'env'");
}
- return owner.entries.get("content");
+ String content = owner.entries.get("content");
+ return ConfigUtils.replaceProps(owner.entries, content);
}
}
+ public static class TemplateOutputter extends EnvOutputter {
+ public TemplateOutputter(PublishedConfiguration owner) {
+ super(owner);
+ }
+ }
+
+ public static class YamlOutputter extends PublishedConfigurationOutputter {
+
+ private final Yaml yaml;
+
+ public YamlOutputter(PublishedConfiguration owner) {
+ super(owner);
+ DumperOptions options = new DumperOptions();
+ options.setDefaultFlowStyle(FlowStyle.BLOCK);
+ yaml = new Yaml(options);
+ }
+
+ public String asString() throws IOException {
+ return yaml.dump(owner.entries);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
index fcab65e..510de5d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
@@ -21,6 +21,7 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTreeOperations;
@@ -223,18 +224,22 @@ public abstract class AbstractClientProvider extends Configured {
/**
* Process client operations for applications such as install, configure
* @param fileSystem
+ * @param registryOperations
+ * @param configuration
* @param operation
* @param clientInstallPath
* @param clientPackage
- * @param config
+ * @param clientConfig
* @param name
* @throws SliderException
*/
public void processClientOperation(SliderFileSystem fileSystem,
+ RegistryOperations registryOperations,
+ Configuration configuration,
String operation,
File clientInstallPath,
File clientPackage,
- JSONObject config,
+ JSONObject clientConfig,
String name)
throws SliderException {
throw new SliderException("Provider does not support client operations.");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index f3dcd1d..4c6a50b 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -22,13 +22,17 @@ import com.google.common.io.Files;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.client.ClientUtils;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
@@ -38,13 +42,18 @@ import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.Component;
+import org.apache.slider.providers.agent.application.metadata.ConfigFile;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
+import org.apache.slider.providers.agent.application.metadata.OSPackage;
+import org.apache.slider.providers.agent.application.metadata.OSSpecific;
+import org.apache.slider.providers.agent.application.metadata.Package;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
@@ -53,7 +62,6 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -291,6 +299,8 @@ public class AgentClientProvider extends AbstractClientProvider
@Override
public void processClientOperation(SliderFileSystem fileSystem,
+ RegistryOperations rops,
+ Configuration configuration,
String operation,
File clientInstallPath,
File appPackage,
@@ -319,51 +329,37 @@ public class AgentClientProvider extends AbstractClientProvider
{
ZipEntry zipEntry = zipInputStream.getNextEntry();
while (zipEntry != null) {
- if ("metainfo.xml".equals(zipEntry.getName())) {
- int size = (int) zipEntry.getSize();
- if (size != -1) {
- log.info("Reading {} of size {}", zipEntry.getName(),
- zipEntry.getSize());
- byte[] content = new byte[size];
- int offset = 0;
- while (offset < size) {
- offset += zipInputStream.read(content, offset, size - offset);
- }
- metaInfo = new MetainfoParser().fromXmlStream(new ByteArrayInputStream(content));
- }
- } else if ("metainfo.json".equals(zipEntry.getName())) {
- int size = (int) zipEntry.getSize();
- if (size != -1) {
- log.info("Reading {} of size {}", zipEntry.getName(),
- zipEntry.getSize());
- byte[] content = new byte[size];
- int offset = 0;
- while (offset < size) {
- offset += zipInputStream.read(content, offset, size - offset);
+ log.info("Processing {}", zipEntry.getName());
+ String filePath = appPkgDir + File.separator + zipEntry.getName();
+ if (!zipEntry.isDirectory()) {
+ log.info("Extracting file {}", filePath);
+ extractFile(zipInputStream, filePath);
+
+ if ("metainfo.xml".equals(zipEntry.getName())) {
+ FileInputStream input = null;
+ try {
+ input = new FileInputStream(filePath);
+ metaInfo = new MetainfoParser().fromXmlStream(input);
+ } finally {
+ IOUtils.closeStream(input);
}
- metaInfo = new MetainfoParser().fromJsonStream(new ByteArrayInputStream(content));
- }
- } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) {
- int size = (int) zipEntry.getSize();
- if (size != -1) {
- log.info("Reading {} of size {}", zipEntry.getName(),
- zipEntry.getSize());
- byte[] content = new byte[size];
- int offset = 0;
- while (offset < size) {
- offset += zipInputStream.read(content, offset, size - offset);
+ } else if ("metainfo.json".equals(zipEntry.getName())) {
+ FileInputStream input = null;
+ try {
+ input = new FileInputStream(filePath);
+ metaInfo = new MetainfoParser().fromJsonStream(input);
+ } finally {
+ IOUtils.closeStream(input);
}
+ } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) {
try {
- defaultConfig = new JSONObject(new String(content, Charset.defaultCharset()));
+ defaultConfig = new JSONObject(FileUtils.readFileToString(new File(filePath), Charset.defaultCharset()));
} catch (JSONException jex) {
throw new SliderException("Unable to read default client config.", jex);
}
}
- }
- String filePath = appPkgDir + File.separator + zipEntry.getName();
- if (!zipEntry.isDirectory()) {
- extractFile(zipInputStream, filePath);
} else {
+ log.info("Creating dir {}", filePath);
File dir = new File(filePath);
dir.mkdir();
}
@@ -379,34 +375,109 @@ public class AgentClientProvider extends AbstractClientProvider
throw new BadConfigException(E_COULD_NOT_READ_METAINFO);
}
- expandAgentTar(agentPkgDir);
-
- JSONObject commandJson = getCommandJson(defaultConfig, config, metaInfo, clientInstallPath, name);
- FileWriter file = new FileWriter(new File(cmdDir, "command.json"));
- try {
- file.write(commandJson.toString());
-
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- file.flush();
- file.close();
- }
-
- String client_script = null;
+ String clientScript = null;
+ String clientComponent = null;
for (Component component : metaInfo.getApplication().getComponents()) {
if (component.getCategory().equals("CLIENT")) {
- client_script = component.getCommandScript().getScript();
- log.info("Installing CLIENT {} using script {}", component.getName(), client_script);
+ clientComponent = component.getName();
+ if (component.getCommandScript() != null) {
+ clientScript = component.getCommandScript().getScript();
+ }
break;
}
}
- if (SliderUtils.isUnset(client_script)) {
- throw new SliderException("No valid CLIENT component found. Aborting install.");
- }
+ if (SliderUtils.isUnset(clientScript)) {
+ log.info("Installing CLIENT without script");
+ List<Package> packages = metaInfo.getApplication().getPackages();
+ if (packages.size() > 0) {
+ // retrieve package resources from HDFS and extract
+ for (Package pkg : packages) {
+ Path pkgPath = fileSystem.buildResourcePath(pkg.getName());
+ if (!fileSystem.isFile(pkgPath) && name != null) {
+ pkgPath = fileSystem.buildResourcePath(name, pkg.getName());
+ }
+ if (!fileSystem.isFile(pkgPath)) {
+ throw new IOException("Package doesn't exist as a resource: " +
+ pkg.getName());
+ }
+ if ("archive".equals(pkg.getType())) {
+ File pkgFile = new File(tmpDir, pkg.getName());
+ fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile);
+ expandTar(pkgFile, clientInstallPath);
+ } else {
+ File pkgFile = new File(clientInstallPath, pkg.getName());
+ fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile);
+ }
+ }
+ } else {
+ // extract tarball from app def
+ for (OSSpecific osSpecific : metaInfo.getApplication()
+ .getOSSpecifics()) {
+ for (OSPackage pkg : osSpecific.getPackages()) {
+ if ("tarball".equals(pkg.getType())) {
+ File pkgFile = new File(appPkgDir, pkg.getName());
+ expandTar(pkgFile, clientInstallPath);
+ }
+ }
+ }
+ }
+ if (name == null) {
+ log.warn("Conf files not being generated because no app name was " +
+ "provided");
+ return;
+ }
+ File confInstallDir;
+ String clientRoot = null;
+ if (config != null) {
+ try {
+ clientRoot = config.getJSONObject("global")
+ .getString(AgentKeys.APP_CLIENT_ROOT);
+ } catch (JSONException e) {
+ log.info("Couldn't read {} from provided client config, falling " +
+ "back on default", AgentKeys.APP_CLIENT_ROOT);
+ }
+ }
+ if (clientRoot == null && defaultConfig != null) {
+ try {
+ clientRoot = defaultConfig.getJSONObject("global")
+ .getString(AgentKeys.APP_CLIENT_ROOT);
+ } catch (JSONException e) {
+ log.info("Couldn't read {} from default client config, using {}",
+ AgentKeys.APP_CLIENT_ROOT, clientInstallPath);
+ }
+ }
+ if (clientRoot == null) {
+ confInstallDir = clientInstallPath;
+ } else {
+ confInstallDir = new File(new File(clientInstallPath, clientRoot), "conf");
+ if (!confInstallDir.exists()) {
+ confInstallDir.mkdirs();
+ }
+ }
+ String user = RegistryUtils.currentUser();
+ for (ConfigFile configFile : metaInfo.getComponentConfigFiles(clientComponent)) {
+ retrieveConfigFile(rops, configuration, configFile, name, user,
+ confInstallDir);
+ }
+ } else {
+ log.info("Installing CLIENT using script {}", clientScript);
+ expandAgentTar(agentPkgDir);
+
+ JSONObject commandJson = getCommandJson(defaultConfig, config, metaInfo, clientInstallPath, name);
+ FileWriter file = new FileWriter(new File(cmdDir, "command.json"));
+ try {
+ file.write(commandJson.toString());
+
+ } catch (IOException e) {
+ log.error("Couldn't write command.json to file");
+ } finally {
+ file.flush();
+ file.close();
+ }
- runCommand(appPkgDir, agentPkgDir, cmdDir, client_script);
+ runCommand(appPkgDir, agentPkgDir, cmdDir, clientScript);
+ }
} catch (IOException ioex) {
log.warn("Error while executing INSTALL command {}", ioex.getMessage());
@@ -481,6 +552,11 @@ public class AgentClientProvider extends AbstractClientProvider
String libDirProp =
System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
File tarFile = new File(libDirProp, SliderKeys.AGENT_TAR);
+ expandTar(tarFile, agentPkgDir);
+ }
+
+ private void expandTar(File tarFile, File destDir) throws IOException {
+ log.info("Expanding tar {} to {}", tarFile, destDir);
TarArchiveInputStream tarIn = new TarArchiveInputStream(
new GzipCompressorInputStream(
new BufferedInputStream(
@@ -491,11 +567,14 @@ public class AgentClientProvider extends AbstractClientProvider
try {
TarArchiveEntry tarEntry = tarIn.getNextTarEntry();
while (tarEntry != null) {
- File destPath = new File(agentPkgDir, tarEntry.getName());
+ File destPath = new File(destDir, tarEntry.getName());
+ File parent = destPath.getParentFile();
+ if (!parent.exists()) {
+ parent.mkdirs();
+ }
if (tarEntry.isDirectory()) {
destPath.mkdirs();
} else {
- destPath.createNewFile();
byte[] byteToRead = new byte[1024];
BufferedOutputStream buffOut =
new BufferedOutputStream(new FileOutputStream(destPath));
@@ -508,6 +587,9 @@ public class AgentClientProvider extends AbstractClientProvider
buffOut.close();
}
}
+ if ((tarEntry.getMode() & 0100) != 0) {
+ destPath.setExecutable(true);
+ }
tarEntry = tarIn.getNextTarEntry();
}
} finally {
@@ -515,6 +597,17 @@ public class AgentClientProvider extends AbstractClientProvider
}
}
+ private void retrieveConfigFile(RegistryOperations rops,
+ Configuration configuration, ConfigFile configFile, String name,
+ String user, File destDir) throws IOException, SliderException {
+ log.info("Retrieving config {} to {}", configFile.getDictionaryName(),
+ destDir);
+ PublishedConfiguration published = ClientUtils.getConfigFromRegistry(rops,
+ configuration, configFile.getDictionaryName(), name, user, true);
+ ClientUtils.saveOrReturnConfig(published, configFile.getType(),
+ destDir, configFile.getFileName());
+ }
+
protected JSONObject getCommandJson(JSONObject defaultConfig,
JSONObject inputConfig,
Metainfo metainfo,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index b027939..01a3f1a 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -48,6 +48,7 @@ public interface AgentKeys {
*/
String APP_HOME = "app.home";
String APP_ROOT = "site.global.app_root";
+ String APP_CLIENT_ROOT = "client_root";
/**
* Runas user of the application
*/
@@ -77,11 +78,16 @@ public interface AgentKeys {
String APP_RESOURCES = "application.resources";
String APP_RESOURCES_DIR = "app/resources";
+ String APP_CONF_DIR = "app/conf";
+
String AGENT_INSTALL_DIR = "infra/agent";
String APP_DEFINITION_DIR = "app/definition";
String ADDON_DEFINITION_DIR = "addon/definition";
String AGENT_CONFIG_FILE = "infra/conf/agent.ini";
String AGENT_VERSION_FILE = "infra/version";
+ String APP_PACKAGES_DIR = "app/packages";
+ String PER_COMPONENT = "per.component";
+ String PER_GROUP = "per.group";
String JAVA_HOME = "java_home";
String PACKAGE_LIST = "package_list";
@@ -97,6 +103,7 @@ public interface AgentKeys {
String CERT_FILE_LOCALIZATION_PATH = INFRA_RUN_SECURITY_DIR + "ca.crt";
String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
String TEST_RELAX_VERIFICATION = "test.relax.validation";
+ String AM_CONFIG_GENERATION = "am.config.generation";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index f20757a..4ffae7c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ProtocolTypes;
@@ -60,8 +61,11 @@ import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.CommandLineBuilder;
import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.registry.docstore.ConfigFormat;
+import org.apache.slider.core.registry.docstore.ConfigUtils;
import org.apache.slider.core.registry.docstore.ExportEntry;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
import org.apache.slider.core.registry.docstore.PublishedExports;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
import org.apache.slider.providers.AbstractProviderService;
@@ -124,7 +128,6 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Scanner;
import java.util.Set;
import java.util.TreeMap;
@@ -132,7 +135,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Pattern;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS;
@@ -169,6 +171,7 @@ public class AgentProviderService extends AbstractProviderService implements
private AgentClientProvider clientProvider;
private AtomicInteger taskId = new AtomicInteger(0);
private volatile Metainfo metaInfo = null;
+ private SliderFileSystem fileSystem = null;
private Map<String, DefaultConfig> defaultConfigs = null;
private ComponentCommandOrder commandOrder = null;
private HeartbeatMonitor monitor;
@@ -281,6 +284,7 @@ public class AgentProviderService extends AbstractProviderService implements
if (metaInfo == null) {
synchronized (syncLock) {
if (metaInfo == null) {
+ this.fileSystem = fileSystem;
readAndSetHeartbeatMonitoringInterval(instanceDefinition);
initializeAgentDebugCommands(instanceDefinition);
@@ -435,6 +439,26 @@ public class AgentProviderService extends AbstractProviderService implements
LocalResourceType.ARCHIVE);
launcher.addLocalResource(AgentKeys.APP_DEFINITION_DIR, appDefRes);
+ for (Package pkg : getMetaInfo().getApplication().getPackages()) {
+ Path pkgPath = fileSystem.buildResourcePath(pkg.getName());
+ if (!fileSystem.isFile(pkgPath)) {
+ pkgPath = fileSystem.buildResourcePath(getClusterName(),
+ pkg.getName());
+ }
+ if (!fileSystem.isFile(pkgPath)) {
+ throw new IOException("Package doesn't exist as a resource: " +
+ pkg.getName());
+ }
+ log.info("Adding resource {}", pkg.getName());
+ LocalResourceType type = LocalResourceType.FILE;
+ if ("archive".equals(pkg.getType())) {
+ type = LocalResourceType.ARCHIVE;
+ }
+ LocalResource packageResource = fileSystem.createAmResource(
+ pkgPath, type);
+ launcher.addLocalResource(AgentKeys.APP_PACKAGES_DIR, packageResource);
+ }
+
String agentConf = instanceDefinition.getAppConfOperations().
getGlobalOptions().getOption(AgentKeys.AGENT_CONF, "");
if (SliderUtils.isSet(agentConf)) {
@@ -476,6 +500,15 @@ public class AgentProviderService extends AbstractProviderService implements
generatedConfPath,
SliderKeys.PROPAGATED_CONF_DIR_NAME));
+ if (appComponent.getOptionBool(AgentKeys.AM_CONFIG_GENERATION, false)) {
+ // build and localize configuration files
+ Map<String, Map<String, String>> configurations =
+ buildCommandConfigurations(instanceDefinition.getAppConfOperations(),
+ container.getId().toString(), roleName, roleGroup);
+ localizeConfigFiles(launcher, roleName, roleGroup, getMetaInfo(),
+ configurations, launcher.getEnv(), fileSystem);
+ }
+
String label = getContainerLabel(container, roleName, roleGroup);
CommandLineBuilder operation = new CommandLineBuilder();
@@ -646,19 +679,41 @@ public class AgentProviderService extends AbstractProviderService implements
private Path uploadSecurityResource(File resource, SliderFileSystem fileSystem)
throws IOException {
Path certsDir = fileSystem.buildClusterSecurityDirPath(getClusterName());
- if (!fileSystem.getFileSystem().exists(certsDir)) {
- fileSystem.getFileSystem().mkdirs(certsDir,
+ return uploadResource(resource, fileSystem, certsDir);
+ }
+
+ private Path uploadResource(File resource, SliderFileSystem fileSystem,
+ String roleName) throws IOException {
+ Path dir;
+ if (roleName == null) {
+ dir = fileSystem.buildClusterResourcePath(getClusterName());
+ } else {
+ dir = fileSystem.buildClusterResourcePath(getClusterName(), roleName);
+ }
+ return uploadResource(resource, fileSystem, dir);
+ }
+
+ private static synchronized Path uploadResource(File resource,
+ SliderFileSystem fileSystem, Path parentDir) throws IOException {
+ if (!fileSystem.getFileSystem().exists(parentDir)) {
+ fileSystem.getFileSystem().mkdirs(parentDir,
new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
}
- Path destPath = new Path(certsDir, resource.getName());
+ Path destPath = new Path(parentDir, resource.getName());
if (!fileSystem.getFileSystem().exists(destPath)) {
- FSDataOutputStream os = fileSystem.getFileSystem().create(destPath);
- byte[] contents = FileUtils.readFileToByteArray(resource);
- os.write(contents, 0, contents.length);
-
- os.flush();
- os.close();
+ FSDataOutputStream os = null;
+ try {
+ os = fileSystem.getFileSystem().create(destPath);
+ byte[] contents = FileUtils.readFileToByteArray(resource);
+ os.write(contents, 0, contents.length);
+ os.flush();
+ } finally {
+ IOUtils.closeStream(os);
+ }
log.info("Uploaded {} to localization path {}", resource, destPath);
+ } else {
+ log.info("Resource {} already existed at localization path {}", resource,
+ destPath);
}
while (!fileSystem.getFileSystem().exists(destPath)) {
@@ -718,6 +773,69 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
+ private void createConfigFile(SliderFileSystem fileSystem, File file,
+ ConfigFile configFile, Map<String, String> config)
+ throws IOException {
+ ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType());
+ log.info("Writing {} file {}", configFormat, file);
+
+ ConfigUtils.prepConfigForTemplateOutputter(configFormat, config,
+ fileSystem, getClusterName(), file.getName());
+ PublishedConfiguration publishedConfiguration =
+ new PublishedConfiguration(configFile.getDictionaryName(),
+ config.entrySet());
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(configFormat,
+ publishedConfiguration);
+ configurationOutputter.save(file);
+ }
+
+ @VisibleForTesting
+ protected void localizeConfigFiles(ContainerLauncher launcher,
+ String roleName, String roleGroup,
+ Metainfo metainfo,
+ Map<String, Map<String, String>> configs,
+ MapOperations env,
+ SliderFileSystem fileSystem)
+ throws IOException {
+ for (ConfigFile configFile : metainfo.getComponentConfigFiles(roleGroup)) {
+ Map<String, String> config = ConfigUtils.replacePropsInConfig(
+ configs.get(configFile.getDictionaryName()), env.options);
+ String fileName = ConfigUtils.replaceProps(config,
+ configFile.getFileName());
+ File localFile = new File(SliderKeys.RESOURCE_DIR);
+ if (!localFile.exists()) {
+ localFile.mkdir();
+ }
+ localFile = new File(localFile, new File(fileName).getName());
+
+ String folder = null;
+ if ("true".equals(config.get(PER_COMPONENT))) {
+ folder = roleName;
+ } else if ("true".equals(config.get(PER_GROUP))) {
+ folder = roleGroup;
+ }
+
+ log.info("Localizing {} configs to config file {} (destination {}) " +
+ "based on {} configs", config.size(), localFile, fileName,
+ configFile.getDictionaryName());
+ createConfigFile(fileSystem, localFile, configFile, config);
+ Path destPath = uploadResource(localFile, fileSystem, folder);
+ LocalResource configResource = fileSystem.createAmResource(destPath,
+ LocalResourceType.FILE);
+
+ File destFile = new File(fileName);
+ if (destFile.isAbsolute()) {
+ launcher.addLocalResource(
+ SliderKeys.RESOURCE_DIR + "/" + destFile.getName(),
+ configResource, fileName);
+ } else {
+ launcher.addLocalResource(AgentKeys.APP_CONF_DIR + "/" + fileName,
+ configResource);
+ }
+ }
+ }
+
/**
* build the zookeeper registry path.
*
@@ -1242,6 +1360,69 @@ public class AgentProviderService extends AbstractProviderService implements
} catch (URISyntaxException e) {
throw new IOException(e);
}
+
+ // identify client component
+ Component client = null;
+ for (Component component : getMetaInfo().getApplication().getComponents()) {
+ if (component.getCategory().equals("CLIENT")) {
+ client = component;
+ break;
+ }
+ }
+ if (client == null) {
+ log.info("No client component specified, not publishing client configs");
+ return;
+ }
+
+ // register AM-generated client configs
+ ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
+ MapOperations clientOperations = appConf.getOrAddComponent(client.getName());
+ appConf.resolve();
+ if (!clientOperations.getOptionBool(AgentKeys.AM_CONFIG_GENERATION,
+ false)) {
+ log.info("AM config generation is false, not publishing client configs");
+ return;
+ }
+
+ // build and localize configuration files
+ Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>();
+ Map<String, String> tokens = null;
+ try {
+ tokens = getStandardTokenMap(appConf, client.getName(), client.getName());
+ } catch (SliderException e) {
+ throw new IOException(e);
+ }
+
+ for (ConfigFile configFile : getMetaInfo()
+ .getComponentConfigFiles(client.getName())) {
+ addNamedConfiguration(configFile.getDictionaryName(),
+ appConf.getGlobalOptions().options, configurations, tokens, null,
+ client.getName());
+ if (appConf.getComponent(client.getName()) != null) {
+ addNamedConfiguration(configFile.getDictionaryName(),
+ appConf.getComponent(client.getName()).options, configurations,
+ tokens, null, client.getName());
+ }
+ }
+
+ //do a final replacement of re-used configs
+ dereferenceAllConfigs(configurations);
+
+ for (ConfigFile configFile : getMetaInfo()
+ .getComponentConfigFiles(client.getName())) {
+ ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType());
+
+ Map<String, String> config = configurations.get(configFile.getDictionaryName());
+ ConfigUtils.prepConfigForTemplateOutputter(configFormat, config,
+ fileSystem, getClusterName(),
+ new File(configFile.getFileName()).getName());
+ PublishedConfiguration publishedConfiguration =
+ new PublishedConfiguration(configFile.getDictionaryName(),
+ config.entrySet());
+ getAmState().getPublishedSliderConfigurations().put(
+ configFile.getDictionaryName(), publishedConfiguration);
+ log.info("Publishing AM configuration {}", configFile.getDictionaryName());
+ }
}
@Override
@@ -1585,7 +1766,9 @@ public class AgentProviderService extends AbstractProviderService implements
if (status.getConfigs() != null) {
Application application = getMetaInfo().getApplication();
- if (canAnyMasterPublishConfig() == false || canPublishConfig(componentGroup)) {
+ if ((!canAnyMasterPublishConfig() || canPublishConfig(componentGroup)) &&
+ !getAmState().getAppConfSnapshot().getComponentOptBool(
+ componentGroup, AgentKeys.AM_CONFIG_GENERATION, false)) {
// If no Master can explicitly publish then publish if its a master
// Otherwise, wait till the master that can publish is ready
@@ -1709,7 +1892,11 @@ public class AgentProviderService extends AbstractProviderService implements
simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue());
}
}
- publishApplicationInstanceData(groupName, groupName, simpleEntries.entrySet());
+ if (!getAmState().getAppConfSnapshot().getComponentOptBool(
+ groupName, AgentKeys.AM_CONFIG_GENERATION, false)) {
+ publishApplicationInstanceData(groupName, groupName,
+ simpleEntries.entrySet());
+ }
PublishedExports exports = new PublishedExports(groupName);
exports.setUpdated(new Date().getTime());
@@ -2036,7 +2223,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setConfigurations(configurations);
Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
cmd.setComponentConfigurations(componentConfigurations);
-
+
if (SliderUtils.isSet(scriptPath)) {
cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false));
} else {
@@ -2154,10 +2341,10 @@ public class AgentProviderService extends AbstractProviderService implements
List<String> packages = new ArrayList<>();
if (application != null) {
if (application.getPackages().size() > 0) {
- List<Package> appPackages = application.getPackages();
- for (Package appPackage : appPackages) {
- packages.add(String.format(pkgFormatString, appPackage.getType(), appPackage.getName()));
- }
+ // no-op if there are packages that are not OS-specific, as these
+ // will be localized by AM rather than the Agent
+ // this should be backwards compatible, as there was previously an
+ // XML parsing bug that ensured non-OS-specific packages did not exist
} else {
List<OSSpecific> osSpecifics = application.getOSSpecifics();
if (osSpecifics != null && osSpecifics.size() > 0) {
@@ -2821,14 +3008,41 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
+ boolean finished = false;
+ while (!finished) {
+ finished = true;
+ for (Map.Entry<String, String> entry : allConfigs.entrySet()) {
+ String configValue = entry.getValue();
+ for (Map.Entry<String, String> lookUpEntry : allConfigs.entrySet()) {
+ String lookUpValue = lookUpEntry.getValue();
+ if (lookUpValue.contains("${@//site/")) {
+ continue;
+ }
+ String lookUpKey = lookUpEntry.getKey();
+ if (configValue != null && configValue.contains(lookUpKey)) {
+ configValue = configValue.replace(lookUpKey, lookUpValue);
+ }
+ }
+ if (!configValue.equals(entry.getValue())) {
+ finished = false;
+ allConfigs.put(entry.getKey(), configValue);
+ }
+ }
+ }
+
for (String configType : configurations.keySet()) {
Map<String, String> configBucket = configurations.get(configType);
for (Map.Entry<String, String> entry: configBucket.entrySet()) {
String configName = entry.getKey();
String configValue = entry.getValue();
- for (String lookUpKey : allConfigs.keySet()) {
+ for (Map.Entry<String, String> lookUpEntry : allConfigs.entrySet()) {
+ String lookUpValue = lookUpEntry.getValue();
+ if (lookUpValue.contains("${@//site/")) {
+ continue;
+ }
+ String lookUpKey = lookUpEntry.getKey();
if (configValue != null && configValue.contains(lookUpKey)) {
- configValue = configValue.replace(lookUpKey, allConfigs.get(lookUpKey));
+ configValue = configValue.replace(lookUpKey, lookUpValue);
}
}
configBucket.put(configName, configValue);
@@ -2974,6 +3188,7 @@ public class AgentProviderService extends AbstractProviderService implements
config.put("app_log_dir", "${AGENT_LOG_ROOT}");
config.put("app_pid_dir", "${AGENT_WORK_ROOT}/app/run");
config.put("app_install_dir", "${AGENT_WORK_ROOT}/app/install");
+ config.put("app_conf_dir", "${AGENT_WORK_ROOT}/" + AgentKeys.APP_CONF_DIR);
config.put("app_input_conf_dir", "${AGENT_WORK_ROOT}/" + SliderKeys.PROPAGATED_CONF_DIR_NAME);
config.put("app_container_id", containerId);
config.put("app_container_tag", tags.getTag(roleName, containerId));
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java
index 1b63b58..b6ae4de 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java
@@ -65,6 +65,10 @@ public abstract class AbstractComponent implements Validate {
this.commands = commands;
}
+ public void addCommand(ComponentCommand command) {
+ commands.add(command);
+ }
+
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("{");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
index 63546a4..5556c7f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
@@ -116,6 +116,10 @@ public class Application extends AbstractMetainfoSchema {
return commandOrders;
}
+ public void addPackage(Package pkg) {
+ packages.add(pkg);
+ }
+
@JsonProperty("packages")
public List<Package> getPackages() {
return packages;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
index 3f23455..78bb8c1 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -38,7 +38,8 @@ public class Component extends AbstractComponent {
String type = TYPE_STANDARD;
List<ComponentExport> componentExports = new ArrayList<>();
List<DockerContainer> dockerContainers = new ArrayList<>();
-
+ List<ConfigFile> configFiles = new ArrayList<>();
+
public Component() {
}
@@ -155,6 +156,15 @@ public class Component extends AbstractComponent {
return Boolean.parseBoolean(this.autoStartOnFailure);
}
+ public void addConfigFile(ConfigFile configFile) {
+ this.configFiles.add(configFile);
+ }
+
+ @JsonProperty("configFiles")
+ public List<ConfigFile> getConfigFiles() {
+ return configFiles;
+ }
+
@Override
public String toString() {
final StringBuilder sb =