You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bi...@apache.org on 2016/05/13 01:55:07 UTC
[6/7] incubator-slider git commit: SLIDER-1107 fix AM-generated
config publishing and race condition in config generation
SLIDER-1107 fix AM-generated config publishing and race condition in config generation
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1699a714
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1699a714
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1699a714
Branch: refs/heads/feature/SLIDER-1107_AM_config_generation
Commit: 1699a714dfcea6473d9256a7e0a4536976485acc
Parents: c7a7c0d
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Thu May 12 18:22:08 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Thu May 12 18:51:38 2016 -0700
----------------------------------------------------------------------
.../core/registry/docstore/ConfigUtils.java | 37 ++++++
.../docstore/PublishedConfiguration.java | 22 ----
.../PublishedConfigurationOutputter.java | 53 +--------
.../providers/agent/AgentProviderService.java | 113 ++++++++++++++-----
4 files changed, 120 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1699a714/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
index 7e6ee5a..2e1615b 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
@@ -17,6 +17,10 @@
*/
package org.apache.slider.core.registry.docstore;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.SliderFileSystem;
+
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
@@ -24,6 +28,8 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class ConfigUtils {
+ public static final String TEMPLATE_FILE = "template.file";
+
public static String replaceProps(Map<String, String> config, String content) {
Map<String, String> tokens = new HashMap<>();
for (Entry<String, String> entry : config.entrySet()) {
@@ -56,4 +62,35 @@ public class ConfigUtils {
return newConfig;
}
+ public static void prepConfigForTemplateOutputter(ConfigFormat configFormat,
+ Map<String, String> config, SliderFileSystem fileSystem,
+ String clusterName, String fileName) throws IOException {
+ if (!configFormat.equals(ConfigFormat.TEMPLATE)) {
+ return;
+ }
+ Path templateFile = null;
+ if (config.containsKey(TEMPLATE_FILE)) {
+ templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE));
+ if (!fileSystem.isFile(templateFile)) {
+ templateFile = fileSystem.buildResourcePath(clusterName,
+ config.get(TEMPLATE_FILE));
+ }
+ if (!fileSystem.isFile(templateFile)) {
+ throw new IOException("config specified template file " + config
+ .get(TEMPLATE_FILE) + " but " + templateFile + " doesn't exist");
+ }
+ }
+ if (templateFile == null && fileName != null) {
+ templateFile = fileSystem.buildResourcePath(fileName);
+ if (!fileSystem.isFile(templateFile)) {
+ templateFile = fileSystem.buildResourcePath(clusterName,
+ fileName);
+ }
+ }
+ if (fileSystem.isFile(templateFile)) {
+ config.put("content", fileSystem.cat(templateFile));
+ } else {
+ config.put("content", "");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1699a714/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
index cabdce6..477f7d3 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
@@ -20,7 +20,6 @@ package org.apache.slider.core.registry.docstore;
import org.apache.hadoop.conf.Configuration;
import org.apache.slider.common.tools.ConfigHelper;
-import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.exceptions.BadConfigException;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.ObjectMapper;
@@ -50,9 +49,6 @@ public class PublishedConfiguration {
public Map<String, String> entries = new HashMap<>();
- public SliderFileSystem fileSystem;
- public String clusterName;
-
public PublishedConfiguration() {
}
@@ -90,24 +86,6 @@ public class PublishedConfiguration {
}
/**
- * Build a configuration from the entries
- * @param description configuration description
- * @param entries entries to put
- * @param fileSystem Slider file system (source of configuration templates)
- * @param clusterName cluster name
- */
- public PublishedConfiguration(String description,
- Iterable<Map.Entry<String, String>> entries,
- SliderFileSystem fileSystem,
- String clusterName) {
- this.description = description;
- putValues(entries);
- this.fileSystem = fileSystem;
- this.clusterName = clusterName;
- }
-
-
- /**
* Is the configuration empty. This means either that it has not
* been given any values, or it is stripped down copy set down over the
* wire.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1699a714/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
index 23cfc8f..9bdcfcb 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
@@ -23,9 +23,7 @@ import com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.ConfigHelper;
-import org.apache.slider.common.tools.SliderFileSystem;
import org.yaml.snakeyaml.DumperOptions;
import org.yaml.snakeyaml.DumperOptions.FlowStyle;
import org.yaml.snakeyaml.Yaml;
@@ -34,7 +32,6 @@ import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.StringWriter;
-import java.util.Map;
import java.util.Properties;
/**
@@ -188,58 +185,10 @@ public abstract class PublishedConfigurationOutputter {
}
}
- public static class TemplateOutputter extends PublishedConfigurationOutputter {
-
- public static final String TEMPLATE_FILE = "template.file";
-
+ public static class TemplateOutputter extends EnvOutputter {
public TemplateOutputter(PublishedConfiguration owner) {
super(owner);
}
-
- @Override
- public void save(File dest) throws IOException {
- FileUtils.writeStringToFile(dest, asString(dest.getName()),
- Charsets.UTF_8);
- }
-
- public String asString(String fileName) throws IOException {
- if (owner.fileSystem == null) {
- throw new IOException("File system not specified for template " +
- "configuration");
- }
- Map<String,String> config = owner.entries;
- SliderFileSystem fileSystem = owner.fileSystem;
- Path templateFile = null;
- if (config.containsKey(TEMPLATE_FILE)) {
- templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE));
- if (!fileSystem.isFile(templateFile)) {
- templateFile = fileSystem.buildResourcePath(owner.clusterName,
- config.get(TEMPLATE_FILE));
- }
- if (!fileSystem.isFile(templateFile)) {
- throw new IOException("config specified template file " + config
- .get(TEMPLATE_FILE) + " for config " + owner.description +
- " but " + templateFile + " doesn't exist");
- }
- }
- if (templateFile == null && fileName != null) {
- templateFile = fileSystem.buildResourcePath(fileName);
- if (!fileSystem.isFile(templateFile)) {
- templateFile = fileSystem.buildResourcePath(owner.clusterName,
- fileName);
- }
- }
- if (fileSystem.isFile(templateFile)) {
- return ConfigUtils.replaceProps(config, fileSystem.cat(templateFile));
- } else {
- return "";
- }
- }
-
- @Override
- public String asString() throws IOException {
- return asString(null);
- }
}
public static class YamlOutputter extends PublishedConfigurationOutputter {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1699a714/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index ab4da47..155ff09 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -170,6 +170,8 @@ public class AgentProviderService extends AbstractProviderService implements
private AgentClientProvider clientProvider;
private AtomicInteger taskId = new AtomicInteger(0);
private volatile Metainfo metaInfo = null;
+ private AggregateConf instanceDefinition = null;
+ private SliderFileSystem fileSystem = null;
private Map<String, DefaultConfig> defaultConfigs = null;
private ComponentCommandOrder commandOrder = null;
private HeartbeatMonitor monitor;
@@ -282,6 +284,8 @@ public class AgentProviderService extends AbstractProviderService implements
if (metaInfo == null) {
synchronized (syncLock) {
if (metaInfo == null) {
+ this.instanceDefinition = instanceDefinition;
+ this.fileSystem = fileSystem;
readAndSetHeartbeatMonitoringInterval(instanceDefinition);
initializeAgentDebugCommands(instanceDefinition);
@@ -679,21 +683,6 @@ public class AgentProviderService extends AbstractProviderService implements
return uploadResource(resource, fileSystem, certsDir);
}
- private Path checkResourceExists(File resource, SliderFileSystem
- fileSystem, String roleName) throws IOException {
- Path dir;
- if (roleName == null) {
- dir = fileSystem.buildClusterResourcePath(getClusterName());
- } else {
- dir = fileSystem.buildClusterResourcePath(getClusterName(), roleName);
- }
- Path destPath = new Path(dir, resource.getName());
- if (fileSystem.getFileSystem().exists(destPath)) {
- return destPath;
- }
- return null;
- }
-
private Path uploadResource(File resource, SliderFileSystem fileSystem,
String roleName) throws IOException {
Path dir;
@@ -705,8 +694,8 @@ public class AgentProviderService extends AbstractProviderService implements
return uploadResource(resource, fileSystem, dir);
}
- private static Path uploadResource(File resource, SliderFileSystem fileSystem,
- Path parentDir) throws IOException {
+ private static synchronized Path uploadResource(File resource,
+ SliderFileSystem fileSystem, Path parentDir) throws IOException {
if (!fileSystem.getFileSystem().exists(parentDir)) {
fileSystem.getFileSystem().mkdirs(parentDir,
new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
@@ -783,14 +772,19 @@ public class AgentProviderService extends AbstractProviderService implements
}
private void createConfigFile(SliderFileSystem fileSystem, File file,
- ConfigFile configFile, Map<String, String> config) throws IOException {
+ ConfigFile configFile, Map<String, String> config)
+ throws IOException {
ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType());
log.info("Writing {} file {}", configFormat, file);
+ ConfigUtils.prepConfigForTemplateOutputter(configFormat, config,
+ fileSystem, getClusterName(), file.getName());
+ PublishedConfiguration publishedConfiguration =
+ new PublishedConfiguration(configFile.getDictionaryName(),
+ config.entrySet());
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(configFormat,
- new PublishedConfiguration(configFile.getDictionaryName(),
- config.entrySet(), fileSystem, getClusterName()));
+ publishedConfiguration);
configurationOutputter.save(file);
}
@@ -820,16 +814,11 @@ public class AgentProviderService extends AbstractProviderService implements
folder = roleGroup;
}
- Path destPath = checkResourceExists(localFile, fileSystem, folder);
- if (destPath == null) {
- log.info("Localizing {} configs to config file {} (destination {}) " +
- "based on {} configs",
- config.size(), localFile, fileName, configFile.getDictionaryName());
- createConfigFile(fileSystem, localFile, configFile, config);
- destPath = uploadResource(localFile, fileSystem, folder);
- } else {
- log.info("Config already exists at {}, not recreating it", destPath);
- }
+ log.info("Localizing {} configs to config file {} (destination {}) " +
+ "based on {} configs", config.size(), localFile, fileName,
+ configFile.getDictionaryName());
+ createConfigFile(fileSystem, localFile, configFile, config);
+ Path destPath = uploadResource(localFile, fileSystem, folder);
LocalResource configResource = fileSystem.createAmResource(destPath,
LocalResourceType.FILE);
@@ -1033,7 +1022,7 @@ public class AgentProviderService extends AbstractProviderService implements
role = amState.getOwnedContainer(containerId);
role.ip = status.getIp();
}
- if(status.getHostname() != null & !status.getHostname().isEmpty()){
+ if(status.getHostname() != null && !status.getHostname().isEmpty()){
role = amState.getOwnedContainer(containerId);
role.hostname = status.getHostname();
}
@@ -1369,6 +1358,68 @@ public class AgentProviderService extends AbstractProviderService implements
} catch (URISyntaxException e) {
throw new IOException(e);
}
+
+ // identify client component
+ Component client = null;
+ for (Component component : getMetaInfo().getApplication().getComponents()) {
+ if (component != null && component.getCategory().equals("CLIENT")) {
+ client = component;
+ break;
+ }
+ }
+ if (client == null) {
+ log.info("No client component specified, not publishing client configs");
+ return;
+ }
+
+ // register AM-generated client configs
+ ConfTreeOperations appConf = instanceDefinition.getAppConfOperations();
+ MapOperations clientOperations = instanceDefinition.getAppConfOperations()
+ .getOrAddComponent(client.getName());
+ if (!clientOperations.getOptionBool(AgentKeys.AM_CONFIG_GENERATION,
+ false)) {
+ log.info("AM config generation is false, not publishing client configs");
+ return;
+ }
+
+ // build and localize configuration files
+ Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>();
+ Map<String, String> tokens = null;
+ try {
+ tokens = getStandardTokenMap(appConf, client.getName(), client.getName());
+ } catch (SliderException e) {
+ throw new IOException(e);
+ }
+
+ for (ConfigFile configFile : getMetaInfo()
+ .getComponentConfigFiles(client.getName())) {
+ addNamedConfiguration(configFile.getDictionaryName(),
+ appConf.getGlobalOptions().options, configurations, tokens, null,
+ client.getName());
+ if (appConf.getComponent(client.getName()) != null) {
+ addNamedConfiguration(configFile.getDictionaryName(),
+ appConf.getComponent(client.getName()).options, configurations,
+ tokens, null, client.getName());
+ }
+ }
+
+ //do a final replacement of re-used configs
+ dereferenceAllConfigs(configurations);
+
+ for (ConfigFile configFile : getMetaInfo()
+ .getComponentConfigFiles(client.getName())) {
+ ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType());
+
+ Map<String, String> config = configurations.get(configFile.getDictionaryName());
+ ConfigUtils.prepConfigForTemplateOutputter(configFormat, config,
+ fileSystem, getClusterName(),
+ new File(configFile.getFileName()).getName());
+ PublishedConfiguration publishedConfiguration =
+ new PublishedConfiguration(configFile.getDictionaryName(),
+ config.entrySet());
+ getAmState().getPublishedSliderConfigurations().put(
+ configFile.getDictionaryName(), publishedConfiguration);
+ }
}
@Override