You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/05/07 14:48:05 UTC
svn commit: r1592996 - in /incubator/slider/trunk/slider-core/src:
main/java/org/apache/slider/core/registry/docstore/
main/java/org/apache/slider/server/appmaster/
test/groovy/org/apache/slider/agent/standalone/
test/groovy/org/apache/slider/registry/
Author: stevel
Date: Wed May 7 12:48:04 2014
New Revision: 1592996
URL: http://svn.apache.org/r1592996
Log:
SLIDER-17 limit conf filenames to exclude ".", make filetype the suffix
Added:
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java (with props)
incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/TestConfigSetNaming.groovy
Modified:
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java?rev=1592996&r1=1592995&r2=1592996&view=diff
==============================================================================
--- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java (original)
+++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java Wed May 7 12:48:04 2014
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
+import java.util.regex.Pattern;
/**
* Represents a set of configurations for an application, component, etc.
@@ -34,13 +35,32 @@ import java.util.TreeSet;
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class PublishedConfigSet {
+ public static final String VALID_NAME_PATTERN = "[A-Za-z0-9_-]+";
+ public static final String E_INVALID_NAME =
+ "Invalid configuration name -it must match the pattern " +
+ VALID_NAME_PATTERN;
+ private static final Pattern validNames = Pattern.compile(VALID_NAME_PATTERN);
+
public Map<String, PublishedConfiguration> configurations =
new HashMap<>();
public void put(String name, PublishedConfiguration conf) {
+ validateName(name);
configurations.put(name, conf);
}
+ /**
+ * Validate the name -restricting it to the set defined in
+ * {@link #VALID_NAME_PATTERN}
+ * @param name name to validate
+ * @throws IllegalArgumentException if not
+ */
+ public static void validateName(String name) {
+ if (!validNames.matcher(name).matches()) {
+ throw new IllegalArgumentException(E_INVALID_NAME);
+ }
+ }
+
public PublishedConfiguration get(String name) {
return configurations.get(name);
}
Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java?rev=1592996&r1=1592995&r2=1592996&view=diff
==============================================================================
--- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java (original)
+++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java Wed May 7 12:48:04 2014
@@ -46,17 +46,21 @@ public class PublishedConfiguration {
public String updatedTime;
- public void setUpdated(long updated) {
- this.updated = updated;
- this.updatedTime = new Date(updated).toString();
- }
+ public Map<String, String> entries = new HashMap<>();
- public long getUpdated() {
- return updated;
+ public PublishedConfiguration() {
}
- public Map<String, String> entries = new HashMap<>();
+ public PublishedConfiguration(String description) {
+ this.description = description;
+ }
+ public PublishedConfiguration(String description,
+ Iterable<Map.Entry<String, String>> entries) {
+ this.description = description;
+ putValues(entries);
+ }
+
/**
* Is the configuration empty. This means either that it has not
* been given any values, or it is stripped down copy set down over the
@@ -67,6 +71,16 @@ public class PublishedConfiguration {
return entries.isEmpty();
}
+
+ public void setUpdated(long updated) {
+ this.updated = updated;
+ this.updatedTime = new Date(updated).toString();
+ }
+
+ public long getUpdated() {
+ return updated;
+ }
+
/**
* Set the values from an iterable (this includes a Hadoop Configuration
* and Java properties object).
@@ -140,7 +154,7 @@ public class PublishedConfiguration {
final StringBuilder sb =
new StringBuilder("PublishedConfiguration{");
sb.append("description='").append(description).append('\'');
- sb.append("entries = ").append(entries.size());
+ sb.append(" entries = ").append(entries.size());
sb.append('}');
return sb.toString();
}
Added: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java?rev=1592996&view=auto
==============================================================================
--- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java (added)
+++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java Wed May 7 12:48:04 2014
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+/**
+ * This is the name of YARN artifacts that are published
+ */
+public interface PublishedArtifacts {
+
+ String COMPLETE_CONFIG = "complete-config";
+ String CORE_SITE_CONFIG = "core-site";
+ String HDFS_SITE_CONFIG = "hdfs-site";
+ String YARN_SITE_CONFIG = "yarn-site";
+}
Propchange: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java?rev=1592996&r1=1592995&r2=1592996&view=diff
==============================================================================
--- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java (original)
+++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java Wed May 7 12:48:04 2014
@@ -710,11 +710,28 @@ public class SliderAppMaster extends Abs
log.info("service instances already running: {}", serviceInstancesRunning);
- // now publish yarn-site.xml
- PublishedConfiguration pubconf = new PublishedConfiguration();
- pubconf.description = "YARN site settings";
- pubconf.putValues(new YarnConfiguration());
- appState.getPublishedConfigurations().put("yarn-site.xml", pubconf);
+ // now publish site.xml files
+ appState.getPublishedConfigurations().put(
+ PublishedArtifacts.COMPLETE_CONFIG,
+ new PublishedConfiguration(
+ "Complete site settings",
+ new YarnConfiguration()));
+ appState.getPublishedConfigurations().put(
+ PublishedArtifacts.YARN_SITE_CONFIG,
+ new PublishedConfiguration(
+ "YARN site settings",
+ ConfigHelper.loadFromResource("yarn-site.xml")));
+
+ appState.getPublishedConfigurations().put(
+ PublishedArtifacts.CORE_SITE_CONFIG,
+ new PublishedConfiguration(
+ "Core site settings",
+ ConfigHelper.loadFromResource("core-site.xml")));
+ appState.getPublishedConfigurations().put(
+ PublishedArtifacts.HDFS_SITE_CONFIG,
+ new PublishedConfiguration(
+ "HDFS site settings",
+ ConfigHelper.loadFromResource("hdfs-site.xml")));
ServiceInstanceData instanceData = new ServiceInstanceData();
instanceData.id = registryId;
Modified: incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy?rev=1592996&r1=1592995&r2=1592996&view=diff
==============================================================================
--- incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy (original)
+++ incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy Wed May 7 12:48:04 2014
@@ -30,12 +30,12 @@ import org.apache.slider.common.SliderKe
import org.apache.slider.common.params.ActionRegistryArgs
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.persist.JsonSerDeser
-import org.apache.slider.core.registry.docstore.ConfigFormat
import org.apache.slider.core.registry.docstore.PublishedConfigSet
import org.apache.slider.core.registry.docstore.PublishedConfiguration
import org.apache.slider.core.registry.info.CustomRegistryConstants
import org.apache.slider.core.registry.info.ServiceInstanceData
import org.apache.slider.core.registry.retrieve.RegistryRetriever
+import org.apache.slider.server.appmaster.PublishedArtifacts
import org.apache.slider.server.appmaster.web.rest.RestPaths
import org.apache.slider.server.services.curator.CuratorServiceInstance
import org.apache.slider.server.services.curator.RegistryBinderService
@@ -51,7 +51,7 @@ import org.junit.Test
class TestStandaloneRegistryAM extends AgentMiniClusterTestBase {
- public static final String YARN_SITE = "yarn-site.xml"
+ public static final String ARTIFACT_NAME = PublishedArtifacts.COMPLETE_CONFIG
@Test
public void testRegistryAM() throws Throwable {
@@ -152,13 +152,13 @@ class TestStandaloneRegistryAM extends A
PublishedConfigSet)
def configSet = serDeser.fromJson(publishedJSON)
assert configSet.size() >= 1
- assert configSet.contains(YARN_SITE)
- PublishedConfiguration publishedYarnSite = configSet.get(YARN_SITE)
+ assert configSet.contains(ARTIFACT_NAME)
+ PublishedConfiguration publishedYarnSite = configSet.get(ARTIFACT_NAME)
assert publishedYarnSite.empty
//get the full URL
- def yarnSitePublisher = appendToURL(publisher, YARN_SITE)
+ def yarnSitePublisher = appendToURL(publisher, ARTIFACT_NAME)
String confJSON = GET(yarnSitePublisher)
log.info(confJSON)
@@ -195,10 +195,10 @@ class TestStandaloneRegistryAM extends A
def config = externalConf.get(key)
log.info "$key -- ${config.description}"
}
- assert externalConf[YARN_SITE]
+ assert externalConf[ARTIFACT_NAME]
- def yarnSite = retriever.retrieveConfiguration(YARN_SITE, true)
+ def yarnSite = retriever.retrieveConfiguration(ARTIFACT_NAME, true)
assert !yarnSite.empty
def siteXML = yarnSite.asConfiguration()
def rmAddr = siteXML.get(YarnConfiguration.RM_ADDRESS)
@@ -210,7 +210,7 @@ class TestStandaloneRegistryAM extends A
try {
retriever.getConfigurations(false)
fail( "expected a failure")
- } catch (FileNotFoundException fnfe) {
+ } catch (FileNotFoundException expected) {
//expected
}
@@ -243,7 +243,7 @@ class TestStandaloneRegistryAM extends A
try {
assert 0 == client.actionRegistry(registryArgs)
fail("expected a failure")
- } catch (FileNotFoundException fnfe) {
+ } catch (FileNotFoundException expected) {
//expected
}
@@ -251,7 +251,7 @@ class TestStandaloneRegistryAM extends A
registryArgs.listConf = false
registryArgs.internal = false
registryArgs.format = "properties"
- registryArgs.getConf = YARN_SITE
+ registryArgs.getConf = ARTIFACT_NAME
describe registryArgs.toString()
assert 0 == client.actionRegistry(registryArgs)
@@ -262,11 +262,11 @@ class TestStandaloneRegistryAM extends A
registryArgs.dest = outputDir
describe registryArgs.toString()
assert 0 == client.actionRegistry(registryArgs)
- assert new File(outputDir,YARN_SITE + ".properties").exists()
+ assert new File(outputDir,ARTIFACT_NAME + ".properties").exists()
registryArgs.format = "xml"
assert 0 == client.actionRegistry(registryArgs)
- assert new File(outputDir,YARN_SITE + ".xml").exists()
+ assert new File(outputDir,ARTIFACT_NAME + ".xml").exists()
@@ -282,6 +282,7 @@ class TestStandaloneRegistryAM extends A
sleep(20000)
+ // now verify that the service is not in the registry
instances = client.listRegistryInstances()
assert instances.size() == 0
Added: incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/TestConfigSetNaming.groovy
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/TestConfigSetNaming.groovy?rev=1592996&view=auto
==============================================================================
--- incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/TestConfigSetNaming.groovy (added)
+++ incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/TestConfigSetNaming.groovy Wed May 7 12:48:04 2014
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.registry
+
+import groovy.transform.CompileStatic
+import org.apache.slider.core.registry.docstore.PublishedConfigSet
+import org.junit.Assert
+import org.junit.Test
+
+@CompileStatic
+class TestConfigSetNaming {
+
+ def assertValid(String name) {
+ PublishedConfigSet.validateName(name)
+ }
+ def assertInvalid(String name) {
+ try {
+ PublishedConfigSet.validateName(name)
+ Assert.fail("Invalid name was unexpectedly parsed: $name")
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testLowerCase() throws Throwable {
+ assertValid("abcdefghijklmnopqrstuvwxyz")
+ }
+
+ @Test
+ public void testUpperCase() throws Throwable {
+ assertValid("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+ }
+
+ @Test
+ public void testNumbers() throws Throwable {
+ assertValid("01234567890")
+ }
+
+ @Test
+ public void testChars() throws Throwable {
+ assertValid("-_")
+ }
+
+ @Test
+ public void testInvalids() throws Throwable {
+ [
+ "",
+ " ",
+ "*",
+ "a/b",
+ "b\\a",
+ '"',
+ "'",
+ "\u0000",
+ "\u0f00",
+ ""
+ ].each {String s -> assertInvalid(s)}
+ }
+}