You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/07/25 18:03:28 UTC
[46/50] [abbrv] hadoop git commit: YARN-6613. Update json validation
for new native services providers. Contributed by Billie Rinaldi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java
deleted file mode 100644
index bf6ee2c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.utils.SliderTestBase;
-import org.junit.Test;
-
-import java.net.URI;
-
-/**
- * Test slider utils.
- */
-public class TestMiscSliderUtils extends SliderTestBase {
-
-
- public static final String CLUSTER1 = "cluster1";
-
- @Test
- public void testPurgeTempDir() throws Throwable {
-
- Configuration configuration = new Configuration();
- FileSystem fs = FileSystem.get(new URI("file:///"), configuration);
- SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
- Path inst = sliderFileSystem.createAppInstanceTempPath(CLUSTER1, "001");
-
- assertTrue(fs.exists(inst));
- sliderFileSystem.purgeAppInstanceTempFiles(CLUSTER1);
- assertFalse(fs.exists(inst));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleAppJson.java
new file mode 100644
index 0000000..1700771
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleAppJson.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.conf;
+
+import org.apache.slider.api.resource.Application;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER;
+
+/**
+ * Names of the example configs.
+ */
+public final class ExampleAppJson {
+
+ public static final String APP_JSON = "app.json";
+ public static final String OVERRIDE_JSON = "app-override.json";
+ public static final String DEFAULT_JSON = "default.json";
+ public static final String EXTERNAL_JSON_0 = "external0.json";
+ public static final String EXTERNAL_JSON_1 = "external1.json";
+ public static final String EXTERNAL_JSON_2 = "external2.json";
+
+ public static final String PACKAGE = "/org/apache/slider/core/conf/examples/";
+
+
+ private static final String[] ALL_EXAMPLES = {APP_JSON, OVERRIDE_JSON,
+ DEFAULT_JSON};
+
+ public static final List<String> ALL_EXAMPLE_RESOURCES = new ArrayList<>();
+ static {
+ for (String example : ALL_EXAMPLES) {
+ ALL_EXAMPLE_RESOURCES.add(PACKAGE + example);
+ }
+ }
+
+ private ExampleAppJson() {
+ }
+
+ static Application loadResource(String name) throws IOException {
+ return JSON_SER_DESER.fromResource(PACKAGE + name);
+ }
+
+ public static String resourceName(String name) {
+ return "target/test-classes" + PACKAGE + name;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java
deleted file mode 100644
index f13fbcc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.conf;
-
-import org.apache.slider.api.resource.Application;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER;
-
-/**
- * Names of the example configs.
- */
-public final class ExampleConfResources {
-
- public static final String APP_JSON = "app.json";
- public static final String APP_RES = "app-resolved.json";
- public static final String OVERRIDE_JSON = "app-override.json";
- public static final String OVERRIDE_RES = "app-override-resolved.json";
-
- public static final String PACKAGE = "/org/apache/slider/core/conf/examples/";
-
-
- private static final String[] ALL_EXAMPLES = {APP_JSON, APP_RES,
- OVERRIDE_JSON, OVERRIDE_RES};
-
- public static final List<String> ALL_EXAMPLE_RESOURCES = new ArrayList<>();
- static {
- for (String example : ALL_EXAMPLES) {
- ALL_EXAMPLE_RESOURCES.add(PACKAGE + example);
- }
- }
-
- private ExampleConfResources() {
- }
-
- static Application loadResource(String name) throws IOException {
- return JSON_SER_DESER.fromResource(PACKAGE + name);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java
deleted file mode 100644
index 48b0736..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.conf;
-
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER;
-
-/**
- * Test loading example resources.
- */
-@RunWith(value = Parameterized.class)
-public class TestConfTreeLoadExamples extends Assert {
- private String resource;
-
- public TestConfTreeLoadExamples(String resource) {
- this.resource = resource;
- }
-
- @Parameterized.Parameters
- public static Collection<String[]> filenames() {
- String[][] stringArray = new String[ExampleConfResources
- .ALL_EXAMPLE_RESOURCES.size()][1];
- int i = 0;
- for (String s : ExampleConfResources.ALL_EXAMPLE_RESOURCES) {
- stringArray[i++][0] = s;
- }
- return Arrays.asList(stringArray);
- }
-
- @Test
- public void testLoadResource() throws Throwable {
- try {
- Application application = JSON_SER_DESER.fromResource(resource);
- SliderUtils.resolve(application);
- } catch (Exception e) {
- throw new Exception("exception loading " + resource + ":" + e.toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java
index 285ddfa..5f5df70 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java
@@ -18,20 +18,40 @@
package org.apache.slider.core.conf;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.ConfigFile;
+import org.apache.slider.api.resource.ConfigFile.TypeEnum;
import org.apache.slider.api.resource.Configuration;
+import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.persist.JsonSerDeser;
+import org.apache.slider.util.ServiceApiUtil;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
import static org.apache.slider.api.InternalKeys.CHAOS_MONKEY_INTERVAL;
import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS;
import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS;
import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES;
-import static org.apache.slider.core.conf.ExampleConfResources.APP_JSON;
-import static org.apache.slider.core.conf.ExampleConfResources.OVERRIDE_JSON;
+import static org.apache.slider.core.conf.ExampleAppJson.APP_JSON;
+import static org.apache.slider.core.conf.ExampleAppJson.EXTERNAL_JSON_1;
+import static org.apache.slider.core.conf.ExampleAppJson.OVERRIDE_JSON;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
/**
* Test global configuration resolution.
@@ -42,23 +62,26 @@ public class TestConfigurationResolve extends Assert {
@Test
public void testOverride() throws Throwable {
-
- Application orig = ExampleConfResources.loadResource(OVERRIDE_JSON);
+ Application orig = ExampleAppJson.loadResource(OVERRIDE_JSON);
Configuration global = orig.getConfiguration();
assertEquals("a", global.getProperty("g1"));
assertEquals("b", global.getProperty("g2"));
+ assertEquals(2, global.getFiles().size());
Configuration simple = orig.getComponent("simple").getConfiguration();
assertEquals(0, simple.getProperties().size());
+ assertEquals(1, simple.getFiles().size());
Configuration master = orig.getComponent("master").getConfiguration();
assertEquals("m", master.getProperty("name"));
assertEquals("overridden", master.getProperty("g1"));
+ assertEquals(0, master.getFiles().size());
Configuration worker = orig.getComponent("worker").getConfiguration();
LOG.info("worker = {}", worker);
assertEquals(3, worker.getProperties().size());
+ assertEquals(0, worker.getFiles().size());
assertEquals("worker", worker.getProperty("name"));
assertEquals("overridden-by-worker", worker.getProperty("g1"));
@@ -66,18 +89,36 @@ public class TestConfigurationResolve extends Assert {
assertEquals("1000", worker.getProperty("timeout"));
// here is the resolution
- SliderUtils.resolve(orig);
+ SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ replay(sfs, mockFs);
+ ServiceApiUtil.validateAndResolveApplication(orig, sfs);
global = orig.getConfiguration();
LOG.info("global = {}", global);
assertEquals("a", global.getProperty("g1"));
assertEquals("b", global.getProperty("g2"));
+ assertEquals(2, global.getFiles().size());
simple = orig.getComponent("simple").getConfiguration();
assertEquals(2, simple.getProperties().size());
assertEquals("a", simple.getProperty("g1"));
assertEquals("b", simple.getProperty("g2"));
-
+ assertEquals(2, simple.getFiles().size());
+
+ Set<ConfigFile> files = new HashSet<>();
+ Map<String, String> props = new HashMap<>();
+ props.put("k1", "overridden");
+ props.put("k2", "v2");
+ files.add(new ConfigFile().destFile("file1").type(TypeEnum
+ .PROPERTIES).props(props));
+ files.add(new ConfigFile().destFile("file2").type(TypeEnum
+ .XML).props(Collections.singletonMap("k3", "v3")));
+ assertTrue(files.contains(simple.getFiles().get(0)));
+ assertTrue(files.contains(simple.getFiles().get(1)));
master = orig.getComponent("master").getConfiguration();
LOG.info("master = {}", master);
@@ -85,6 +126,17 @@ public class TestConfigurationResolve extends Assert {
assertEquals("m", master.getProperty("name"));
assertEquals("overridden", master.getProperty("g1"));
assertEquals("b", master.getProperty("g2"));
+ assertEquals(2, master.getFiles().size());
+
+ props.put("k1", "v1");
+ files.clear();
+ files.add(new ConfigFile().destFile("file1").type(TypeEnum
+ .PROPERTIES).props(props));
+ files.add(new ConfigFile().destFile("file2").type(TypeEnum
+ .XML).props(Collections.singletonMap("k3", "v3")));
+
+ assertTrue(files.contains(master.getFiles().get(0)));
+ assertTrue(files.contains(master.getFiles().get(1)));
worker = orig.getComponent("worker").getConfiguration();
LOG.info("worker = {}", worker);
@@ -94,13 +146,91 @@ public class TestConfigurationResolve extends Assert {
assertEquals("overridden-by-worker", worker.getProperty("g1"));
assertEquals("b", worker.getProperty("g2"));
assertEquals("1000", worker.getProperty("timeout"));
+ assertEquals(2, worker.getFiles().size());
+ assertTrue(files.contains(worker.getFiles().get(0)));
+ assertTrue(files.contains(worker.getFiles().get(1)));
}
@Test
- public void testTimeIntervalLoading() throws Throwable {
+ public void testOverrideExternalConfiguration() throws IOException {
+ Application orig = ExampleAppJson.loadResource(EXTERNAL_JSON_1);
+
+ Configuration global = orig.getConfiguration();
+ assertEquals(0, global.getProperties().size());
+
+ assertEquals(3, orig.getComponents().size());
- Application orig = ExampleConfResources.loadResource(APP_JSON);
+ Configuration simple = orig.getComponent("simple").getConfiguration();
+ assertEquals(0, simple.getProperties().size());
+
+ Configuration master = orig.getComponent("master").getConfiguration();
+ assertEquals(1, master.getProperties().size());
+ assertEquals("is-overridden", master.getProperty("g3"));
+
+ Configuration other = orig.getComponent("other").getConfiguration();
+ assertEquals(0, other.getProperties().size());
+
+ // load the external application
+ SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ replay(sfs, mockFs);
+ Application ext = ExampleAppJson.loadResource(APP_JSON);
+ ServiceApiUtil.validateAndResolveApplication(ext, sfs);
+ reset(sfs, mockFs);
+
+ // perform the resolution on original application
+ JsonSerDeser<Application> jsonSerDeser = createNiceMock(JsonSerDeser
+ .class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
+ .anyTimes();
+ replay(sfs, mockFs, jsonSerDeser);
+ ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
+ ServiceApiUtil.validateAndResolveApplication(orig, sfs);
+
+ global = orig.getConfiguration();
+ assertEquals(0, global.getProperties().size());
+
+ assertEquals(4, orig.getComponents().size());
+
+ simple = orig.getComponent("simple").getConfiguration();
+ assertEquals(3, simple.getProperties().size());
+ assertEquals("a", simple.getProperty("g1"));
+ assertEquals("b", simple.getProperty("g2"));
+ assertEquals("60",
+ simple.getProperty("internal.chaos.monkey.interval.seconds"));
+
+ master = orig.getComponent("master").getConfiguration();
+ assertEquals(5, master.getProperties().size());
+ assertEquals("512M", master.getProperty("jvm.heapsize"));
+ assertEquals("overridden", master.getProperty("g1"));
+ assertEquals("b", master.getProperty("g2"));
+ assertEquals("is-overridden", master.getProperty("g3"));
+ assertEquals("60",
+ simple.getProperty("internal.chaos.monkey.interval.seconds"));
+
+ Configuration worker = orig.getComponent("worker").getConfiguration();
+ LOG.info("worker = {}", worker);
+ assertEquals(4, worker.getProperties().size());
+ assertEquals("512M", worker.getProperty("jvm.heapsize"));
+ assertEquals("overridden-by-worker", worker.getProperty("g1"));
+ assertEquals("b", worker.getProperty("g2"));
+ assertEquals("60",
+ worker.getProperty("internal.chaos.monkey.interval.seconds"));
+
+ other = orig.getComponent("other").getConfiguration();
+ assertEquals(0, other.getProperties().size());
+ }
+
+ @Test
+ public void testTimeIntervalLoading() throws Throwable {
+ Application orig = ExampleAppJson.loadResource(APP_JSON);
Configuration conf = orig.getConfiguration();
long s = conf.getPropertyLong(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestExampleAppJson.java
new file mode 100644
index 0000000..09096d0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestExampleAppJson.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.conf;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.util.ServiceApiUtil;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+
+/**
+ * Test loading example resources.
+ */
+@RunWith(value = Parameterized.class)
+public class TestExampleAppJson extends Assert {
+ private String resource;
+
+ public TestExampleAppJson(String resource) {
+ this.resource = resource;
+ }
+
+ @Parameterized.Parameters
+ public static Collection<String[]> filenames() {
+ String[][] stringArray = new String[ExampleAppJson
+ .ALL_EXAMPLE_RESOURCES.size()][1];
+ int i = 0;
+ for (String s : ExampleAppJson.ALL_EXAMPLE_RESOURCES) {
+ stringArray[i++][0] = s;
+ }
+ return Arrays.asList(stringArray);
+ }
+
+ @Test
+ public void testLoadResource() throws Throwable {
+ try {
+ Application application = JSON_SER_DESER.fromResource(resource);
+
+ SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ replay(sfs, mockFs);
+
+ ServiceApiUtil.validateAndResolveApplication(application, sfs);
+ } catch (Exception e) {
+ throw new Exception("exception loading " + resource + ":" + e.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java
new file mode 100644
index 0000000..162d34c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.ConfigFile;
+import org.apache.slider.api.resource.ConfigFile.TypeEnum;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+
+/**
+ * Test the AbstractClientProvider shared methods.
+ */
+public class TestAbstractClientProvider {
+ private static final String EXCEPTION_PREFIX = "Should have thrown " +
+ "exception: ";
+ private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
+ "exception: ";
+
+ private static class ClientProvider extends AbstractClientProvider {
+ @Override
+ public void validateArtifact(Artifact artifact, FileSystem fileSystem)
+ throws IOException {
+ }
+
+ @Override
+ protected void validateConfigFile(ConfigFile configFile,
+ FileSystem fileSystem) throws IOException {
+ }
+ }
+
+ @Test
+ public void testConfigFiles() throws IOException {
+ ClientProvider clientProvider = new ClientProvider();
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes();
+ replay(mockFs);
+
+ ConfigFile configFile = new ConfigFile();
+ List<ConfigFile> configFiles = new ArrayList<>();
+ configFiles.add(configFile);
+
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "null file type");
+ } catch (IllegalArgumentException e) {
+ }
+
+ configFile.setType(TypeEnum.TEMPLATE);
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "empty src_file for type template");
+ } catch (IllegalArgumentException e) {
+ }
+
+ configFile.setSrcFile("srcfile");
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "empty dest file");
+ } catch (IllegalArgumentException e) {
+ }
+
+ configFile.setDestFile("destfile");
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ configFile = new ConfigFile();
+ configFile.setType(TypeEnum.JSON);
+ configFile.setSrcFile(null);
+ configFile.setDestFile("path/destfile2");
+ configFiles.add(configFile);
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "dest file with multiple path elements");
+ } catch (IllegalArgumentException e) {
+ }
+
+ configFile.setDestFile("/path/destfile2");
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ configFile.setDestFile("destfile");
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "duplicate dest file");
+ } catch (IllegalArgumentException e) {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java
new file mode 100644
index 0000000..6df660d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers;
+
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.client.SliderClient;
+import org.apache.slider.common.params.SliderActions;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.conf.ExampleAppJson;
+import org.apache.slider.core.main.ServiceLauncher;
+import org.apache.slider.util.ServiceApiUtil;
+import org.apache.slider.utils.YarnZKMiniClusterTestBase;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.slider.common.params.Arguments.ARG_APPDEF;
+
+/**
+ * Test for building / resolving components of type APPLICATION.
+ */
+public class TestBuildApplicationComponent extends YarnZKMiniClusterTestBase {
+
+ private static void checkComponentNames(List<Component> components,
+ Set<String> names) {
+ assertEquals(names.size(), components.size());
+ for (Component comp : components) {
+ assertTrue(names.contains(comp.getName()));
+ }
+ }
+
+ public void buildAndCheckComponents(String appName, String appDef,
+ SliderFileSystem sfs, Set<String> names) throws Throwable {
+ ServiceLauncher<SliderClient> launcher = createOrBuildCluster(
+ SliderActions.ACTION_BUILD, appName, Arrays.asList(ARG_APPDEF,
+ ExampleAppJson.resourceName(appDef)), true, false);
+ SliderClient sliderClient = launcher.getService();
+ addToTeardown(sliderClient);
+
+ // verify the cluster exists
+ assertEquals(0, sliderClient.actionExists(appName, false));
+ // verify generated conf
+ List<Component> components = ServiceApiUtil.getApplicationComponents(sfs,
+ appName);
+ checkComponentNames(components, names);
+ }
+
+ @Test
+ public void testExternalComponentBuild() throws Throwable {
+ String clustername = createMiniCluster("", getConfiguration(), 1, true);
+
+ describe("verify external components");
+
+ SliderFileSystem sfs = createSliderFileSystem();
+
+ Set<String> nameSet = new HashSet<>();
+ nameSet.add("simple");
+ nameSet.add("master");
+ nameSet.add("worker");
+
+ buildAndCheckComponents("app-1", ExampleAppJson.APP_JSON, sfs,
+ nameSet);
+ buildAndCheckComponents("external-0", ExampleAppJson
+ .EXTERNAL_JSON_0, sfs, nameSet);
+
+ nameSet.add("other");
+
+ buildAndCheckComponents("external-1", ExampleAppJson
+ .EXTERNAL_JSON_1, sfs, nameSet);
+
+ nameSet.add("another");
+
+ buildAndCheckComponents("external-2", ExampleAppJson
+ .EXTERNAL_JSON_2, sfs, nameSet);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java
new file mode 100644
index 0000000..f1afe67
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers;
+
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.client.SliderClient;
+import org.apache.slider.common.params.SliderActions;
+import org.apache.slider.core.conf.ExampleAppJson;
+import org.apache.slider.core.main.ServiceLauncher;
+import org.apache.slider.utils.YarnZKMiniClusterTestBase;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+import static org.apache.slider.common.params.Arguments.ARG_APPDEF;
+
+/**
+ * Simple end-to-end test.
+ */
+public class TestDefaultProvider extends YarnZKMiniClusterTestBase {
+
+ // TODO figure out how to run client commands against minicluster
+ // (currently errors out unable to find containing jar of AM for upload)
+ @Ignore
+ @Test
+ public void testDefaultProvider() throws Throwable {
+ createMiniCluster("", getConfiguration(), 1, true);
+ String appName = "default-1";
+
+ describe("verify default provider");
+
+ String appDef = ExampleAppJson.resourceName(ExampleAppJson
+ .DEFAULT_JSON);
+
+ ServiceLauncher<SliderClient> launcher = createOrBuildCluster(
+ SliderActions.ACTION_CREATE, appName, Arrays.asList(ARG_APPDEF,
+ appDef), true, true);
+ SliderClient sliderClient = launcher.getService();
+ addToTeardown(sliderClient);
+
+ Application application = sliderClient.actionStatus(appName);
+ assertEquals(1L, application.getContainers().size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java
index c1f2886..6f4ca42 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java
@@ -43,7 +43,7 @@ public class BaseMockAppStateAATest extends BaseMockAppStateTest
@Override
public Application buildApplication() {
Application application = factory.newApplication(0, 0, 0)
- .name(getTestName());
+ .name(getValidTestName());
application.getComponent(ROLE1).getConfiguration().setProperty(
COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy
.ANTI_AFFINITY_REQUIRED));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
index eb25b40..571e9d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
@@ -362,7 +362,7 @@ public class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
// now destroy the app state
AppStateBindingInfo bindingInfo = buildBindingInfo();
bindingInfo.application = factory.newApplication(0, 0, desiredAA).name(
- getTestName());
+ getValidTestName());
bindingInfo.application.getComponent(ROLE2)
.getConfiguration().setProperty(COMPONENT_PLACEMENT_POLICY,
Integer.toString(PlacementPolicy.ANTI_AFFINITY_REQUIRED));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java
index ea0dcf4..9cbda4f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java
@@ -203,7 +203,7 @@ public class TestMockAppStateContainerFailure extends BaseMockAppStateTest
// Update instance definition to allow containers to fail any number of
// times
AppStateBindingInfo bindingInfo = buildBindingInfo();
- bindingInfo.application.getConfiguration().setProperty(
+ bindingInfo.application.getComponent(ROLE0).getConfiguration().setProperty(
ResourceKeys.CONTAINER_FAILURE_THRESHOLD, "0");
appState = new MockAppState(bindingInfo);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java
index 6d8e963..7f7f93a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java
@@ -38,6 +38,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.io.IOException;
import java.util.Collections;
/**
@@ -65,7 +66,7 @@ public class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
}
@Override
- public AppStateBindingInfo buildBindingInfo() {
+ public AppStateBindingInfo buildBindingInfo() throws IOException {
AppStateBindingInfo bindingInfo = super.buildBindingInfo();
bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector();
return bindingInfo;
@@ -145,7 +146,7 @@ public class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
appState = new MockAppState();
AppStateBindingInfo binding2 = buildBindingInfo();
binding2.application = factory.newApplication(0, 0, 0)
- .name(getTestName());
+ .name(getValidTestName());
binding2.historyPath = historyPath2;
appState.buildInstance(binding2);
// on this read there won't be the right number of roles
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java
index b0634bf..d9c675d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java
@@ -70,7 +70,7 @@ public class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest
AppStateBindingInfo bindingInfo = buildBindingInfo();
bindingInfo.application = factory.newApplication(r0, r1, r2)
- .name(getTestName());
+ .name(getValidTestName());
bindingInfo.liveContainers = containers;
appState = new MockAppState(bindingInfo);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
index b7e967f..703d65f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
@@ -30,6 +30,7 @@ import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.junit.Test;
+import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -59,7 +60,7 @@ public class TestMockAppStateUniqueNames extends BaseMockAppStateTest
}
@Override
- public AppStateBindingInfo buildBindingInfo() {
+ public AppStateBindingInfo buildBindingInfo() throws IOException {
AppStateBindingInfo bindingInfo = super.buildBindingInfo();
bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector();
return bindingInfo;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
index d382c8a..4aa5895 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
@@ -40,7 +40,7 @@ public class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
@Override
public Application buildApplication() {
- return factory.newApplication(1, 0, 0).name(getTestName());
+ return factory.newApplication(1, 0, 0).name(getValidTestName());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
index 69abccf..5af87f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
@@ -51,6 +51,7 @@ import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.util.ServiceApiUtil;
import org.apache.slider.utils.SliderTestBase;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,6 +63,7 @@ import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Locale;
import java.util.Map.Entry;
/**
@@ -118,7 +120,7 @@ public abstract class BaseMockAppStateTest extends SliderTestBase implements
historyPath = new Path(historyWorkDir.toURI());
fs.delete(historyPath, true);
appState = new MockAppState(buildBindingInfo());
- stateAccess = new ProviderAppState(getTestName(), appState);
+ stateAccess = new ProviderAppState(getValidTestName(), appState);
}
/**
@@ -127,9 +129,11 @@ public abstract class BaseMockAppStateTest extends SliderTestBase implements
* from {@link #buildApplication()} ()}
* @return
*/
- protected AppStateBindingInfo buildBindingInfo() {
+ protected AppStateBindingInfo buildBindingInfo() throws IOException {
AppStateBindingInfo binding = new AppStateBindingInfo();
binding.application = buildApplication();
+ ServiceApiUtil.validateAndResolveApplication(binding.application,
+ sliderFileSystem);
//binding.roles = new ArrayList<>(factory.ROLES);
binding.fs = fs;
binding.historyPath = historyPath;
@@ -142,7 +146,7 @@ public abstract class BaseMockAppStateTest extends SliderTestBase implements
* @return the instance definition
*/
public Application buildApplication() {
- return factory.newApplication(0, 0, 0).name(getTestName());
+ return factory.newApplication(0, 0, 0).name(getValidTestName());
}
/**
@@ -153,6 +157,10 @@ public abstract class BaseMockAppStateTest extends SliderTestBase implements
return methodName.getMethodName();
}
+ public String getValidTestName() {
+ return getTestName().toLowerCase(Locale.ENGLISH);
+ }
+
public RoleStatus getRole0Status() {
return lookupRole(ROLE0);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java
index 2ac5087..8785b92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Component;
+import org.apache.slider.api.resource.Resource;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
@@ -190,6 +191,8 @@ public class MockFactory implements MockRoles {
*/
public Application newApplication(long r1, long r2, long r3) {
Application application = new Application();
+ application.setLaunchCommand("sleep 60");
+ application.setResource(new Resource().memory("256"));
application.getConfiguration().setProperty(ResourceKeys
.NODE_FAILURE_THRESHOLD, Integer.toString(NODE_FAILURE_THRESHOLD));
List<Component> components = application.getComponents();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java
new file mode 100644
index 0000000..9ca3242
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java
@@ -0,0 +1,393 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.utils;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.api.resource.Resource;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.persist.JsonSerDeser;
+import org.apache.slider.util.RestApiConstants;
+import org.apache.slider.util.RestApiErrorMessages;
+import org.apache.slider.util.ServiceApiUtil;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.apache.slider.util.RestApiConstants.DEFAULT_COMPONENT_NAME;
+import static org.apache.slider.util.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
+import static org.apache.slider.util.RestApiErrorMessages.*;
+import static org.apache.slider.util.RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID;
+import static org.apache.slider.util.RestApiErrorMessages.ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Test for ServiceApiUtil helper methods.
+ */
+public class TestServiceApiUtil {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestServiceApiUtil.class);
+ private static final String EXCEPTION_PREFIX = "Should have thrown " +
+ "exception: ";
+ private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
+ "exception: ";
+
+ @Test(timeout = 90000)
+ public void testResourceValidation() throws Exception {
+ SliderFileSystem sfs = initMock(null);
+
+ Application app = new Application();
+
+ // no name
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with no name");
+ } catch (IllegalArgumentException e) {
+ assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
+ }
+
+ // bad format name
+ String[] badNames = {"4finance", "Finance", "finance@home"};
+ for (String badName : badNames) {
+ app.setName(badName);
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName);
+ } catch (IllegalArgumentException e) {
+ assertEquals(String.format(
+ ERROR_APPLICATION_NAME_INVALID_FORMAT, badName), e.getMessage());
+ }
+ }
+
+ // launch command not specified
+ app.setName("finance_home");
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with no launch command");
+ } catch (IllegalArgumentException e) {
+ assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
+ e.getMessage());
+ }
+
+ // resource not specified
+ app.setLaunchCommand("sleep 3600");
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with no resource");
+ } catch (IllegalArgumentException e) {
+ assertEquals(String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID,
+ RestApiConstants.DEFAULT_COMPONENT_NAME), e.getMessage());
+ }
+
+ // memory not specified
+ Resource res = new Resource();
+ app.setResource(res);
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with no memory");
+ } catch (IllegalArgumentException e) {
+ assertEquals(String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID,
+ RestApiConstants.DEFAULT_COMPONENT_NAME), e.getMessage());
+ }
+
+ // invalid no of cpus
+ res.setMemory("100mb");
+ res.setCpus(-2);
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ Assert.fail(
+ EXCEPTION_PREFIX + "application with invalid no of cpus");
+ } catch (IllegalArgumentException e) {
+ assertEquals(String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE,
+ RestApiConstants.DEFAULT_COMPONENT_NAME), e.getMessage());
+ }
+
+ // number of containers not specified
+ res.setCpus(2);
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with no container count");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage()
+ .contains(ERROR_CONTAINERS_COUNT_INVALID));
+ }
+
+ // specifying profile along with cpus/memory raises exception
+ res.setProfile("hbase_finance_large");
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX
+ + "application with resource profile along with cpus/memory");
+ } catch (IllegalArgumentException e) {
+ assertEquals(String.format(RestApiErrorMessages
+ .ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
+ RestApiConstants.DEFAULT_COMPONENT_NAME),
+ e.getMessage());
+ }
+
+ // currently resource profile alone is not supported.
+ // TODO: remove the next test once resource profile alone is supported.
+ res.setCpus(null);
+ res.setMemory(null);
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with resource profile only");
+ } catch (IllegalArgumentException e) {
+ assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET,
+ e.getMessage());
+ }
+
+ // unset profile here and add cpus/memory back
+ res.setProfile(null);
+ res.setCpus(2);
+ res.setMemory("2gb");
+
+ // null number of containers
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "null number of containers");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
+ }
+
+ // negative number of containers
+ app.setNumberOfContainers(-1L);
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "negative number of containers");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
+ }
+
+ // everything valid here
+ app.setNumberOfContainers(5L);
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ } catch (IllegalArgumentException e) {
+ LOG.error("application attributes specified should be valid here", e);
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testArtifacts() throws IOException {
+ SliderFileSystem sfs = initMock(null);
+
+ Application app = new Application();
+ app.setName("name");
+ Resource res = new Resource();
+ app.setResource(res);
+ res.setMemory("512M");
+ app.setNumberOfContainers(3L);
+
+ // no artifact id fails with default type
+ Artifact artifact = new Artifact();
+ app.setArtifact(artifact);
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
+ } catch (IllegalArgumentException e) {
+ assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
+ }
+
+ // no artifact id fails with APPLICATION type
+ artifact.setType(Artifact.TypeEnum.APPLICATION);
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
+ } catch (IllegalArgumentException e) {
+ assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
+ }
+
+ // no artifact id fails with TARBALL type
+ artifact.setType(Artifact.TypeEnum.TARBALL);
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
+ } catch (IllegalArgumentException e) {
+ assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
+ }
+
+ // everything valid here
+ artifact.setType(Artifact.TypeEnum.DOCKER);
+ artifact.setId("docker.io/centos:centos7");
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ } catch (IllegalArgumentException e) {
+ LOG.error("application attributes specified should be valid here", e);
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ // defaults assigned
+ assertEquals(app.getComponents().get(0).getName(),
+ DEFAULT_COMPONENT_NAME);
+ assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME);
+ }
+
+ private static Resource createValidResource() {
+ Resource res = new Resource();
+ res.setMemory("512M");
+ return res;
+ }
+
+ private static Component createValidComponent(String compName) {
+ Component comp = new Component();
+ comp.setName(compName);
+ comp.setResource(createValidResource());
+ comp.setNumberOfContainers(1L);
+ return comp;
+ }
+
+ private static Application createValidApplication(String compName) {
+ Application app = new Application();
+ app.setLaunchCommand("sleep 3600");
+ app.setName("name");
+ app.setResource(createValidResource());
+ app.setNumberOfContainers(1L);
+ if (compName != null) {
+ app.addComponent(createValidComponent(compName));
+ }
+ return app;
+ }
+
+ private static SliderFileSystem initMock(Application ext) throws IOException {
+ SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ JsonSerDeser<Application> jsonSerDeser = createNiceMock(JsonSerDeser
+ .class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ if (ext != null) {
+ expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
+ .anyTimes();
+ }
+ replay(sfs, mockFs, jsonSerDeser);
+ ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
+ return sfs;
+ }
+
+ @Test
+ public void testExternalApplication() throws IOException {
+ Application ext = createValidApplication("comp1");
+ SliderFileSystem sfs = initMock(ext);
+
+ Application app = createValidApplication(null);
+
+ Artifact artifact = new Artifact();
+ artifact.setType(Artifact.TypeEnum.APPLICATION);
+ artifact.setId("id");
+ app.setArtifact(artifact);
+
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ assertEquals(1, app.getComponents().size());
+ assertNotNull(app.getComponent("comp1"));
+ }
+
+ @Test
+ public void testDuplicateComponents() throws IOException {
+ SliderFileSystem sfs = initMock(null);
+
+ String compName = "comp1";
+ Application app = createValidApplication(compName);
+ app.addComponent(createValidComponent(compName));
+
+ // duplicate component name fails
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with component collision");
+ } catch (IllegalArgumentException e) {
+ assertEquals("Component name collision: " + compName, e.getMessage());
+ }
+ }
+
+ @Test
+ public void testExternalDuplicateComponent() throws IOException {
+ Application ext = createValidApplication("comp1");
+ SliderFileSystem sfs = initMock(ext);
+
+ Application app = createValidApplication("comp1");
+ Artifact artifact = new Artifact();
+ artifact.setType(Artifact.TypeEnum.APPLICATION);
+ artifact.setId("id");
+ app.getComponent("comp1").setArtifact(artifact);
+
+ // duplicate component name okay in the case of APPLICATION component
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testExternalComponent() throws IOException {
+ Application ext = createValidApplication("comp1");
+ SliderFileSystem sfs = initMock(ext);
+
+ Application app = createValidApplication("comp2");
+ Artifact artifact = new Artifact();
+ artifact.setType(Artifact.TypeEnum.APPLICATION);
+ artifact.setId("id");
+ app.setArtifact(artifact);
+
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ assertEquals(1, app.getComponents().size());
+ // artifact ID not inherited from global
+ assertNotNull(app.getComponent("comp2"));
+
+ // set APPLICATION artifact id on component
+ app.getComponent("comp2").setArtifact(artifact);
+
+ try {
+ ServiceApiUtil.validateAndResolveApplication(app, sfs);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ assertEquals(1, app.getComponents().size());
+ // original component replaced by external component
+ assertNotNull(app.getComponent("comp1"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java
index 746a0ec..5e62fc2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java
@@ -37,6 +37,7 @@ import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.params.ActionFreezeArgs;
import org.apache.slider.common.params.Arguments;
+import org.apache.slider.common.params.SliderActions;
import org.apache.slider.common.tools.Duration;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
@@ -328,11 +329,8 @@ public abstract class YarnMiniClusterTestBase extends SliderTestBase {
*/
public void stopRunningClusters() {
for (SliderClient client : clustersToTeardown) {
- try {
- maybeStopCluster(client, "", "Teardown at end of test case", true);
- } catch (Exception e) {
- LOG.warn("While stopping cluster " + e, e);
- }
+ maybeStopCluster(client, client.getDeployedClusterName(),
+ "Teardown at end of test case", true);
}
}
@@ -502,6 +500,62 @@ public abstract class YarnMiniClusterTestBase extends SliderTestBase {
}
/**
+ * Create or build a cluster (the action is set by the first verb).
+ * @param action operation to invoke: SliderActions.ACTION_CREATE or
+ * SliderActions.ACTION_BUILD
+ * @param clustername cluster name
+ * @param extraArgs list of extra args to add to the creation command
+ * @param deleteExistingData should the data of any existing cluster
+ * of this name be deleted
+ * @param blockUntilRunning block until the AM is running
+ * @return launcher which will have executed the command.
+ */
+ public ServiceLauncher<SliderClient> createOrBuildCluster(String action,
+ String clustername, List<String> extraArgs, boolean deleteExistingData,
+ boolean blockUntilRunning) throws Throwable {
+ assertNotNull(clustername);
+ assertNotNull(miniCluster);
+ // update action should keep existing data
+ Configuration config = miniCluster.getConfig();
+ if (deleteExistingData && !SliderActions.ACTION_UPDATE.equals(action)) {
+ FileSystem dfs = FileSystem.get(new URI(getFsDefaultName()), config);
+
+ SliderFileSystem sliderFileSystem = new SliderFileSystem(dfs, config);
+ Path clusterDir = sliderFileSystem.buildClusterDirPath(clustername);
+ LOG.info("deleting instance data at {}", clusterDir);
+ //this is a safety check to stop us doing something stupid like deleting /
+ assertTrue(clusterDir.toString().contains("/.slider/"));
+ rigorousDelete(sliderFileSystem, clusterDir, 60000);
+ }
+
+
+ List<String> argsList = new ArrayList<>();
+ argsList.addAll(Arrays.asList(
+ action, clustername,
+ Arguments.ARG_MANAGER, getRMAddr(),
+ Arguments.ARG_FILESYSTEM, getFsDefaultName(),
+ Arguments.ARG_DEBUG));
+
+ argsList.addAll(getExtraCLIArgs());
+
+ if (extraArgs != null) {
+ argsList.addAll(extraArgs);
+ }
+ ServiceLauncher<SliderClient> launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(config),
+ //varargs list of command line params
+ argsList
+ );
+ assertEquals(0, launcher.getServiceExitCode());
+ SliderClient client = launcher.getService();
+ if (blockUntilRunning) {
+ client.monitorAppToRunning(new Duration(CLUSTER_GO_LIVE_TIME));
+ }
+ return launcher;
+ }
+
+ /**
* Delete with some pauses and backoff; designed to handle slow delete
* operation in windows.
*/
@@ -652,28 +706,6 @@ public abstract class YarnMiniClusterTestBase extends SliderTestBase {
return getTestConfiguration().getTrimmed(getApplicationHomeKey());
}
- public List<String> getImageCommands() {
- if (switchToImageDeploy) {
- // its an image that had better be defined
- assertNotNull(getArchivePath());
- if (!imageIsRemote) {
- // its not remote, so assert it exists
- File f = new File(getArchivePath());
- assertTrue(f.exists());
- return Arrays.asList(Arguments.ARG_IMAGE, f.toURI().toString());
- } else {
- assertNotNull(remoteImageURI);
-
- // if it is remote, then its whatever the archivePath property refers to
- return Arrays.asList(Arguments.ARG_IMAGE, remoteImageURI.toString());
- }
- } else {
- assertNotNull(getApplicationHome());
- assertTrue(new File(getApplicationHome()).exists());
- return Arrays.asList(Arguments.ARG_APP_HOME, getApplicationHome());
- }
- }
-
/**
* Get the resource configuration dir in the source tree.
*
@@ -746,14 +778,23 @@ public abstract class YarnMiniClusterTestBase extends SliderTestBase {
SliderClient sliderClient,
String clustername,
String message,
- boolean force) throws IOException, YarnException {
+ boolean force) {
if (sliderClient != null) {
if (SliderUtils.isUnset(clustername)) {
clustername = sliderClient.getDeployedClusterName();
}
//only stop a cluster that exists
if (SliderUtils.isSet(clustername)) {
- return clusterActionFreeze(sliderClient, clustername, message, force);
+ try {
+ clusterActionFreeze(sliderClient, clustername, message, force);
+ } catch (Exception e) {
+ LOG.warn("While stopping cluster " + e, e);
+ }
+ try {
+ sliderClient.actionDestroy(clustername);
+ } catch (Exception e) {
+ LOG.warn("While destroying cluster " + e, e);
+ }
}
}
return 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java
index 322b346..cf9e616 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.zk.BlockingZKWatcher;
import org.apache.slider.core.zk.ZKIntegration;
import org.slf4j.Logger;
@@ -109,9 +108,7 @@ public abstract class YarnZKMiniClusterTestBase extends
int numLogDirs,
boolean startZK,
boolean startHDFS) throws IOException {
- if (SliderUtils.isUnset(name)) {
- name = methodName.getMethodName();
- }
+ name = buildClustername(name);
createMicroZKCluster("-" + name, conf);
conf.setBoolean(RegistryConstants.KEY_REGISTRY_ENABLED, true);
conf.set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, getZKBinding());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json
deleted file mode 100644
index e2a21ea..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json
+++ /dev/null
@@ -1,49 +0,0 @@
-{
- "name": "app-1",
- "lifetime": "3600",
- "configuration": {
- "properties": {
- "g1": "a",
- "g2": "b"
- }
- },
- "resource": {
- "cpus": 1,
- "memory": "512"
- },
- "number_of_containers": 2,
- "components": [
- {
- "name": "simple",
- "configuration": {
- "properties": {
- "g1": "a",
- "g2": "b"
- }
- }
- },
- {
- "name": "master",
- "configuration": {
- "properties": {
- "g1": "overridden",
- "g2": "b"
- }
- }
- },
- {
- "name": "worker",
- "resource": {
- "cpus": 1,
- "memory": "1024"
- },
- "configuration": {
- "properties": {
- "g1": "overridden-by-worker",
- "g2": "b",
- "timeout": "1000"
- }
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json
index 552cdef..d7e2fd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json
@@ -1,11 +1,29 @@
{
"name": "app-1",
"lifetime": "3600",
+ "launch_command": "sleep 3600",
"configuration": {
"properties": {
"g1": "a",
"g2": "b"
- }
+ },
+ "files": [
+ {
+ "type": "PROPERTIES",
+ "dest_file": "file1",
+ "props": {
+ "k1": "v1",
+ "k2": "v2"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "file2",
+ "props": {
+ "k3": "v3"
+ }
+ }
+ ]
},
"resource": {
"cpus": 1,
@@ -14,7 +32,18 @@
"number_of_containers": 2,
"components": [
{
- "name": "simple"
+ "name": "simple",
+ "configuration": {
+ "files": [
+ {
+ "type": "PROPERTIES",
+ "dest_file": "file1",
+ "props": {
+ "k1": "overridden"
+ }
+ }
+ ]
+ }
},
{
"name": "master",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json
deleted file mode 100644
index cd1ab6f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json
+++ /dev/null
@@ -1,81 +0,0 @@
-{
- "name": "zk-app-1",
- "lifetime": "3600",
- "configuration": {
- "properties": {
- "internal.chaos.monkey.interval.seconds": "60",
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0"
- }
- },
- "resource": {
- "cpus": 1,
- "memory": "512"
- },
- "number_of_containers": 2,
- "components": [
- {
- "name": "simple",
- "number_of_containers": 2,
- "configuration": {
- "properties": {
- "g1": "a",
- "g2": "b"
- }
- }
- },
- {
- "name": "master",
- "number_of_containers": 1,
- "resource": {
- "cpus": 1,
- "memory": "512"
- },
- "configuration": {
- "properties": {
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0",
- "jvm.heapsize": "512M"
- }
- }
- },
- {
- "name": "worker",
- "number_of_containers": 5,
- "resource": {
- "cpus": 1,
- "memory": "1024"
- },
- "configuration": {
- "properties": {
- "g1": "overridden-by-worker",
- "g2": "b",
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0",
- "jvm.heapsize": "512M"
- }
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json
index 90857db..b1d73c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json
@@ -1,20 +1,12 @@
{
"name": "app-1",
"lifetime": "3600",
+ "launch_command": "sleep 3600",
"configuration": {
"properties": {
"g1": "a",
"g2": "b",
- "internal.chaos.monkey.interval.seconds": "60",
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0"
+ "internal.chaos.monkey.interval.seconds": "60"
}
},
"resource": {
@@ -32,6 +24,7 @@
"configuration": {
"properties": {
"g1": "overridden",
+ "g3": "will-be-overridden",
"jvm.heapsize": "512M"
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org