You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/07/21 18:39:34 UTC
[32/52] [abbrv] hadoop git commit: YARN-6335. Port slider's groovy
unit tests to yarn native services. Contributed by Billie Rinaldi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestExecutionEnvironment.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestExecutionEnvironment.java
new file mode 100644
index 0000000..3e56a72
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestExecutionEnvironment.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import org.apache.slider.utils.SliderTestBase;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test execution environment.
+ */
+public class TestExecutionEnvironment extends SliderTestBase {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(TestExecutionEnvironment.class);
+
+ @Test
+ public void testClientEnv() throws Throwable {
+ SliderUtils.validateSliderClientEnvironment(LOG);
+ }
+
+ @Test
+ public void testWinutils() throws Throwable {
+ SliderUtils.maybeVerifyWinUtilsValid();
+ }
+
+ @Test
+ public void testServerEnv() throws Throwable {
+ SliderUtils.validateSliderServerEnvironment(LOG, true);
+ }
+
+ @Test
+ public void testServerEnvNoDependencies() throws Throwable {
+ SliderUtils.validateSliderServerEnvironment(LOG, false);
+ }
+
+ @Test
+ public void testopenSSLEnv() throws Throwable {
+ SliderUtils.validateOpenSSLEnv(LOG);
+ }
+
+ @Test
+ public void testValidatePythonEnv() throws Throwable {
+ SliderUtils.validatePythonEnv(LOG);
+ }
+
+ @Test
+ public void testNativeLibs() throws Throwable {
+ assertNativeLibrariesPresent();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java
new file mode 100644
index 0000000..bf6ee2c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.utils.SliderTestBase;
+import org.junit.Test;
+
+import java.net.URI;
+
+/**
+ * Test slider utils.
+ */
+public class TestMiscSliderUtils extends SliderTestBase {
+
+
+ public static final String CLUSTER1 = "cluster1";
+
+ @Test
+ public void testPurgeTempDir() throws Throwable {
+
+ Configuration configuration = new Configuration();
+ FileSystem fs = FileSystem.get(new URI("file:///"), configuration);
+ SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
+ Path inst = sliderFileSystem.createAppInstanceTempPath(CLUSTER1, "001");
+
+ assertTrue(fs.exists(inst));
+ sliderFileSystem.purgeAppInstanceTempFiles(CLUSTER1);
+ assertFalse(fs.exists(inst));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestPortScan.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestPortScan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestPortScan.java
new file mode 100644
index 0000000..0953a8b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestPortScan.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.junit.Test;
+
+import java.net.ServerSocket;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+/**
+ * Test finding a port in a range.
+ */
+public class TestPortScan {
+
+ @Test
+ public void testScanPorts() throws Throwable {
+
+ ServerSocket server = new ServerSocket(0);
+
+ try {
+ int serverPort = server.getLocalPort();
+ assertFalse(SliderUtils.isPortAvailable(serverPort));
+ int port = SliderUtils.findFreePort(serverPort, 10);
+ assertTrue(port > 0 && serverPort < port);
+ } finally {
+ server.close();
+ }
+ }
+
+ @Test
+ public void testRequestedPortsLogic() throws Throwable {
+ PortScanner portScanner = new PortScanner();
+ portScanner.setPortRange("5,6,8-10, 11,14 ,20 - 22");
+ List<Integer> ports = portScanner.getRemainingPortsToCheck();
+ List<Integer> expectedPorts =
+ Arrays.asList(5, 6, 8, 9, 10, 11, 14, 20, 21, 22);
+ assertEquals(expectedPorts, ports);
+ }
+
+ @Test
+ public void testRequestedPortsOutOfOrder() throws Throwable {
+ PortScanner portScanner = new PortScanner();
+ portScanner.setPortRange("8-10,5,6, 11,20 - 22, 14 ");
+ List<Integer> ports = portScanner.getRemainingPortsToCheck();
+ List<Integer> expectedPorts =
+ Arrays.asList(5, 6, 8, 9, 10, 11, 14, 20, 21, 22);
+ assertEquals(expectedPorts, ports);
+ }
+
+ @Test
+ public void testFindAvailablePortInRange() throws Throwable {
+ ServerSocket server = new ServerSocket(0);
+ try {
+ int serverPort = server.getLocalPort();
+
+ PortScanner portScanner = new PortScanner();
+ portScanner.setPortRange("" + (serverPort-1) + "-" + (serverPort + 3));
+ int port = portScanner.getAvailablePort();
+ assertNotEquals(port, serverPort);
+ assertTrue(port >= serverPort -1 && port <= serverPort + 3);
+ } finally {
+ server.close();
+ }
+ }
+
+ @Test
+ public void testFindAvailablePortInList() throws Throwable {
+ ServerSocket server = new ServerSocket(0);
+ try {
+ int serverPort = server.getLocalPort();
+
+ PortScanner portScanner = new PortScanner();
+ portScanner.setPortRange("" + (serverPort-1) + ", " + (serverPort + 1));
+ int port = portScanner.getAvailablePort();
+ assertNotEquals(port, serverPort);
+ assertTrue(port == serverPort -1 || port == serverPort + 1);
+ } finally {
+ server.close();
+ }
+ }
+
+ @Test
+ public void testNoAvailablePorts() throws Throwable {
+ ServerSocket server1 = new ServerSocket(0);
+ ServerSocket server2 = new ServerSocket(0);
+ try {
+ int serverPort1 = server1.getLocalPort();
+ int serverPort2 = server2.getLocalPort();
+
+ PortScanner portScanner = new PortScanner();
+ portScanner.setPortRange("" + serverPort1+ ", " + serverPort2);
+ try {
+ portScanner.getAvailablePort();
+ fail("expected SliderException");
+ } catch (SliderException e) {
+ // expected
+ }
+ } finally {
+ server1.close();
+ server2.close();
+ }
+ }
+
+ @Test
+ public void testPortRemovedFromRange() throws Throwable {
+ ServerSocket server = new ServerSocket(0);
+ try {
+ int serverPort = server.getLocalPort();
+
+ PortScanner portScanner = new PortScanner();
+ portScanner.setPortRange("" + (serverPort-1) + "-" + (serverPort + 3));
+ int port = portScanner.getAvailablePort();
+ assertNotEquals(port, serverPort);
+ assertTrue(port >= serverPort -1 && port <= serverPort + 3);
+ assertFalse(portScanner.getRemainingPortsToCheck().contains(port));
+ } finally {
+ server.close();
+ }
+ }
+
+ @Test(expected = BadConfigException.class)
+ public void testBadRange() throws BadConfigException {
+ PortScanner portScanner = new PortScanner();
+ // note the em dash
+ portScanner.setPortRange("2000–2010");
+ }
+
+ @Test(expected = BadConfigException.class)
+ public void testEndBeforeStart() throws BadConfigException {
+ PortScanner portScanner = new PortScanner();
+ portScanner.setPortRange("2001-2000");
+ }
+
+ @Test(expected = BadConfigException.class)
+ public void testEmptyRange() throws BadConfigException {
+ PortScanner portScanner = new PortScanner();
+ portScanner.setPortRange("");
+ }
+
+ @Test(expected = BadConfigException.class)
+ public void testBlankRange() throws BadConfigException {
+ PortScanner portScanner = new PortScanner();
+ portScanner.setPortRange(" ");
+ }
+
+ @Test
+ public void testExtraComma() throws BadConfigException {
+ PortScanner portScanner = new PortScanner();
+ portScanner.setPortRange("2000-2001, ");
+ List<Integer> ports = portScanner.getRemainingPortsToCheck();
+ List<Integer> expectedPorts = Arrays.asList(2000, 2001);
+ assertEquals(expectedPorts, ports);
+ }
+
+ @Test
+ public void testExtraCommas() throws BadConfigException {
+ PortScanner portScanner = new PortScanner();
+ portScanner.setPortRange("2000-2001,, ,2003,");
+ List<Integer> ports = portScanner.getRemainingPortsToCheck();
+ List<Integer> expectedPorts = Arrays.asList(2000, 2001, 2003);
+ assertEquals(expectedPorts, ports);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderFileSystem.java
new file mode 100644
index 0000000..dbb7791
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderFileSystem.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.common.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.utils.SliderTestBase;
+import org.junit.Test;
+
+/**
+ * Test slider file system.
+ */
+public class TestSliderFileSystem extends SliderTestBase {
+ private static Configuration defaultConfiguration() {
+ return new Configuration();
+ }
+
+ private static Configuration createConfigurationWithKV(String key, String
+ value) {
+ Configuration conf = defaultConfiguration();
+ conf.set(key, value);
+ return conf;
+ }
+
+ @Test
+ public void testSliderBasePathDefaultValue() throws Throwable {
+ Configuration configuration = defaultConfiguration();
+ FileSystem fileSystem = FileSystem.get(configuration);
+
+ SliderFileSystem fs2 = new SliderFileSystem(fileSystem, configuration);
+ assertEquals(fs2.getBaseApplicationPath(), new Path(fileSystem
+ .getHomeDirectory(), ".slider"));
+ }
+
+ @Test
+ public void testSliderBasePathCustomValue() throws Throwable {
+ Configuration configuration = createConfigurationWithKV(SliderXmlConfKeys
+ .KEY_SLIDER_BASE_PATH, "/slider/cluster");
+ FileSystem fileSystem = FileSystem.get(configuration);
+ SliderFileSystem fs2 = new SliderFileSystem(fileSystem, configuration);
+
+ assertEquals(fs2.getBaseApplicationPath(), new Path("/slider/cluster"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderTestUtils.java
new file mode 100644
index 0000000..a6e7db8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderTestUtils.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.utils.SliderTestUtils;
+import org.junit.Test;
+import org.junit.internal.AssumptionViolatedException;
+
+/**
+ * Test slider test utils.
+ */
+public class TestSliderTestUtils extends SliderTestUtils {
+
+ @Test
+ public void testAssumeTrue() throws Throwable {
+
+ try {
+ assume(true, "true");
+ } catch (AssumptionViolatedException e) {
+ throw new Exception(e);
+ }
+ }
+
+ @Test
+ public void testAssumeFalse() throws Throwable {
+
+ try {
+ assume(false, "false");
+ fail("expected an exception");
+ } catch (AssumptionViolatedException ignored) {
+ //expected
+ }
+ }
+
+ @Test
+ public void testAssumeBoolOptionSetInConf() throws Throwable {
+ Configuration conf = new Configuration(false);
+ conf.set("key", "true");
+ try {
+ assumeBoolOption(conf, "key", false);
+ } catch (AssumptionViolatedException e) {
+ throw new Exception(e);
+ }
+ }
+
+ @Test
+ public void testAssumeBoolOptionUnsetInConf() throws Throwable {
+ Configuration conf = new Configuration(false);
+ try {
+ assumeBoolOption(conf, "key", true);
+ } catch (AssumptionViolatedException e) {
+ throw new Exception(e);
+ }
+ }
+
+
+ @Test
+ public void testAssumeBoolOptionFalseInConf() throws Throwable {
+ Configuration conf = new Configuration(false);
+ conf.set("key", "false");
+ try {
+ assumeBoolOption(conf, "key", true);
+ fail("expected an exception");
+ } catch (AssumptionViolatedException ignored) {
+ //expected
+ }
+ }
+
+ @Test
+ public void testAssumeBoolOptionFalseUnsetInConf() throws Throwable {
+ Configuration conf = new Configuration(false);
+ try {
+ assumeBoolOption(conf, "key", false);
+ fail("expected an exception");
+ } catch (AssumptionViolatedException ignored) {
+ //expected
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
index 0df6047..a525e09 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
@@ -17,13 +17,9 @@
package org.apache.slider.common.tools;
import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl;
-import org.apache.slider.tools.TestUtility;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -33,7 +29,6 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
@@ -45,26 +40,6 @@ public class TestSliderUtils {
public TemporaryFolder folder = new TemporaryFolder();
@Test
- public void testGetMetaInfoStreamFromZip() throws Exception {
- String zipFileName = TestUtility.createAppPackage(
- folder,
- "testpkg",
- "test.zip",
- "target/test-classes/org/apache/slider/common/tools/test");
- Configuration configuration = new Configuration();
- FileSystem fs = FileSystem.getLocal(configuration);
- log.info("fs working dir is {}", fs.getWorkingDirectory().toString());
- SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
-
- InputStream stream = SliderUtils.getApplicationResourceInputStream(
- sliderFileSystem.getFileSystem(),
- new Path(zipFileName),
- "metainfo.xml");
- Assert.assertTrue(stream != null);
- Assert.assertTrue(stream.available() > 0);
- }
-
- @Test
public void testTruncate() {
Assert.assertEquals(SliderUtils.truncate(null, 5), null);
Assert.assertEquals(SliderUtils.truncate("323", -1), "323");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java
new file mode 100644
index 0000000..829b897
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ChecksumFileSystem;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.util.Shell;
+import org.apache.slider.utils.YarnMiniClusterTestBase;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Pattern;
+
+/**
+ * Test windows support.
+ */
+public class TestWindowsSupport extends YarnMiniClusterTestBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestWindowsSupport.class);
+
+ private static final Pattern HAS_DRIVE_LETTER_SPECIFIER =
+ Pattern.compile("^/?[a-zA-Z]:");
+ public static final String WINDOWS_FILE =
+ "C:\\Users\\Administrator\\AppData\\Local\\Temp" +
+ "\\junit3180177850133852404\\testpkg\\appdef_1.zip";
+
+
+ private static boolean hasWindowsDrive(String path) {
+ return HAS_DRIVE_LETTER_SPECIFIER.matcher(path).find();
+ }
+
+ private static int startPositionWithoutWindowsDrive(String path) {
+ if (hasWindowsDrive(path)) {
+ return path.charAt(0) == '/' ? 3 : 2;
+ } else {
+ return 0;
+ }
+ }
+
+ @Test
+ public void testHasWindowsDrive() throws Throwable {
+ assertTrue(hasWindowsDrive(WINDOWS_FILE));
+ }
+
+ @Test
+ public void testStartPosition() throws Throwable {
+ assertEquals(2, startPositionWithoutWindowsDrive(WINDOWS_FILE));
+ }
+
+ @Test
+ public void testPathHandling() throws Throwable {
+ assumeWindows();
+
+ Path path = new Path(WINDOWS_FILE);
+ URI uri = path.toUri();
+ // assert "file" == uri.scheme
+ assertNull(uri.getAuthority());
+
+ Configuration conf = new Configuration();
+
+ FileSystem localfs = FileSystem.get(uri, conf);
+ assertTrue(localfs instanceof ChecksumFileSystem);
+ try {
+ FileStatus stat = localfs.getFileStatus(path);
+ fail("expected an exception, got " + stat);
+ } catch (FileNotFoundException fnfe) {
+ // expected
+ }
+
+ try {
+ FSDataInputStream appStream = localfs.open(path);
+ } catch (FileNotFoundException fnfe) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testExecNonexistentBinary() throws Throwable {
+ assumeWindows();
+ List<String> commands = Arrays.asList("undefined-application", "--version");
+ try {
+ exec(0, commands);
+ fail("expected an exception");
+ } catch (ServiceStateException e) {
+ if (!(e.getCause() instanceof FileNotFoundException)) {
+ throw e;
+ }
+ }
+ }
+ @Test
+ public void testExecNonexistentBinary2() throws Throwable {
+ assumeWindows();
+ assertFalse(doesAppExist(Arrays.asList("undefined-application",
+ "--version")));
+ }
+
+ @Test
+ public void testEmitKillCommand() throws Throwable {
+
+ int result = killJavaProcesses("regionserver", 9);
+ // we know the exit code if there is no supported kill operation
+ assertTrue(getKillSupported() || result == -1);
+ }
+
+ @Test
+ public void testHadoopHomeDefined() throws Throwable {
+ assumeWindows();
+ String hadoopHome = Shell.getHadoopHome();
+ LOG.info("HADOOP_HOME={}", hadoopHome);
+ }
+
+ @Test
+ public void testHasWinutils() throws Throwable {
+ assumeWindows();
+ SliderUtils.maybeVerifyWinUtilsValid();
+ }
+
+ @Test
+ public void testExecWinutils() throws Throwable {
+ assumeWindows();
+ String winUtilsPath = Shell.getWinUtilsPath();
+ assertTrue(SliderUtils.isSet(winUtilsPath));
+ File winUtils = new File(winUtilsPath);
+ LOG.debug("Winutils is at {}", winUtils);
+
+ exec(0, Arrays.asList(winUtilsPath, "systeminfo"));
+ }
+
+ @Test
+ public void testPath() throws Throwable {
+ String path = extractPath();
+ LOG.info("Path value = {}", path);
+ }
+
+ @Test
+ public void testFindJavac() throws Throwable {
+ String name = Shell.WINDOWS ? "javac.exe" : "javac";
+ assertNotNull(locateExecutable(name));
+ }
+
+ @Test
+ public void testHadoopDLL() throws Throwable {
+ assumeWindows();
+ // split the path
+ File exepath = locateExecutable("HADOOP.DLL");
+ assertNotNull(exepath);
+ LOG.info("Hadoop DLL at: {}", exepath);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java
new file mode 100644
index 0000000..186123d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.registry.server.services.MicroZookeeperServiceKeys;
+import org.apache.slider.client.SliderClient;
+import org.apache.slider.core.zk.ZKIntegration;
+import org.apache.slider.utils.KeysForTests;
+import org.apache.slider.utils.YarnZKMiniClusterTestBase;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooDefs;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Test ZK integration.
+ */
+public class TestZKIntegration extends YarnZKMiniClusterTestBase implements
+ KeysForTests {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestZKIntegration.class);
+
+ public static final String USER = KeysForTests.USERNAME;
+ public static final int CONNECT_TIMEOUT = 5000;
+ private ZKIntegration zki;
+
+ @Before
+ public void createCluster() {
+ Configuration conf = getConfiguration();
+ String name = methodName.getMethodName();
+ File zkdir = new File("target/zk/${name}");
+ FileUtil.fullyDelete(zkdir);
+ conf.set(MicroZookeeperServiceKeys.KEY_ZKSERVICE_DIR, zkdir
+ .getAbsolutePath());
+ createMicroZKCluster("-"+ name, conf);
+ }
+
+ @After
+ public void closeZKI() throws IOException {
+ if (zki != null) {
+ zki.close();
+ zki = null;
+ }
+ }
+
+ public ZKIntegration initZKI() throws IOException, InterruptedException {
+ zki = createZKIntegrationInstance(
+ getZKBinding(), methodName.getMethodName(), true, false,
+ CONNECT_TIMEOUT);
+ return zki;
+ }
+
+ @Test
+ public void testListUserClustersWithoutAnyClusters() throws Throwable {
+ assertHasZKCluster();
+ initZKI();
+ String userPath = ZKIntegration.mkSliderUserPath(USER);
+ List<String> clusters = this.zki.getClusters();
+ assertTrue(SliderUtils.isEmpty(clusters));
+ }
+
+ @Test
+ public void testListUserClustersWithOneCluster() throws Throwable {
+ assertHasZKCluster();
+
+ initZKI();
+ String userPath = ZKIntegration.mkSliderUserPath(USER);
+ String fullPath = zki.createPath(userPath, "/cluster-",
+ ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ CreateMode.EPHEMERAL_SEQUENTIAL);
+ LOG.info("Ephemeral path {}", fullPath);
+ List<String> clusters = zki.getClusters();
+ assertEquals(1, clusters.size());
+ assertTrue(fullPath.endsWith(clusters.get(0)));
+ }
+
+ @Test
+ public void testListUserClustersWithTwoCluster() throws Throwable {
+ initZKI();
+ String userPath = ZKIntegration.mkSliderUserPath(USER);
+ String c1 = createEphemeralChild(zki, userPath);
+ LOG.info("Ephemeral path $c1");
+ String c2 = createEphemeralChild(zki, userPath);
+ LOG.info("Ephemeral path $c2");
+ List<String> clusters = zki.getClusters();
+ assertEquals(2, clusters.size());
+ assertTrue((c1.endsWith(clusters.get(0)) && c2.endsWith(clusters.get(1))) ||
+ (c1.endsWith(clusters.get(1)) && c2.endsWith(clusters.get(0))));
+ }
+
+ @Test
+ public void testCreateAndDeleteDefaultZKPath() throws Throwable {
+ MockSliderClient client = new MockSliderClient();
+
+ String path = client.createZookeeperNodeInner("cl1", true);
+ zki = client.getLastZKIntegration();
+
+ String zkPath = ZKIntegration.mkClusterPath(USER, "cl1");
+ assertEquals("zkPath must be as expected", zkPath,
+ "/services/slider/users/" + USER + "/cl1");
+ assertEquals(path, zkPath);
+ assertNull("ZKIntegration should be null.", zki);
+ zki = createZKIntegrationInstance(getZKBinding(), "cl1", true, false,
+ CONNECT_TIMEOUT);
+ assertFalse(zki.exists(zkPath));
+
+ path = client.createZookeeperNodeInner("cl1", false);
+ zki = client.getLastZKIntegration();
+ assertNotNull(zki);
+ assertEquals("zkPath must be as expected", zkPath,
+ "/services/slider/users/" + USER + "/cl1");
+ assertEquals(path, zkPath);
+ assertTrue(zki.exists(zkPath));
+ zki.createPath(zkPath, "/cn", ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode
+ .PERSISTENT);
+ assertTrue(zki.exists(zkPath + "/cn"));
+ client.deleteZookeeperNode("cl1");
+ assertFalse(zki.exists(zkPath));
+ }
+
+ public static String createEphemeralChild(ZKIntegration zki, String userPath)
+ throws KeeperException, InterruptedException {
+ return zki.createPath(userPath, "/cluster-",
+ ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ CreateMode.EPHEMERAL_SEQUENTIAL);
+ }
+
+ /**
+ * Test slider client that overriddes ZK client.
+ */
+ public class MockSliderClient extends SliderClient {
+ private ZKIntegration zki;
+
+ @Override
+ public String getUsername() {
+ return USER;
+ }
+
+ @Override
+ protected ZKIntegration getZkClient(String clusterName, String user) {
+ try {
+ zki = createZKIntegrationInstance(getZKBinding(), clusterName, true,
+ false, CONNECT_TIMEOUT);
+ } catch (Exception e) {
+ fail("creating ZKIntergration threw an exception");
+ }
+ return zki;
+ }
+
+ @Override
+ public Configuration getConfig() {
+ return new Configuration();
+ }
+
+ public ZKIntegration getLastZKIntegration() {
+ return zki;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java
new file mode 100644
index 0000000..f13fbcc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.conf;
+
+import org.apache.slider.api.resource.Application;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER;
+
+/**
+ * Names of the example configs.
+ */
+public final class ExampleConfResources {
+
+ public static final String APP_JSON = "app.json";
+ public static final String APP_RES = "app-resolved.json";
+ public static final String OVERRIDE_JSON = "app-override.json";
+ public static final String OVERRIDE_RES = "app-override-resolved.json";
+
+ public static final String PACKAGE = "/org/apache/slider/core/conf/examples/";
+
+
+ private static final String[] ALL_EXAMPLES = {APP_JSON, APP_RES,
+ OVERRIDE_JSON, OVERRIDE_RES};
+
+ public static final List<String> ALL_EXAMPLE_RESOURCES = new ArrayList<>();
+ static {
+ for (String example : ALL_EXAMPLES) {
+ ALL_EXAMPLE_RESOURCES.add(PACKAGE + example);
+ }
+ }
+
+ private ExampleConfResources() {
+ }
+
+ static Application loadResource(String name) throws IOException {
+ return JSON_SER_DESER.fromResource(PACKAGE + name);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java
new file mode 100644
index 0000000..48b0736
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.conf;
+
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.common.tools.SliderUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER;
+
+/**
+ * Test loading example resources.
+ */
+@RunWith(value = Parameterized.class)
+public class TestConfTreeLoadExamples extends Assert {
+ private String resource;
+
+ public TestConfTreeLoadExamples(String resource) {
+ this.resource = resource;
+ }
+
+ @Parameterized.Parameters
+ public static Collection<String[]> filenames() {
+ String[][] stringArray = new String[ExampleConfResources
+ .ALL_EXAMPLE_RESOURCES.size()][1];
+ int i = 0;
+ for (String s : ExampleConfResources.ALL_EXAMPLE_RESOURCES) {
+ stringArray[i++][0] = s;
+ }
+ return Arrays.asList(stringArray);
+ }
+
+ @Test
+ public void testLoadResource() throws Throwable {
+ try {
+ Application application = JSON_SER_DESER.fromResource(resource);
+ SliderUtils.resolve(application);
+ } catch (Exception e) {
+ throw new Exception("exception loading " + resource + ":" + e.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java
new file mode 100644
index 0000000..285ddfa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.conf;
+
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Configuration;
+import org.apache.slider.common.tools.SliderUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.slider.api.InternalKeys.CHAOS_MONKEY_INTERVAL;
+import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS;
+import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS;
+import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES;
+import static org.apache.slider.core.conf.ExampleConfResources.APP_JSON;
+import static org.apache.slider.core.conf.ExampleConfResources.OVERRIDE_JSON;
+
+/**
+ * Test global configuration resolution.
+ */
+public class TestConfigurationResolve extends Assert {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(TestConfigurationResolve.class);
+
+ @Test
+ public void testOverride() throws Throwable {
+
+ Application orig = ExampleConfResources.loadResource(OVERRIDE_JSON);
+
+ Configuration global = orig.getConfiguration();
+ assertEquals("a", global.getProperty("g1"));
+ assertEquals("b", global.getProperty("g2"));
+
+ Configuration simple = orig.getComponent("simple").getConfiguration();
+ assertEquals(0, simple.getProperties().size());
+
+ Configuration master = orig.getComponent("master").getConfiguration();
+ assertEquals("m", master.getProperty("name"));
+ assertEquals("overridden", master.getProperty("g1"));
+
+ Configuration worker = orig.getComponent("worker").getConfiguration();
+ LOG.info("worker = {}", worker);
+ assertEquals(3, worker.getProperties().size());
+
+ assertEquals("worker", worker.getProperty("name"));
+ assertEquals("overridden-by-worker", worker.getProperty("g1"));
+ assertNull(worker.getProperty("g2"));
+ assertEquals("1000", worker.getProperty("timeout"));
+
+ // here is the resolution
+ SliderUtils.resolve(orig);
+
+ global = orig.getConfiguration();
+ LOG.info("global = {}", global);
+ assertEquals("a", global.getProperty("g1"));
+ assertEquals("b", global.getProperty("g2"));
+
+ simple = orig.getComponent("simple").getConfiguration();
+ assertEquals(2, simple.getProperties().size());
+ assertEquals("a", simple.getProperty("g1"));
+ assertEquals("b", simple.getProperty("g2"));
+
+
+ master = orig.getComponent("master").getConfiguration();
+ LOG.info("master = {}", master);
+ assertEquals(3, master.getProperties().size());
+ assertEquals("m", master.getProperty("name"));
+ assertEquals("overridden", master.getProperty("g1"));
+ assertEquals("b", master.getProperty("g2"));
+
+ worker = orig.getComponent("worker").getConfiguration();
+ LOG.info("worker = {}", worker);
+ assertEquals(4, worker.getProperties().size());
+
+ assertEquals("worker", worker.getProperty("name"));
+ assertEquals("overridden-by-worker", worker.getProperty("g1"));
+ assertEquals("b", worker.getProperty("g2"));
+ assertEquals("1000", worker.getProperty("timeout"));
+
+ }
+
+ @Test
+ public void testTimeIntervalLoading() throws Throwable {
+
+ Application orig = ExampleConfResources.loadResource(APP_JSON);
+
+ Configuration conf = orig.getConfiguration();
+ long s = conf.getPropertyLong(
+ CHAOS_MONKEY_INTERVAL + SliderUtils.SECONDS,
+ 0);
+ assertEquals(60, s);
+ long monkeyInterval = SliderUtils.getTimeRange(conf,
+ CHAOS_MONKEY_INTERVAL,
+ DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS,
+ DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS,
+ DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES,
+ 0);
+ assertEquals(60L, monkeyInterval);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java
new file mode 100644
index 0000000..fd794ea
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.other;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.DiskChecker;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import org.apache.slider.utils.YarnMiniClusterTestBase;
+import org.junit.After;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This test class exists to look at permissions of the filesystem, especially
+ * that created by Mini YARN clusters. On some windows jenkins machines,
+ * YARN actions were failing as the directories had the wrong permissions
+ * (i.e. too lax)
+ */
+public class TestFilesystemPermissions extends YarnMiniClusterTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ TestFilesystemPermissions.class);
+
+ private List<File> filesToDelete = new ArrayList<>();
+
+ @After
+ public void deleteFiles() {
+ for (File f : filesToDelete) {
+ FileUtil.fullyDelete(f, true);
+ }
+ }
+
+ @Test
+ public void testJavaFSOperations() throws Throwable {
+ assertNativeLibrariesPresent();
+ File subdir = testDir();
+ subdir.mkdir();
+ assertTrue(subdir.isDirectory());
+ assertTrue(FileUtil.canRead(subdir));
+ assertTrue(FileUtil.canWrite(subdir));
+ assertTrue(FileUtil.canExecute(subdir));
+ }
+
+ @Test
+ public void testDiskCheckerOperations() throws Throwable {
+ assertNativeLibrariesPresent();
+ File subdir = testDir();
+ subdir.mkdir();
+ DiskChecker checker = new DiskChecker();
+ checker.checkDir(subdir);
+ }
+
+ @Test
+ public void testDiskCheckerMkdir() throws Throwable {
+ assertNativeLibrariesPresent();
+ File subdir = testDir();
+ subdir.mkdirs();
+ DiskChecker checker = new DiskChecker();
+ checker.checkDir(subdir);
+ }
+
+ /**
+ * Get a test dir for this method; one that will be deleted on teardown.
+ * @return a filename unique to this test method
+ */
+ File testDir() {
+ File parent = new File("target/testfspermissions");
+ parent.mkdir();
+ File testdir = new File(parent, methodName.getMethodName());
+ filesToDelete.add(testdir);
+ return testdir;
+ }
+
+
+ @Test
+ public void testPermsMap() throws Throwable {
+ File dir = testDir();
+ String diruri = dir.toURI().toString();
+ FileContext lfs = createLocalFS(dir, getConfiguration());
+ getLocalDirsPathPermissionsMap(lfs, diruri);
+ }
+
+ @Test
+ public void testInitLocaldir() throws Throwable {
+ File dir = testDir();
+ String diruri = dir.toURI().toString();
+ FileContext lfs = createLocalFS(dir, getConfiguration());
+ initializeLocalDir(lfs, diruri);
+ List<String> localDirs = getInitializedLocalDirs(lfs, Arrays.asList(
+ diruri));
+ assertEquals(1, localDirs.size());
+ }
+
+
+ @Test
+ public void testValidateMiniclusterPerms() throws Throwable {
+ int numLocal = 1;
+ String cluster = createMiniCluster("", getConfiguration(), 1, numLocal, 1,
+ false);
+ File workDir = getMiniCluster().getTestWorkDir();
+ List<File> localdirs = new ArrayList<>();
+ for (File file : workDir.listFiles()) {
+ if (file.isDirectory() && file.getAbsolutePath().contains("-local")) {
+ // local dir
+ localdirs.add(file);
+ }
+ }
+ assertEquals(numLocal, localdirs.size());
+ FileContext lfs = createLocalFS(workDir, getConfiguration());
+ for (File file : localdirs) {
+ checkLocalDir(lfs, file.toURI().toString());
+ }
+ }
+
+ FileContext createLocalFS(File dir, Configuration conf)
+ throws UnsupportedFileSystemException {
+ return FileContext.getFileContext(dir.toURI(), conf);
+ }
+
+ /**
+ * Extracted from ResourceLocalizationService.
+ * @param lfs
+ * @param localDir
+ * @return perms map
+ * @see ResourceLocalizationService
+ */
+ private Map<Path, FsPermission> getLocalDirsPathPermissionsMap(
+ FileContext lfs,
+ String localDir) {
+ Map<Path, FsPermission> localDirPathFsPermissionsMap = new HashMap<>();
+
+ FsPermission defaultPermission =
+ FsPermission.getDirDefault().applyUMask(lfs.getUMask());
+ FsPermission nmPrivatePermission =
+ ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask());
+
+ Path userDir = new Path(localDir, ContainerLocalizer.USERCACHE);
+ Path fileDir = new Path(localDir, ContainerLocalizer.FILECACHE);
+ Path sysDir = new Path(
+ localDir,
+ ResourceLocalizationService.NM_PRIVATE_DIR);
+
+ localDirPathFsPermissionsMap.put(userDir, defaultPermission);
+ localDirPathFsPermissionsMap.put(fileDir, defaultPermission);
+ localDirPathFsPermissionsMap.put(sysDir, nmPrivatePermission);
+ return localDirPathFsPermissionsMap;
+ }
+
+ private boolean checkLocalDir(FileContext lfs, String localDir)
+ throws IOException {
+
+ Map<Path, FsPermission> pathPermissionMap =
+ getLocalDirsPathPermissionsMap(lfs, localDir);
+
+ for (Map.Entry<Path, FsPermission> entry : pathPermissionMap.entrySet()) {
+ FileStatus status;
+ status = lfs.getFileStatus(entry.getKey());
+
+ if (!status.getPermission().equals(entry.getValue())) {
+ String msg =
+ "Permissions incorrectly set for dir " + entry.getKey() +
+ ", should be " + entry.getValue() + ", actual value = " +
+ status.getPermission();
+ throw new YarnRuntimeException(msg);
+ }
+ }
+ return true;
+ }
+
+
+ private void initializeLocalDir(FileContext lfs, String localDir)
+ throws IOException {
+
+ Map<Path, FsPermission> pathPermissionMap =
+ getLocalDirsPathPermissionsMap(lfs, localDir);
+ for (Map.Entry<Path, FsPermission> entry : pathPermissionMap.entrySet()) {
+ FileStatus status;
+ try {
+ status = lfs.getFileStatus(entry.getKey());
+ } catch (FileNotFoundException fs) {
+ status = null;
+ }
+
+ if (status == null) {
+ lfs.mkdir(entry.getKey(), entry.getValue(), true);
+ status = lfs.getFileStatus(entry.getKey());
+ }
+ FsPermission perms = status.getPermission();
+ if (!perms.equals(entry.getValue())) {
+ lfs.setPermission(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+
+ synchronized private List<String> getInitializedLocalDirs(FileContext lfs,
+ List<String> dirs) throws IOException {
+ List<String> checkFailedDirs = new ArrayList<String>();
+ for (String dir : dirs) {
+ try {
+ checkLocalDir(lfs, dir);
+ } catch (YarnRuntimeException e) {
+ checkFailedDirs.add(dir);
+ }
+ }
+ for (String dir : checkFailedDirs) {
+ LOG.info("Attempting to initialize " + dir);
+ initializeLocalDir(lfs, dir);
+ checkLocalDir(lfs, dir);
+ }
+ return dirs;
+ }
+
+
+ private void createDir(FileContext localFs, Path dir, FsPermission perm)
+ throws IOException {
+ if (dir == null) {
+ return;
+ }
+ try {
+ localFs.getFileStatus(dir);
+ } catch (FileNotFoundException e) {
+ createDir(localFs, dir.getParent(), perm);
+ localFs.mkdir(dir, perm, false);
+ if (!perm.equals(perm.applyUMask(localFs.getUMask()))) {
+ localFs.setPermission(dir, perm);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java
new file mode 100644
index 0000000..704c71e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.other;
+
+import org.apache.slider.utils.SliderTestUtils;
+import org.apache.slider.utils.TestUtility;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+/**
+ * This test exists to diagnose local FS permissions.
+ */
+public class TestLocalDirStatus extends SliderTestUtils {
+
+
+ public static final int SIZE = 0x200000;
+
+ @Test
+ public void testTempDir() throws Throwable {
+ File tmpf = null;
+ try {
+ tmpf = File.createTempFile("testl", ".bin");
+ createAndReadFile(tmpf, SIZE);
+ tmpf.delete();
+ assertFalse(tmpf.exists());
+ } finally {
+ if (tmpf != null) {
+ tmpf.delete();
+ }
+ }
+ }
+
+ @Test
+ public void testTargetDir() throws Throwable {
+ File target = target();
+ File tmpf = null;
+ try {
+ tmpf = File.createTempFile("testl", ".bin", target);
+ createAndReadFile(tmpf, SIZE);
+ tmpf.delete();
+ assertFalse(tmpf.exists());
+ } finally {
+ if (tmpf != null) {
+ tmpf.delete();
+ }
+
+ }
+ }
+
+ public File target() {
+ File target = new File("target").getAbsoluteFile();
+ assertTrue(target.exists());
+ return target;
+ }
+
+ @Test
+ public void testRenameInTargetDir() throws Throwable {
+ File target = target();
+ File tmpf = null;
+ File dst= null;
+ try {
+ tmpf = File.createTempFile("testl", ".bin", target);
+ dst = File.createTempFile("test-dest", ".bin", target);
+ createRenameAndReadFile(tmpf, dst, SIZE);
+ assertFalse(tmpf.exists());
+ dst.delete();
+ } finally {
+ if (tmpf != null) {
+ tmpf.delete();
+ }
+ if (dst != null) {
+ dst.delete();
+ }
+ }
+ }
+
+ @Test
+ public void testRenameInTmpDir() throws Throwable {
+ File tmpf = null;
+ File dst= null;
+ try {
+ tmpf = File.createTempFile("testl", ".bin");
+ dst = File.createTempFile("test-dest", ".bin");
+ createRenameAndReadFile(tmpf, dst, SIZE);
+ assertFalse(tmpf.exists());
+ dst.delete();
+ } finally {
+ if (tmpf != null) {
+ tmpf.delete();
+ }
+ if (dst != null) {
+ dst.delete();
+ }
+ }
+ }
+
+ protected void createAndReadFile(File path, int len) throws IOException {
+ byte[] dataset = TestUtility.dataset(len, 32, 128);
+ writeFile(path, dataset);
+ assertTrue(path.exists());
+ assertEquals(len, path.length());
+ byte[] persisted = readFile(path);
+ TestUtility.compareByteArrays(dataset, persisted, len);
+ }
+
+ protected void createRenameAndReadFile(File src, File dst, int len)
+ throws IOException {
+ byte[] dataset = TestUtility.dataset(len, 32, 128);
+ writeFile(src, dataset);
+ assertTrue(src.exists());
+ assertEquals(len, src.length());
+ dst.delete();
+ assertFalse(dst.exists());
+ assertTrue(src.renameTo(dst));
+ assertEquals(len, dst.length());
+ byte[] persisted = readFile(dst);
+ TestUtility.compareByteArrays(dataset, persisted, len);
+ }
+
+ protected void writeFile(File path, byte[] dataset)
+ throws IOException {
+ FileOutputStream out = new FileOutputStream(path);
+ try {
+ out.write(dataset);
+ out.flush();
+ } finally {
+ out.close();
+ }
+ }
+
+ protected byte[] readFile(File path) throws IOException {
+ assertTrue(path.getAbsoluteFile().exists());
+ assertTrue(path.getAbsoluteFile().isFile());
+ int len = (int)path.length();
+ byte[] dataset = new byte[len];
+ FileInputStream ins = new FileInputStream(path);
+ try {
+ ins.read(dataset);
+ } finally {
+ ins.close();
+ }
+ return dataset;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java
new file mode 100644
index 0000000..11abdfe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers;
+
+import org.apache.slider.providers.docker.DockerKeys;
+import org.apache.slider.providers.docker.DockerProviderFactory;
+import org.junit.Test;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test provider factories.
+ */
+public class TestProviderFactory {
+ @Test
+ public void testLoadAgentProvider() throws Throwable {
+ SliderProviderFactory factory = SliderProviderFactory
+ .createSliderProviderFactory(DockerKeys.PROVIDER_DOCKER);
+ assertTrue(factory instanceof DockerProviderFactory);
+ }
+
+ @Test
+ public void testCreateClientProvider() throws Throwable {
+ SliderProviderFactory factory = SliderProviderFactory
+ .createSliderProviderFactory(DockerKeys.PROVIDER_DOCKER);
+ assertNotNull(factory.createClientProvider());
+ }
+
+ @Test
+ public void testCreateProviderByClassname() throws Throwable {
+ SliderProviderFactory factory = SliderProviderFactory
+ .createSliderProviderFactory(DockerKeys.PROVIDER_DOCKER);
+ assertNotNull(factory.createServerProvider());
+ assertTrue(factory instanceof DockerProviderFactory);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java
new file mode 100644
index 0000000..100518e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.registry;
+
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+/**
+ * Test config set name validation.
+ */
+public class TestConfigSetNaming {
+
+ void assertValid(String name) {
+ PublishedConfigSet.validateName(name);
+ }
+
+ void assertInvalid(String name) {
+ try {
+ PublishedConfigSet.validateName(name);
+ Assert.fail("Invalid name was unexpectedly parsed: " + name);
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testLowerCase() throws Throwable {
+ assertValid("abcdefghijklmnopqrstuvwxyz");
+ }
+
+ @Test
+ public void testUpperCaseInvalid() throws Throwable {
+ assertInvalid("ABCDEFGHIJKLMNOPQRSTUVWXYZ");
+ }
+
+ @Test
+ public void testNumbers() throws Throwable {
+ assertValid("01234567890");
+ }
+
+ @Test
+ public void testChars() throws Throwable {
+ assertValid("a-_+");
+ }
+
+ @Test
+ public void testInvalids() throws Throwable {
+ for (String s : Arrays.asList(
+ "",
+ " ",
+ "*",
+ "a/b",
+ "b\\a",
+ "\"",
+ "'",
+ "\u0000",
+ "\u0f00",
+ "key.value",
+ "-",
+ "+",
+ "_",
+ "?")) {
+ assertInvalid(s);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java
new file mode 100644
index 0000000..5fcfd89
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.registry;
+
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.slider.core.registry.SliderRegistryUtils;
+import org.apache.slider.utils.SliderTestUtils;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test registry paths.
+ */
+public class TestRegistryPaths {
+
+ @Test
+ public void testHomedirKerberos() throws Throwable {
+ String home = RegistryUtils.homePathForUser("hbase@HADOOP.APACHE.ORG");
+ try {
+ assertEquals("/users/hbase", home);
+ } catch (AssertionError e) {
+ SliderTestUtils.skip("homedir filtering not yet in hadoop registry " +
+ "module");
+ }
+ }
+
+ @Test
+ public void testHomedirKerberosHost() throws Throwable {
+ String home = RegistryUtils.homePathForUser("hbase/localhost@HADOOP" +
+ ".APACHE.ORG");
+ try {
+ assertEquals("/users/hbase", home);
+ } catch (AssertionError e) {
+ SliderTestUtils.skip("homedir filtering not yet in hadoop registry " +
+ "module");
+ }
+ }
+
+ @Test
+ public void testRegistryPathForInstance() throws Throwable {
+ String path = SliderRegistryUtils.registryPathForInstance("instance");
+ assertTrue(path.endsWith("/instance"));
+ }
+
+ @Test
+ public void testPathResolution() throws Throwable {
+ String home = RegistryUtils.homePathForCurrentUser();
+ assertEquals(home, SliderRegistryUtils.resolvePath("~"));
+ assertEquals(home +"/", SliderRegistryUtils.resolvePath("~/"));
+ assertEquals(home +"/something", SliderRegistryUtils.resolvePath(
+ "~/something"));
+ assertEquals("~unresolved", SliderRegistryUtils.resolvePath(
+ "~unresolved"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java
new file mode 100644
index 0000000..68d55aa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.ServiceOperations;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.services.workflow.ServiceThreadFactory;
+import org.apache.slider.server.services.workflow.WorkflowExecutorService;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test AM actions.
+ */
+public class TestActions {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(TestActions.class);
+
+ private QueueService queues;
+ private WorkflowExecutorService<ExecutorService> executorService;
+
+
+ @Before
+ public void createService() {
+ queues = new QueueService();
+
+ Configuration conf = new Configuration();
+ queues.init(conf);
+
+ queues.start();
+
+ executorService = new WorkflowExecutorService<>("AmExecutor",
+ Executors.newCachedThreadPool(
+ new ServiceThreadFactory("AmExecutor", true)));
+
+ executorService.init(conf);
+ executorService.start();
+ }
+
+ @After
+ public void destroyService() {
+ ServiceOperations.stop(executorService);
+ ServiceOperations.stop(queues);
+ }
+
+ @Test
+ public void testBasicService() throws Throwable {
+ queues.start();
+ }
+
+ @Test
+ public void testDelayLogic() throws Throwable {
+ ActionNoteExecuted action = new ActionNoteExecuted("", 1000);
+ long now = System.currentTimeMillis();
+
+ long delay = action.getDelay(TimeUnit.MILLISECONDS);
+ assertTrue(delay >= 800);
+ assertTrue(delay <= 1800);
+
+ ActionNoteExecuted a2 = new ActionNoteExecuted("a2", 10000);
+ assertTrue(action.compareTo(a2) < 0);
+ assertTrue(a2.compareTo(action) > 0);
+ assertEquals(0, action.compareTo(action));
+
+ }
+
+ @Test
+ public void testActionDelayedExecutorTermination() throws Throwable {
+ long start = System.currentTimeMillis();
+
+ ActionStopQueue stopAction = new ActionStopQueue(1000);
+ queues.scheduledActions.add(stopAction);
+ queues.run();
+ AsyncAction take = queues.actionQueue.take();
+ assertEquals(take, stopAction);
+ long stop = System.currentTimeMillis();
+ assertTrue(stop - start > 500);
+ assertTrue(stop - start < 1500);
+ }
+
+ @Test
+ public void testImmediateQueue() throws Throwable {
+ ActionNoteExecuted noteExecuted = new ActionNoteExecuted("executed", 0);
+ queues.put(noteExecuted);
+ queues.put(new ActionStopQueue(0));
+ QueueExecutor ex = new QueueExecutor(queues);
+ ex.run();
+ assertTrue(queues.actionQueue.isEmpty());
+ assertTrue(noteExecuted.executed.get());
+ }
+
+ @Test
+ public void testActionOrdering() throws Throwable {
+
+ ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500);
+ ActionStopQueue stop = new ActionStopQueue(1500);
+ ActionNoteExecuted note2 = new ActionNoteExecuted("note2", 800);
+
+ List<AsyncAction> actions = Arrays.asList(note1, stop, note2);
+ Collections.sort(actions);
+ assertEquals(actions.get(0), note1);
+ assertEquals(actions.get(1), note2);
+ assertEquals(actions.get(2), stop);
+ }
+
+ @Test
+ public void testDelayedQueueWithReschedule() throws Throwable {
+
+ ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500);
+ ActionStopQueue stop = new ActionStopQueue(1500);
+ ActionNoteExecuted note2 = new ActionNoteExecuted("note2", 800);
+
+ assertTrue(note2.compareTo(stop) < 0);
+ assertTrue(note1.getNanos() < note2.getNanos());
+ assertTrue(note2.getNanos() < stop.getNanos());
+ queues.schedule(note1);
+ queues.schedule(note2);
+ queues.schedule(stop);
+ // async to sync expected to run in order
+ runQueuesToCompletion();
+ assertTrue(note1.executed.get());
+ assertTrue(note2.executed.get());
+ }
+
+ public void runQueuesToCompletion() {
+ queues.run();
+ assertTrue(queues.scheduledActions.isEmpty());
+ assertFalse(queues.actionQueue.isEmpty());
+ QueueExecutor ex = new QueueExecutor(queues);
+ ex.run();
+ // flush all stop commands from the queue
+ queues.flushActionQueue(ActionStopQueue.class);
+
+ assertTrue(queues.actionQueue.isEmpty());
+ }
+
+ @Test
+ public void testRenewedActionFiresOnceAtLeast() throws Throwable {
+ ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500);
+ RenewingAction renewer = new RenewingAction(
+ note1,
+ 500,
+ 100,
+ TimeUnit.MILLISECONDS,
+ 3);
+ queues.schedule(renewer);
+ ActionStopQueue stop = new ActionStopQueue(4, TimeUnit.SECONDS);
+ queues.schedule(stop);
+ // this runs all the delayed actions FIRST, so can't be used
+ // to play tricks of renewing actions ahead of the stop action
+ runQueuesToCompletion();
+ assertEquals(1, renewer.executionCount.intValue());
+ assertEquals(1, note1.executionCount.intValue());
+ // assert the renewed item is back in
+ assertTrue(queues.scheduledActions.contains(renewer));
+ }
+
+
+ @Test
+ public void testRenewingActionOperations() throws Throwable {
+ ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500);
+ RenewingAction renewer = new RenewingAction(
+ note1,
+ 100,
+ 100,
+ TimeUnit.MILLISECONDS,
+ 3);
+ queues.renewing("note", renewer);
+ assertTrue(queues.removeRenewingAction("note"));
+ queues.stop();
+ assertTrue(queues.waitForServiceToStop(10000));
+ }
+
+ /**
+ * Test action.
+ */
+ public class ActionNoteExecuted extends AsyncAction {
+ private final AtomicBoolean executed = new AtomicBoolean(false);
+ private final AtomicLong executionTimeNanos = new AtomicLong();
+ private final AtomicLong executionCount = new AtomicLong();
+
+ public ActionNoteExecuted(String text, int delay) {
+ super(text, delay);
+ }
+
+ @Override
+ public void execute(
+ SliderAppMaster appMaster,
+ QueueAccess queueService,
+ AppState appState) throws Exception {
+ LOG.info("Executing {}", name);
+ executed.set(true);
+ executionTimeNanos.set(System.nanoTime());
+ executionCount.incrementAndGet();
+ LOG.info(this.toString());
+
+ synchronized (this) {
+ this.notify();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + " executed=" + executed.get() + "; count=" +
+ executionCount.get() + ";";
+ }
+
+ public long getExecutionCount() {
+ return executionCount.get();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java
new file mode 100644
index 0000000..c1f2886
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate;
+
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.providers.PlacementPolicy;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockRoles;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+
+import static org.apache.slider.api.ResourceKeys.COMPONENT_PLACEMENT_POLICY;
+
+/**
+ * Class for basis of Anti-affine placement tests; sets up role2
+ * for anti-affinity.
+ */
+public class BaseMockAppStateAATest extends BaseMockAppStateTest
+ implements MockRoles {
+
+ /** Role status for the base AA role. */
+ private RoleStatus aaRole;
+
+ /** Role status for the AA role requiring a node with the gpu label. */
+ private RoleStatus gpuRole;
+
+ @Override
+ public Application buildApplication() {
+ Application application = factory.newApplication(0, 0, 0)
+ .name(getTestName());
+ application.getComponent(ROLE1).getConfiguration().setProperty(
+ COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy
+ .ANTI_AFFINITY_REQUIRED));
+ application.getComponent(ROLE1).getConfiguration().setProperty(
+ ResourceKeys.YARN_LABEL_EXPRESSION, LABEL_GPU);
+ application.getComponent(ROLE2).getConfiguration().setProperty(
+ COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy
+ .ANTI_AFFINITY_REQUIRED));
+ return application;
+ }
+
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ aaRole = lookupRole(ROLE2);
+ gpuRole = lookupRole(ROLE1);
+ }
+
+ protected RoleStatus getAaRole() {
+ return aaRole;
+ }
+
+ protected RoleStatus getGpuRole() {
+ return gpuRole;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org