You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/08/22 19:54:57 UTC
[01/11] git commit: SLIDER-350: AM not coming up on Windows
Repository: incubator-slider
Updated Branches:
refs/heads/develop 4e8c6ffbb -> aa21ad64d
SLIDER-350: AM not coming up on Windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a5df3975
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a5df3975
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a5df3975
Branch: refs/heads/develop
Commit: a5df39755bfd044f1d2ee4cc49e8d96f80a855d2
Parents: 4e8c6ff
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 22 14:32:39 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 22 14:32:39 2014 +0100
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 2 +-
.../server/appmaster/web/AgentService.java | 28 +++++---------------
.../appmaster/web/rest/agent/AgentWebApp.java | 16 ++++++++---
.../LaunchedWorkflowCompositeService.java | 4 +++
.../services/workflow/ClosingService.java | 3 +++
.../web/rest/agent/TestAMAgentWebServices.java | 2 +-
.../workflow/TestWorkflowClosingService.java | 4 +--
7 files changed, 29 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a5df3975/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index c006d0d..b4515f1 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -813,7 +813,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
private void startAgentWebApp(MapOperations appInformation,
- Configuration serviceConf) {
+ Configuration serviceConf) throws IOException {
URL[] urls = ((URLClassLoader) AgentWebApp.class.getClassLoader() ).getURLs();
StringBuilder sb = new StringBuilder("AM classpath:");
for (URL url : urls) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a5df3975/slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
index 08338e8..f840035 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
@@ -16,38 +16,22 @@
*/
package org.apache.slider.server.appmaster.web;
-import org.apache.hadoop.service.AbstractService;
import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
+import org.apache.slider.server.services.workflow.ClosingService;
+import org.apache.slider.server.services.workflow.WorkflowCompositeService;
/**
- *
+ * agent service gives the agent webapp lifecycle integration
*/
-public class AgentService extends AbstractService {
- private volatile AgentWebApp webApp;
+public class AgentService extends ClosingService<AgentWebApp> {
+
public AgentService(String name) {
super(name);
}
public AgentService(String name, AgentWebApp app) {
- super(name);
- webApp = app;
+ super(name, app);
}
- @Override
- protected void serviceStart() throws Exception {
-
- }
-
- /**
- * Stop operation stops the webapp; sets the reference to null
- * @throws Exception
- */
- @Override
- protected void serviceStop() throws Exception {
- if (webApp != null) {
- webApp.stop();
- webApp = null;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a5df3975/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
index 54d2b1f..8aac490 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
@@ -36,13 +36,15 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.ext.Provider;
+import java.io.Closeable;
import java.io.File;
+import java.io.IOException;
import java.util.Set;
/**
*
*/
-public class AgentWebApp {
+public class AgentWebApp implements Closeable {
protected static final Logger LOG = LoggerFactory.getLogger(AgentWebApp.class);
private int port;
private int securedPort;
@@ -66,7 +68,7 @@ public class AgentWebApp {
return this;
}
- public AgentWebApp start() {
+ public AgentWebApp start() throws IOException {
if (configsMap == null) {
throw new IllegalStateException("No SSL Configuration Available");
}
@@ -100,8 +102,12 @@ public class AgentWebApp {
try {
agentServer.start();
+ } catch (IOException e) {
+ LOG.error("Unable to start agent server", e);
+ throw e;
} catch (Exception e) {
LOG.error("Unable to start agent server", e);
+ throw new IOException("Unable to start agent server: " + e, e);
}
AgentWebApp webApp = new AgentWebApp();
@@ -191,12 +197,14 @@ public class AgentWebApp {
return securedPort;
}
- public void stop() {
+ public void close() throws IOException{
//need to stop server and reset injector
try {
agentServer.stop();
+ } catch (IOException e) {
+ throw e;
} catch (Exception e) {
- LOG.warn("Unable to stop agent server", e);
+ throw new IOException(e.toString(), e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a5df3975/slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
index 0d47c3b..e939c20 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
@@ -28,6 +28,10 @@ import org.apache.slider.server.services.workflow.WorkflowCompositeService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * This is a workflow compositoe service which can be launched from the CLI
+ * ... catches the arguments and implements a stub runService operation.
+ */
public class LaunchedWorkflowCompositeService extends WorkflowCompositeService
implements RunService {
private static final Logger log = LoggerFactory.getLogger(
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a5df3975/slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
index 7a475cc..8b711aa 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
@@ -33,6 +33,9 @@ public class ClosingService<C extends Closeable> extends AbstractService {
private C closeable;
+ public ClosingService(String name) {
+ super(name);
+ }
/**
* Construct an instance of the service
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a5df3975/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
index 7e2ab3c..8590cbc 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -152,7 +152,7 @@ public class TestAMAgentWebServices {
@After
public void tearDown () throws Exception {
- webApp.stop();
+ webApp.close();
webApp = null;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a5df3975/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java
index 638547f..39516b7 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java
@@ -37,7 +37,7 @@ public class TestWorkflowClosingService extends WorkflowServiceTestBase {
@Test
public void testNullClose() throws Throwable {
- ClosingService<OpenClose> svc = new ClosingService<OpenClose>(null);
+ ClosingService<OpenClose> svc = new ClosingService<OpenClose>("", null);
svc.init(new Configuration());
svc.start();
assertNull(svc.getCloseable());
@@ -81,7 +81,7 @@ public class TestWorkflowClosingService extends WorkflowServiceTestBase {
@Test
public void testCloseSelf() throws Throwable {
ClosingService<ClosingService> svc =
- new ClosingService<ClosingService>(null);
+ new ClosingService<ClosingService>("");
svc.setCloseable(svc);
svc.stop();
}
[05/11] git commit: SLIDER-352 more testing of windows
Posted by st...@apache.org.
SLIDER-352 more testing of windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/db2d2985
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/db2d2985
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/db2d2985
Branch: refs/heads/develop
Commit: db2d29857faf53da2e8fce0c107209883980b266
Parents: 83665f6
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 22 16:15:02 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 22 16:15:02 2014 +0100
----------------------------------------------------------------------
.../apache/slider/common/tools/TestWindowsSupport.groovy | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/db2d2985/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index 4da42f2..8387237 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -24,13 +24,15 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.ChecksumFileSystem
import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.util.Shell
+import org.apache.slider.test.SliderTestUtils
import org.junit.Test
import java.util.regex.Pattern
@CompileStatic
@Slf4j
-class TestWindowsSupport {
+class TestWindowsSupport extends SliderTestUtils {
private static final Pattern hasDriveLetterSpecifier =
Pattern.compile("^/?[a-zA-Z]:");
@@ -62,17 +64,18 @@ class TestWindowsSupport {
@Test
public void testPathHandling() throws Throwable {
- System.setProperty("os.name", "Windows/286")
+ assume(Shell.WINDOWS, "not windows")
+
Path path = new Path(windowsFile);
def uri = path.toUri()
// assert "file" == uri.scheme
assert uri.authority == null;
-
-
Configuration conf = new Configuration()
def localfs = HadoopFS.get(uri, conf)
assert localfs instanceof ChecksumFileSystem
+ def stat = localfs.getFileStatus(path)
+
}
}
[11/11] git commit: Merge branch
'feature/SLIDER-350-slider-core_tests_failing_on_windows' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-350-slider-core_tests_failing_on_windows' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/aa21ad64
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/aa21ad64
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/aa21ad64
Branch: refs/heads/develop
Commit: aa21ad64d201651c154114e6fd187ca156807552
Parents: 4e8c6ff 87ff4d8
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 22 18:54:42 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 22 18:54:42 2014 +0100
----------------------------------------------------------------------
.../slider/providers/agent/AgentUtils.java | 11 +-
.../server/appmaster/SliderAppMaster.java | 2 +-
.../server/appmaster/web/AgentService.java | 28 ++---
.../appmaster/web/rest/agent/AgentWebApp.java | 16 ++-
.../LaunchedWorkflowCompositeService.java | 4 +
.../services/workflow/ClosingService.java | 3 +
.../agent/AgentMiniClusterTestBase.groovy | 9 +-
.../common/tools/TestWindowsSupport.groovy | 104 +++++++++++++++++++
.../publisher/TestPublisherRestResources.groovy | 11 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 4 +-
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../workflow/TestWorkflowClosingService.java | 4 +-
.../providers/accumulo/AccumuloTestBase.groovy | 7 +-
.../minicluster/HBaseMiniClusterTestBase.groovy | 9 +-
14 files changed, 173 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
[07/11] git commit: SLIDER-352 more testing of windows
Posted by st...@apache.org.
SLIDER-352 more testing of windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/475507b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/475507b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/475507b5
Branch: refs/heads/develop
Commit: 475507b598b6b23815d5b6f983206fb9cbb20412
Parents: 53457d9
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 22 16:25:46 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 22 16:26:06 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/providers/agent/AgentUtils.java | 2 +-
.../slider/common/tools/TestWindowsSupport.groovy | 12 ++++++++++++
2 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/475507b5/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
index f329756..6bb6132 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
@@ -35,7 +35,7 @@ import java.io.InputStream;
public class AgentUtils {
private static final Logger log = LoggerFactory.getLogger(AgentUtils.class);
- static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
+ public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
String appDef) throws IOException {
log.info("Reading metainfo at {}", appDef);
FileSystem fs = fileSystem.getFileSystem();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/475507b5/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index 2d74760..e6d2225 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FSDataInputStream
import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.util.Shell
+import org.apache.slider.providers.agent.AgentUtils
import org.apache.slider.test.SliderTestUtils
import org.junit.Test
@@ -89,4 +90,15 @@ class TestWindowsSupport extends SliderTestUtils {
// expected
}
}
+
+ @Test
+ public void testSliderFS() throws Throwable {
+ SliderFileSystem sfs = new SliderFileSystem(new Configuration())
+ try {
+ def metainfo = AgentUtils.getApplicationMetainfo(sfs, windowsFile)
+ } catch (FileNotFoundException fnfe) {
+ // expected
+ }
+
+ }
}
[04/11] git commit: SLIDER-352 some tests
Posted by st...@apache.org.
SLIDER-352 some tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/83665f61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/83665f61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/83665f61
Branch: refs/heads/develop
Commit: 83665f617025fea9f8f7e5462e30d66caab025d0
Parents: b3a835d
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 22 16:08:24 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 22 16:08:24 2014 +0100
----------------------------------------------------------------------
.../common/tools/TestWindowsSupport.groovy | 78 ++++++++++++++++++++
1 file changed, 78 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/83665f61/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
new file mode 100644
index 0000000..4da42f2
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.ChecksumFileSystem
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.junit.Test
+
+import java.util.regex.Pattern
+
+@CompileStatic
+@Slf4j
+class TestWindowsSupport {
+
+ private static final Pattern hasDriveLetterSpecifier =
+ Pattern.compile("^/?[a-zA-Z]:");
+ public static
+ final String windowsFile = "C:\\Users\\Administrator\\AppData\\Local\\Temp\\junit3180177850133852404\\testpkg\\appdef_1.zip"
+
+
+ private static boolean hasWindowsDrive(String path) {
+ return hasDriveLetterSpecifier.matcher(path).find();
+ }
+
+ private static int startPositionWithoutWindowsDrive(String path) {
+ if (hasWindowsDrive(path)) {
+ return path.charAt(0) == '/' ? 3 : 2;
+ } else {
+ return 0;
+ }
+ }
+
+ @Test
+ public void testHasWindowsDrive() throws Throwable {
+ assert hasWindowsDrive(windowsFile)
+ }
+
+ @Test
+ public void testStartPosition() throws Throwable {
+ assert 2 == startPositionWithoutWindowsDrive(windowsFile)
+ }
+
+ @Test
+ public void testPathHandling() throws Throwable {
+ System.setProperty("os.name", "Windows/286")
+ Path path = new Path(windowsFile);
+ def uri = path.toUri()
+// assert "file" == uri.scheme
+ assert uri.authority == null;
+
+
+
+ Configuration conf = new Configuration()
+
+ def localfs = HadoopFS.get(uri, conf)
+ assert localfs instanceof ChecksumFileSystem
+ }
+}
[06/11] git commit: SLIDER-352 more testing of windows
Posted by st...@apache.org.
SLIDER-352 more testing of windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/53457d98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/53457d98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/53457d98
Branch: refs/heads/develop
Commit: 53457d98d480e3b4134175c09b4927d0c1dfb4ef
Parents: db2d298
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 22 16:20:26 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 22 16:20:26 2014 +0100
----------------------------------------------------------------------
.../apache/slider/providers/agent/AgentUtils.java | 9 +++++++--
.../slider/common/tools/TestWindowsSupport.groovy | 15 +++++++++++++--
2 files changed, 20 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53457d98/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
index f296a95..f329756 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
@@ -16,6 +16,7 @@
*/
package org.apache.slider.providers.agent;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
@@ -24,6 +25,7 @@ import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@@ -36,11 +38,14 @@ public class AgentUtils {
static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
String appDef) throws IOException {
log.info("Reading metainfo at {}", appDef);
+ FileSystem fs = fileSystem.getFileSystem();
+ Path appPath = new Path(appDef);
InputStream metainfoStream = SliderUtils.getApplicationResourceInputStream(
- fileSystem.getFileSystem(), new Path(appDef), "metainfo.xml");
+ fs, appPath, "metainfo.xml");
if (metainfoStream == null) {
log.error("metainfo.xml is unavailable at {}.", appDef);
- throw new IOException("metainfo.xml is required in app package.");
+ throw new FileNotFoundException("metainfo.xml is required in app package. " +
+ appPath);
}
Metainfo metainfo = new MetainfoParser().parse(metainfoStream);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53457d98/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index 8387237..2d74760 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -22,6 +22,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.ChecksumFileSystem
+import org.apache.hadoop.fs.FSDataInputStream
import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.util.Shell
@@ -75,7 +76,17 @@ class TestWindowsSupport extends SliderTestUtils {
def localfs = HadoopFS.get(uri, conf)
assert localfs instanceof ChecksumFileSystem
- def stat = localfs.getFileStatus(path)
-
+ try {
+ def stat = localfs.getFileStatus(path)
+ fail("expected an exception, got $stat")
+ } catch (FileNotFoundException fnfe) {
+ // expected
+ }
+
+ try {
+ FSDataInputStream appStream = localfs.open(path);
+ } catch (FileNotFoundException fnfe) {
+ // expected
+ }
}
}
[10/11] git commit: SLIDER-353 killJavaProcesses doesn't work on
windows ... skip when reached, and swallowed on teardown
Posted by st...@apache.org.
SLIDER-353 killJavaProcesses doesn't work on windows ... skip when reached, and swallowed on teardown
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/87ff4d80
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/87ff4d80
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/87ff4d80
Branch: refs/heads/develop
Commit: 87ff4d80cad1e7b1badaad6e0a5c04961e633430
Parents: 8b43def
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 22 17:05:09 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 22 17:05:09 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/test/YarnMiniClusterTestBase.groovy | 4 +++-
.../slider/providers/accumulo/AccumuloTestBase.groovy | 7 ++++++-
.../hbase/minicluster/HBaseMiniClusterTestBase.groovy | 9 +++++++--
3 files changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/87ff4d80/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index b6f863b..09c56a5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -64,7 +64,7 @@ import static org.apache.slider.common.SliderXMLConfKeysForTesting.*;
* Base class for mini cluster tests -creates a field for the
* mini yarn cluster
*/
-@CompileStatic
+//@CompileStatic
@Slf4j
public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
/**
@@ -327,6 +327,8 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
*/
public void killJavaProcesses(String grepString, int signal) {
+ assume(!Shell.WINDOWS, "failed to kill -$signal $grepString - no windows support ")
+
GString bashCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill $signal"
log.info("Bash command = $bashCommand" )
Process bash = ["bash", "-c", bashCommand].execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/87ff4d80/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
index bf35207..93b2798 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
@@ -27,6 +27,7 @@ import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ResourceKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.test.YarnZKMiniClusterTestBase
+import org.junit.internal.AssumptionViolatedException
import static org.apache.slider.common.SliderXMLConfKeysForTesting.*
import static org.apache.slider.providers.accumulo.AccumuloKeys.*
@@ -71,7 +72,11 @@ public abstract class AccumuloTestBase extends YarnZKMiniClusterTestBase {
void teardown() {
super.teardown();
if (teardownKillall) {
- killAllAccumuloProcesses();
+ try {
+ killAllAccumuloProcesses();
+ } catch (AssumptionViolatedException e) {
+ log.info e.toString();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/87ff4d80/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index 6a69e17..10de9ac 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -30,6 +30,7 @@ import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.hbase.HBaseTestUtils
import org.apache.slider.test.YarnZKMiniClusterTestBase
+import org.junit.internal.AssumptionViolatedException
import static org.apache.slider.common.params.Arguments.*
import static org.apache.slider.test.SliderTestUtils.*
@@ -76,8 +77,12 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
void teardown() {
super.teardown();
if (teardownKillall) {
- killAllRegionServers();
- killAllMasterServers();
+ try {
+ killAllRegionServers();
+ killAllMasterServers();
+ } catch (AssumptionViolatedException e) {
+ log.info e.toString();
+ }
}
}
[08/11] git commit: SLIDER-352 TestPublisherRestResources failing on
windows
Posted by st...@apache.org.
SLIDER-352 TestPublisherRestResources failing on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4ca25b2c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4ca25b2c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4ca25b2c
Branch: refs/heads/develop
Commit: 4ca25b2c535bd8ebbb8d92b318436dffe778754f
Parents: 475507b
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 22 16:33:07 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 22 16:33:07 2014 +0100
----------------------------------------------------------------------
.../web/rest/publisher/TestPublisherRestResources.groovy | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4ca25b2c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
index c1732e6..7dbd3e2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
@@ -65,14 +65,15 @@ class TestPublisherRestResources extends AgentTestBase {
assert app_def_path.exists()
assert agt_ver_path.exists()
assert agt_conf_path.exists()
+
ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
roles,
[
ARG_PROVIDER, "org.apache.slider.server.appmaster.web.rest.publisher.TestSliderProviderFactory",
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
- ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
- ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath
+ ARG_OPTION, APP_DEF, toFileURI(app_def_path),
+ ARG_OPTION, AGENT_CONF, toFileURI(agt_conf_path),
+ ARG_OPTION, AGENT_VERSION, toFileURI(agt_ver_path)
],
true, true,
true)
@@ -142,4 +143,8 @@ class TestPublisherRestResources extends AgentTestBase {
assert !uris.any {it =~ /foo-bar/}
}
+ public String toFileURI(File filename) {
+ "file://" + filename.absolutePath
+ }
+
}
[03/11] git commit: SLIDER-350 error in test cleanup when file in use
on windows
Posted by st...@apache.org.
SLIDER-350 error in test cleanup when file in use on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b3a835dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b3a835dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b3a835dd
Branch: refs/heads/develop
Commit: b3a835ddc5182d2c82c8355ad5deba86f6063174
Parents: 4ff287b
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 22 15:14:56 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 22 15:14:56 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/agent/AgentMiniClusterTestBase.groovy | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3a835dd/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
index 74f7a3f..a957481 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
@@ -80,8 +80,13 @@ extends YarnZKMiniClusterTestBase {
@AfterClass
public static void cleanSubConfFiles() {
- if (tempFolder.getRoot().exists()) {
- FileUtils.deleteDirectory(tempFolder.getRoot());
+ def tempRoot = tempFolder.root
+ if (tempRoot.exists()) {
+ try {
+ FileUtils.deleteDirectory(tempRoot);
+ } catch (IOException e) {
+ log.warn("Failed to delete $tempRoot :$e", e)
+ }
}
}
[02/11] git commit: SLIDER-350 making teardown of failing tests more
robust
Posted by st...@apache.org.
SLIDER-350 making teardown of failing tests more robust
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4ff287be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4ff287be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4ff287be
Branch: refs/heads/develop
Commit: 4ff287bee75e09dd6c4b119540f8bb26b6760ef5
Parents: a5df397
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 22 14:58:37 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 22 14:58:37 2014 +0100
----------------------------------------------------------------------
.../server/appmaster/web/rest/agent/TestAMAgentWebServices.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4ff287be/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
index 8590cbc..827b47a 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -27,6 +27,7 @@ import com.sun.jersey.api.json.JSONConfiguration;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.SliderUtils;
@@ -152,7 +153,7 @@ public class TestAMAgentWebServices {
@After
public void tearDown () throws Exception {
- webApp.close();
+ IOUtils.closeStream(webApp);
webApp = null;
}
[09/11] git commit: SLIDER-352 TestPublisherRestResources failing on
windows
Posted by st...@apache.org.
SLIDER-352 TestPublisherRestResources failing on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8b43def0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8b43def0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8b43def0
Branch: refs/heads/develop
Commit: 8b43def08bfdb223cfc5d584acdc02bf3492f41c
Parents: 4ca25b2
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 22 16:41:14 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 22 16:41:14 2014 +0100
----------------------------------------------------------------------
.../appmaster/web/rest/publisher/TestPublisherRestResources.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8b43def0/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
index 7dbd3e2..21881be 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
@@ -144,7 +144,7 @@ class TestPublisherRestResources extends AgentTestBase {
}
public String toFileURI(File filename) {
- "file://" + filename.absolutePath
+ filename.toURI().toString()
}
}