You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by jm...@apache.org on 2014/05/16 22:10:21 UTC
svn commit: r1595332 - in /incubator/slider/trunk/slider-core/src:
main/java/org/apache/slider/client/ main/java/org/apache/slider/common/tools/
main/java/org/apache/slider/core/launch/
main/java/org/apache/slider/providers/ main/java/org/apache/slider...
Author: jmaron
Date: Fri May 16 20:10:20 2014
New Revision: 1595332
URL: http://svn.apache.org/r1595332
Log:
SLIDER-59 added name, version, and description fields/tags to slider app listing and made minor unit test corrections for failing tests
Added:
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
incubator/slider/trunk/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
Modified:
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java?rev=1595332&r1=1595331&r2=1595332&view=diff
==============================================================================
--- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java (original)
+++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java Fri May 16 20:10:20 2014
@@ -121,6 +121,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
/**
* Client service for Slider
@@ -688,13 +689,18 @@ public class SliderClient extends Abstra
}
MapOperations sliderAMResourceComponent =
resourceOperations.getOrAddComponent(SliderKeys.COMPONENT_AM);
+
+ // add the tags if available
+ Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem,
+ appOperations.getGlobalOptions().get(AgentKeys.APP_DEF));
AppMasterLauncher amLauncher = new AppMasterLauncher(clustername,
SliderKeys.APP_TYPE,
config,
sliderFileSystem,
yarnClient,
clusterSecure,
- sliderAMResourceComponent);
+ sliderAMResourceComponent,
+ applicationTags);
ApplicationId appId = amLauncher.getApplicationId();
// set the application name;
Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java?rev=1595332&r1=1595331&r2=1595332&view=diff
==============================================================================
--- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java (original)
+++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java Fri May 16 20:10:20 2014
@@ -79,6 +79,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeSet;
@@ -525,9 +526,14 @@ public final class SliderUtils {
public static String appReportToString(ApplicationReport r, String separator) {
StringBuilder builder = new StringBuilder(512);
builder.append("application ").append(
- r.getName()).append("/").append(r.getApplicationType());
- builder.append(separator).append(
- "state: ").append(r.getYarnApplicationState());
+ r.getName()).append("/").append(r.getApplicationType()).append(separator);
+ Set<String> tags = r.getApplicationTags();
+ if (!tags.isEmpty()) {
+ for (String tag : tags) {
+ builder.append(tag).append(separator);
+ }
+ }
+ builder.append("state: ").append(r.getYarnApplicationState());
builder.append(separator).append("URL: ").append(r.getTrackingUrl());
builder.append(separator).append("Started ").append(new Date(r.getStartTime()).toGMTString());
long finishTime = r.getFinishTime();
Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java?rev=1595332&r1=1595331&r2=1595332&view=diff
==============================================================================
--- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java (original)
+++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java Fri May 16 20:10:20 2014
@@ -36,6 +36,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Map;
+import java.util.Set;
public class AppMasterLauncher extends AbstractLauncher {
@@ -73,7 +74,8 @@ public class AppMasterLauncher extends A
CoreFileSystem fs,
SliderYarnClientImpl yarnClient,
boolean secureCluster,
- Map<String, String> options
+ Map<String, String> options,
+ Set<String> applicationTags
) throws IOException, YarnException {
super(conf, fs);
this.yarnClient = yarnClient;
@@ -88,6 +90,9 @@ public class AppMasterLauncher extends A
submissionContext.setApplicationName(name);
// app type used in service enum;
submissionContext.setApplicationType(type);
+ if (!applicationTags.isEmpty()) {
+ submissionContext.setApplicationTags(applicationTags);
+ }
extractResourceRequirements(resource, options);
}
Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java?rev=1595332&r1=1595331&r2=1595332&view=diff
==============================================================================
--- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java (original)
+++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java Fri May 16 20:10:20 2014
@@ -32,7 +32,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Collections;
import java.util.List;
+import java.util.Set;
import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES;
import static org.apache.slider.api.ResourceKeys.DEF_YARN_CORES;
@@ -207,4 +209,13 @@ public abstract class AbstractClientProv
validateInstanceDefinition(instanceDefinition);
}
+ /**
+ * Return a set of application specific string tags.
+ * @return the set of tags.
+ */
+ public Set<String> getApplicationTags (SliderFileSystem fileSystem,
+ String appDef) throws SliderException {
+ return Collections.emptySet();
+ }
+
}
Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java?rev=1595332&r1=1595331&r2=1595332&view=diff
==============================================================================
--- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java (original)
+++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java Fri May 16 20:10:20 2014
@@ -34,11 +34,14 @@ import org.apache.slider.core.launch.Abs
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
+import org.apache.slider.providers.agent.application.metadata.Metainfo;
+import org.apache.slider.providers.agent.application.metadata.Service;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -194,4 +197,23 @@ public class AgentClientProvider extends
IOException,
SliderException {
}
+
+ @Override
+ public Set<String> getApplicationTags(SliderFileSystem fileSystem,
+ String appDef) throws SliderException {
+ Set<String> tags;
+ try {
+ Metainfo metainfo = AgentUtils.getApplicationMetainfo(fileSystem, appDef);
+ Service service = metainfo.getServices().get(0);
+ tags = new HashSet<>();
+ tags.add("Name: " + service.getName());
+ tags.add("Version: " + service.getVersion());
+ tags.add("Description: " + service.getComment());
+ } catch (IOException e) {
+ log.error("error retrieving metainfo from {}", appDef, e);
+ throw new SliderException("error retrieving metainfo", e);
+ }
+
+ return tags;
+ }
}
Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java?rev=1595332&r1=1595331&r2=1595332&view=diff
==============================================================================
--- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java (original)
+++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java Fri May 16 20:10:20 2014
@@ -242,17 +242,7 @@ public class AgentProviderService extend
protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
String appDef) throws IOException {
- log.info("Reading metainfo at {}", appDef);
- InputStream metainfoStream = SliderUtils.getApplicationResourceInputStream(
- fileSystem.getFileSystem(), new Path(appDef), "metainfo.xml");
- if (metainfoStream == null) {
- log.error("metainfo.xml is unavailable at {}.", appDef);
- throw new IOException("metainfo.xml is required in app package.");
- }
-
- Metainfo metainfo = new MetainfoParser().parse(metainfoStream);
-
- return metainfo;
+ return AgentUtils.getApplicationMetainfo(fileSystem, appDef);
}
protected void publishComponentConfiguration(String name, String description,
Added: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java?rev=1595332&view=auto
==============================================================================
--- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java (added)
+++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java Fri May 16 20:10:20 2014
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.providers.agent.application.metadata.Metainfo;
+import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ *
+ */
+public class AgentUtils {
+ private static final Logger log = LoggerFactory.getLogger(AgentUtils.class);
+
+ static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
+ String appDef) throws IOException {
+ log.info("Reading metainfo at {}", appDef);
+ InputStream metainfoStream = SliderUtils.getApplicationResourceInputStream(
+ fileSystem.getFileSystem(), new Path(appDef), "metainfo.xml");
+ if (metainfoStream == null) {
+ log.error("metainfo.xml is unavailable at {}.", appDef);
+ throw new IOException("metainfo.xml is required in app package.");
+ }
+
+ Metainfo metainfo = new MetainfoParser().parse(metainfoStream);
+
+ return metainfo;
+ }
+
+}
Modified: incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy?rev=1595332&r1=1595331&r2=1595332&view=diff
==============================================================================
--- incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy (original)
+++ incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy Fri May 16 20:10:20 2014
@@ -194,7 +194,7 @@ class TestStandaloneRegistryAM extends A
describe("Registry List")
- log.info(GET(registryURL, RestPaths.REGISTRY_SERVICE ))
+ log.info(GET(registryURL))
describe "Registry Retrieval Class"
Modified: incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy?rev=1595332&r1=1595331&r2=1595332&view=diff
==============================================================================
--- incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy (original)
+++ incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy Fri May 16 20:10:20 2014
@@ -116,18 +116,17 @@ class TestRegistryRestResources extends
//WS get
Client client = createTestClient();
- // test the exposed WADL link
WebResource webResource = client.resource(registry_url);
- ClientResponse response = webResource.type(MediaType.APPLICATION_XML)
- .get(ClientResponse.class);
- assert response.status == 200
- assert response.type == (new MediaType("application", WADL))
+ List<String> serviceList = webResource.type(MediaType.APPLICATION_JSON)
+ .get(List.class);
+ log.info("service list: {}", serviceList)
+ assert serviceList.contains(SliderKeys.APP_TYPE)
// test the available GET URIs
webResource = client.resource(
appendToURL(registry_url, RestPaths.REGISTRY_SERVICE));
-
- response = webResource.type(MediaType.APPLICATION_JSON)
+
+ ClientResponse response = webResource.type(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
def responseStr = response.getEntity(String.class)
log.info("response is " + responseStr)
Added: incubator/slider/trunk/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java?rev=1595332&view=auto
==============================================================================
--- incubator/slider/trunk/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java (added)
+++ incubator/slider/trunk/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java Fri May 16 20:10:20 2014
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.junit.Test;
+
+import java.io.InputStream;
+import java.util.Set;
+
+/**
+ *
+ */
+public class TestAgentClientProvider {
+ @Test
+ public void testGetApplicationTags () throws Exception {
+ Configuration configuration = new Configuration();
+ FileSystem fs = FileSystem.getLocal(configuration);
+ SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
+
+ AgentClientProvider provider = new AgentClientProvider(null);
+ Set<String> tags = provider.getApplicationTags(sliderFileSystem,
+ "target/test-classes/org/apache/slider/common/tools/test.zip");
+ assert tags != null;
+ assert !tags.isEmpty();
+ assert tags.contains("Name: STORM");
+ assert tags.contains("Description: Apache Hadoop Stream processing framework");
+ assert tags.contains("Version: 0.9.1.2.1");
+
+ }
+}