You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/09/03 22:47:02 UTC
[1/5] git commit: SLIDER-390 interpret short lived threshold as
seconds instead of milliseconds
Repository: incubator-slider
Updated Branches:
refs/heads/feature/auto_generate_storm_pkg 201686e2e -> 26b4153eb
SLIDER-390 interpret short lived threshold as seconds instead of milliseconds
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b0b7bd38
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b0b7bd38
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b0b7bd38
Branch: refs/heads/feature/auto_generate_storm_pkg
Commit: b0b7bd38542ed287415066d0215c6fda8786336e
Parents: 201686e
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Sep 3 08:38:52 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Sep 3 08:38:52 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/slider/server/appmaster/state/AppState.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b0b7bd38/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 8b70185..dce48d1 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1227,7 +1227,7 @@ public class AppState {
/**
* Is a role short lived by the threshold set for this application
* @param instance instance
- * @return true if the instance is considered short live
+ * @return true if the instance is considered short lived
*/
@VisibleForTesting
public boolean isShortLived(RoleInstance instance) {
@@ -1236,7 +1236,7 @@ public class AppState {
boolean shortlived;
if (started > 0) {
long duration = time - started;
- shortlived = duration < startTimeThreshold;
+ shortlived = duration < (startTimeThreshold * 1000);
} else {
// never even saw a start event
shortlived = true;
[2/5] git commit: SLIDER-385 move shortlife property to internal
Posted by sm...@apache.org.
SLIDER-385 move shortlife property to internal
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e9a353c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e9a353c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e9a353c1
Branch: refs/heads/feature/auto_generate_storm_pkg
Commit: e9a353c1ab201f1481e224b1e2b3e03188b9b2dc
Parents: b0b7bd3
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Sep 3 09:19:21 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Sep 3 09:19:21 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/slider/api/InternalKeys.java | 11 +++++++++++
.../main/java/org/apache/slider/api/ResourceKeys.java | 13 -------------
.../apache/slider/server/appmaster/state/AppState.java | 5 +++--
3 files changed, 14 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e9a353c1/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index ad384e2..b58f1c6 100644
--- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -75,6 +75,17 @@ public interface InternalKeys {
*/
int DEFAULT_INTERNAL_CONTAINER_STARTUP_DELAY = 5000;
/**
+ * Time in seconds before a container is considered long-lived.
+ * Shortlived containers are interpreted as a problem with the role
+ * and/or the host: {@value}
+ */
+ String INTERNAL_CONTAINER_FAILURE_SHORTLIFE =
+ "internal.container.failure.shortlife";
+ /**
+ * Default short life threshold: {@value}
+ */
+ int DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE = 60;
+ /**
* Version of the app: {@value}
*/
String KEYTAB_LOCATION = "internal.keytab.location";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e9a353c1/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index 3d54140..b542f1a 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -92,19 +92,6 @@ public interface ResourceKeys {
/**
- * Time in seconds before a container is considered long-lived.
- * Shortlived containers are interpreted as a problem with the role
- * and/or the host: {@value}
- */
- String CONTAINER_FAILURE_SHORTLIFE =
- "container.failure.shortlife";
-
- /**
- * Default short life threshold: {@value}
- */
- int DEFAULT_CONTAINER_FAILURE_SHORTLIFE = 60;
-
- /**
* maximum number of failed containers (in a single role)
* before the cluster is deemed to have failed {@value}
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e9a353c1/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index dce48d1..3e6bb74 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -36,6 +36,7 @@ import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterDescriptionOperations;
import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.StatusKeys;
@@ -520,8 +521,8 @@ public class AppState {
instanceDefinition.getResourceOperations().getGlobalOptions();
startTimeThreshold = globalResOpts.getOptionInt(
- ResourceKeys.CONTAINER_FAILURE_SHORTLIFE,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_SHORTLIFE);
+ InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
+ InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE);
failureThreshold = globalResOpts.getOptionInt(
ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
[5/5] git commit: Merge branch 'develop' into
feature/auto_generate_storm_pkg
Posted by sm...@apache.org.
Merge branch 'develop' into feature/auto_generate_storm_pkg
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/26b4153e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/26b4153e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/26b4153e
Branch: refs/heads/feature/auto_generate_storm_pkg
Commit: 26b4153ebf070521fb5030d57846f4ccd7f45cb0
Parents: e1129b0 38c816e
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Sep 3 13:46:44 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Sep 3 13:46:44 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/slider/api/InternalKeys.java | 11 +++++++++++
.../main/java/org/apache/slider/api/ResourceKeys.java | 13 -------------
.../apache/slider/server/appmaster/state/AppState.java | 9 +++++----
3 files changed, 16 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[3/5] git commit: SLIDER-181. Storm package updates
Posted by sm...@apache.org.
SLIDER-181. Storm package updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e1129b06
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e1129b06
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e1129b06
Branch: refs/heads/feature/auto_generate_storm_pkg
Commit: e1129b0609621356b47324dcbcabd612fd275be5
Parents: 201686e
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Sep 3 13:15:20 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Sep 3 13:15:20 2014 -0700
----------------------------------------------------------------------
app-packages/storm/README.txt | 19 ++---
app-packages/storm/appConfig.json | 10 +--
app-packages/storm/metainfo.xml | 2 +-
...pache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE | 16 ----
app-packages/storm/pom.xml | 90 ++++++++++++++++++++
app-packages/storm/src/assembly/storm.xml | 68 +++++++++++++++
6 files changed, 172 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt
index d4df85f..49adfce 100644
--- a/app-packages/storm/README.txt
+++ b/app-packages/storm/README.txt
@@ -17,19 +17,16 @@
How to create a Slider app package for Storm?
-To create the app package you will need the Storm tarball copied to a specific location.
-Various configurations provided in this sample are customized for apache-storm-0.9.3.2.2.0.0-578.tar.gz.
-So if you use a different version you may need to edit a few config values.
+To create the app package you will need the Storm tarball and invoke mvn command
+with appropriate parameters.
-Replace the placeholder tarball for Storm.
- cp ~/Downloads/apache-storm-0.9.3.2.2.0.0-578.tar.gz package/files/
- rm package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
+Command:
+mvn clean package -Pstorm-app-package -Dpkg.version=<version>
+ -Dpkg.name=<file name of app tarball> -Dpkg.src=<folder location where the pkg is available>
-Create a zip package at the root of the package (<slider enlistment>/app-packages/storm/)
- zip -r apache-storm-0.9.3.zip .
-
-Verify the content using
- unzip -l "$@" apache-storm-0.9.3.zip
+Example:
+mvn clean package -Pstorm-app-package -Dpkg.version=0.9.3.2.2.0.0-578
+ -Dpkg.name=apache-storm-0.9.3.2.2.0.0-578.tar.gz -Dpkg.src=/Users/user1/Downloads
While appConfig.json and resources.json are not required for the package they work
well as the default configuration for Slider apps. So its advisable that when you
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index b5fe436..df28e8f 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -3,13 +3,13 @@
"metadata": {
},
"global": {
- "application.def": "apache-storm-0.9.3.zip",
+ "application.def": "slider-storm-app-package-${pkg.version}.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"create.default.zookeeper.node": "true",
"system_configs": "core-site",
"site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
"site.global.user_group": "hadoop",
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_id": "Application2",
@@ -22,8 +22,8 @@
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
"site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
@@ -31,7 +31,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index 43535f4..bfd84e8 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -130,7 +130,7 @@
<packages>
<package>
<type>tarball</type>
- <name>files/apache-storm-0.9.3.2.2.0.0-578.tar.gz</name>
+ <name>files/${pkg.name}</name>
</package>
</packages>
</osSpecific>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE b/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
deleted file mode 100644
index dd934d5..0000000
--- a/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-Replace with the actual storm package.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/pom.xml b/app-packages/storm/pom.xml
new file mode 100644
index 0000000..c08ed3d
--- /dev/null
+++ b/app-packages/storm/pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.51.0-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-storm-app-package</artifactId>
+ <packaging>pom</packaging>
+ <name>Slider Storm App Package</name>
+ <description>Slider Storm App Package</description>
+ <version>${pkg.version}</version>
+ <properties>
+ <work.dir>package-tmp</work.dir>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>storm-app-package</id>
+ <build>
+ <plugins>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>validate</phase>
+ <configuration>
+ <target name="copy and rename file">
+ <copy file="${pkg.src}/${pkg.name}" tofile="${project.build.directory}/${pkg.name}" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptor>src/assembly/storm.xml</descriptor>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <build>
+ </build>
+
+ <dependencies>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/src/assembly/storm.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/src/assembly/storm.xml b/app-packages/storm/src/assembly/storm.xml
new file mode 100644
index 0000000..47d223a
--- /dev/null
+++ b/app-packages/storm/src/assembly/storm.xml
@@ -0,0 +1,68 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>slider-storm-v${storm.version}</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <files>
+ <file>
+ <source>appConfig.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>metainfo.xml</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>${pkg.src}/${pkg.name}</source>
+ <outputDirectory>package/files</outputDirectory>
+ <filtered>false</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ </files>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <outputDirectory>/</outputDirectory>
+ <excludes>
+ <exclude>pom.xml</exclude>
+ <exclude>src/**</exclude>
+ <exclude>target/**</exclude>
+ <exclude>appConfig.json</exclude>
+ <exclude>metainfo.xml</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
+ </fileSets>
+</assembly>
[4/5] git commit: SLIDER-181. Storm package updates
Posted by sm...@apache.org.
SLIDER-181. Storm package updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/38c816eb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/38c816eb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/38c816eb
Branch: refs/heads/feature/auto_generate_storm_pkg
Commit: 38c816ebbd3c883308bbe36fc00846d4ff4dac7b
Parents: e9a353c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Sep 3 13:15:20 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Sep 3 13:46:02 2014 -0700
----------------------------------------------------------------------
app-packages/storm/README.txt | 19 ++---
app-packages/storm/appConfig.json | 10 +--
app-packages/storm/metainfo.xml | 2 +-
...pache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE | 16 ----
app-packages/storm/pom.xml | 90 ++++++++++++++++++++
app-packages/storm/src/assembly/storm.xml | 68 +++++++++++++++
6 files changed, 172 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt
index d4df85f..49adfce 100644
--- a/app-packages/storm/README.txt
+++ b/app-packages/storm/README.txt
@@ -17,19 +17,16 @@
How to create a Slider app package for Storm?
-To create the app package you will need the Storm tarball copied to a specific location.
-Various configurations provided in this sample are customized for apache-storm-0.9.3.2.2.0.0-578.tar.gz.
-So if you use a different version you may need to edit a few config values.
+To create the app package you will need the Storm tarball and invoke mvn command
+with appropriate parameters.
-Replace the placeholder tarball for Storm.
- cp ~/Downloads/apache-storm-0.9.3.2.2.0.0-578.tar.gz package/files/
- rm package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
+Command:
+mvn clean package -Pstorm-app-package -Dpkg.version=<version>
+ -Dpkg.name=<file name of app tarball> -Dpkg.src=<folder location where the pkg is available>
-Create a zip package at the root of the package (<slider enlistment>/app-packages/storm/)
- zip -r apache-storm-0.9.3.zip .
-
-Verify the content using
- unzip -l "$@" apache-storm-0.9.3.zip
+Example:
+mvn clean package -Pstorm-app-package -Dpkg.version=0.9.3.2.2.0.0-578
+ -Dpkg.name=apache-storm-0.9.3.2.2.0.0-578.tar.gz -Dpkg.src=/Users/user1/Downloads
While appConfig.json and resources.json are not required for the package they work
well as the default configuration for Slider apps. So its advisable that when you
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index b5fe436..df28e8f 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -3,13 +3,13 @@
"metadata": {
},
"global": {
- "application.def": "apache-storm-0.9.3.zip",
+ "application.def": "slider-storm-app-package-${pkg.version}.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"create.default.zookeeper.node": "true",
"system_configs": "core-site",
"site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
"site.global.user_group": "hadoop",
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_id": "Application2",
@@ -22,8 +22,8 @@
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
"site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
@@ -31,7 +31,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index 43535f4..bfd84e8 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -130,7 +130,7 @@
<packages>
<package>
<type>tarball</type>
- <name>files/apache-storm-0.9.3.2.2.0.0-578.tar.gz</name>
+ <name>files/${pkg.name}</name>
</package>
</packages>
</osSpecific>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE b/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
deleted file mode 100644
index dd934d5..0000000
--- a/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-Replace with the actual storm package.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/pom.xml b/app-packages/storm/pom.xml
new file mode 100644
index 0000000..c08ed3d
--- /dev/null
+++ b/app-packages/storm/pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.51.0-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-storm-app-package</artifactId>
+ <packaging>pom</packaging>
+ <name>Slider Storm App Package</name>
+ <description>Slider Storm App Package</description>
+ <version>${pkg.version}</version>
+ <properties>
+ <work.dir>package-tmp</work.dir>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>storm-app-package</id>
+ <build>
+ <plugins>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>validate</phase>
+ <configuration>
+ <target name="copy and rename file">
+ <copy file="${pkg.src}/${pkg.name}" tofile="${project.build.directory}/${pkg.name}" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptor>src/assembly/storm.xml</descriptor>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <build>
+ </build>
+
+ <dependencies>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/src/assembly/storm.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/src/assembly/storm.xml b/app-packages/storm/src/assembly/storm.xml
new file mode 100644
index 0000000..47d223a
--- /dev/null
+++ b/app-packages/storm/src/assembly/storm.xml
@@ -0,0 +1,68 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>slider-storm-v${storm.version}</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <files>
+ <file>
+ <source>appConfig.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>metainfo.xml</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>${pkg.src}/${pkg.name}</source>
+ <outputDirectory>package/files</outputDirectory>
+ <filtered>false</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ </files>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <outputDirectory>/</outputDirectory>
+ <excludes>
+ <exclude>pom.xml</exclude>
+ <exclude>src/**</exclude>
+ <exclude>target/**</exclude>
+ <exclude>appConfig.json</exclude>
+ <exclude>metainfo.xml</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
+ </fileSets>
+</assembly>