You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/01/15 19:01:07 UTC
[1/3] git commit: Add deb profile to assembly/pom.xml
Updated Branches:
refs/heads/master cef2af9c7 -> 494d3c077
Add deb profile to assembly/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/148757e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/148757e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/148757e8
Branch: refs/heads/master
Commit: 148757e88cc6475deb6dc6cd181532fffc5492f1
Parents: 74b46ac
Author: Mark Hamstra <ma...@gmail.com>
Authored: Tue Jan 14 21:23:09 2014 -0800
Committer: Mark Hamstra <ma...@gmail.com>
Committed: Tue Jan 14 22:05:42 2014 -0800
----------------------------------------------------------------------
assembly/pom.xml | 116 +++++++++++++++++++++++++++++++++-
assembly/src/deb/RELEASE | 2 +
assembly/src/deb/control/control | 8 +++
3 files changed, 125 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/148757e8/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 54a2591..dcd9601 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -30,6 +30,13 @@
<name>Spark Project Assembly</name>
<url>http://spark.incubator.apache.org/</url>
+ <properties>
+ <spark.jar>${project.build.directory}/scala-${scala.binary.version}/${project.artifactId}-${project.version}-hadoop${hadoop.version}.jar</spark.jar>
+ <deb.pkg.name>spark</deb.pkg.name>
+ <deb.install.path>/usr/share/spark</deb.install.path>
+ <deb.user>root</deb.user>
+ </properties>
+
<repositories>
<!-- A repository in the local filesystem for the Py4J JAR, which is not in Maven central -->
<repository>
@@ -79,7 +86,7 @@
<artifactId>maven-shade-plugin</artifactId>
<configuration>
<shadedArtifactAttached>false</shadedArtifactAttached>
- <outputFile>${project.build.directory}/scala-${scala.binary.version}/${project.artifactId}-${project.version}-hadoop${hadoop.version}.jar</outputFile>
+ <outputFile>${spark.jar}</outputFile>
<artifactSet>
<includes>
<include>*:*</include>
@@ -171,5 +178,112 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>deb</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>buildnumber-maven-plugin</artifactId>
+ <version>1.1</version>
+ <executions>
+ <execution>
+ <phase>validate</phase>
+ <goals>
+ <goal>create</goal>
+ </goals>
+ <configuration>
+ <shortRevisionLength>8</shortRevisionLength>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.vafer</groupId>
+ <artifactId>jdeb</artifactId>
+ <version>0.11</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>jdeb</goal>
+ </goals>
+ <configuration>
+ <deb>${project.build.directory}/${deb.pkg.name}_${project.version}-${buildNumber}_all.deb</deb>
+ <attach>false</attach>
+ <compression>gzip</compression>
+ <dataSet>
+ <data>
+ <src>${spark.jar}</src>
+ <type>file</type>
+ <mapper>
+ <type>perm</type>
+ <user>${deb.user}</user>
+ <group>${deb.user}</group>
+ <prefix>${deb.install.path}/jars</prefix>
+ </mapper>
+ </data>
+ <data>
+ <src>${basedir}/src/deb/RELEASE</src>
+ <type>file</type>
+ <mapper>
+ <type>perm</type>
+ <user>${deb.user}</user>
+ <group>${deb.user}</group>
+ <prefix>${deb.install.path}</prefix>
+ </mapper>
+ </data>
+ <data>
+ <src>${basedir}/../conf</src>
+ <type>directory</type>
+ <mapper>
+ <type>perm</type>
+ <user>${deb.user}</user>
+ <group>${deb.user}</group>
+ <prefix>${deb.install.path}/conf</prefix>
+ <filemode>744</filemode>
+ </mapper>
+ </data>
+ <data>
+ <src>${basedir}/../bin</src>
+ <type>directory</type>
+ <mapper>
+ <type>perm</type>
+ <user>${deb.user}</user>
+ <group>${deb.user}</group>
+ <prefix>${deb.install.path}/bin</prefix>
+ <filemode>744</filemode>
+ </mapper>
+ </data>
+ <data>
+ <src>${basedir}/../sbin</src>
+ <type>directory</type>
+ <mapper>
+ <type>perm</type>
+ <user>${deb.user}</user>
+ <group>${deb.user}</group>
+ <prefix>${deb.install.path}/sbin</prefix>
+ <filemode>744</filemode>
+ </mapper>
+ </data>
+ <data>
+ <src>${basedir}/../python</src>
+ <type>directory</type>
+ <mapper>
+ <type>perm</type>
+ <user>${deb.user}</user>
+ <group>${deb.user}</group>
+ <prefix>${deb.install.path}/python</prefix>
+ <filemode>744</filemode>
+ </mapper>
+ </data>
+ </dataSet>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/148757e8/assembly/src/deb/RELEASE
----------------------------------------------------------------------
diff --git a/assembly/src/deb/RELEASE b/assembly/src/deb/RELEASE
new file mode 100644
index 0000000..aad50ee
--- /dev/null
+++ b/assembly/src/deb/RELEASE
@@ -0,0 +1,2 @@
+compute-classpath.sh uses the existence of this file to decide whether to put the assembly jar on the
+classpath or instead to use classfiles in the source tree.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/148757e8/assembly/src/deb/control/control
----------------------------------------------------------------------
diff --git a/assembly/src/deb/control/control b/assembly/src/deb/control/control
new file mode 100644
index 0000000..a6b4471
--- /dev/null
+++ b/assembly/src/deb/control/control
@@ -0,0 +1,8 @@
+Package: [[deb.pkg.name]]
+Version: [[version]]-[[buildNumber]]
+Section: misc
+Priority: extra
+Architecture: all
+Maintainer: Matei Zaharia <ma...@gmail.com>
+Description: [[name]]
+Distribution: development
[3/3] git commit: Merge pull request #433 from markhamstra/debFix
Posted by pw...@apache.org.
Merge pull request #433 from markhamstra/debFix
Updated Debian packaging
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/494d3c07
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/494d3c07
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/494d3c07
Branch: refs/heads/master
Commit: 494d3c077496735e6ebca3217de4f0cc6b6419f2
Parents: cef2af9 147a943
Author: Patrick Wendell <pw...@gmail.com>
Authored: Wed Jan 15 10:00:50 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Wed Jan 15 10:00:50 2014 -0800
----------------------------------------------------------------------
assembly/pom.xml | 116 ++++++++++++++++++-
assembly/src/deb/RELEASE | 2 +
assembly/src/deb/control/control | 8 ++
docs/building-with-maven.md | 6 +-
pom.xml | 10 --
repl-bin/pom.xml | 184 -------------------------------
repl-bin/src/deb/bin/run | 57 ----------
repl-bin/src/deb/bin/spark-executor | 22 ----
repl-bin/src/deb/bin/spark-shell | 21 ----
repl-bin/src/deb/control/control | 8 --
10 files changed, 128 insertions(+), 306 deletions(-)
----------------------------------------------------------------------
[2/3] git commit: Removed repl-bin and updated maven build doc.
Posted by pw...@apache.org.
Removed repl-bin and updated maven build doc.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/147a943d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/147a943d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/147a943d
Branch: refs/heads/master
Commit: 147a943df0f4f0df0bef7c0d41c5f4cfd00e00c2
Parents: 148757e
Author: Mark Hamstra <ma...@gmail.com>
Authored: Tue Jan 14 21:36:58 2014 -0800
Committer: Mark Hamstra <ma...@gmail.com>
Committed: Tue Jan 14 22:17:24 2014 -0800
----------------------------------------------------------------------
docs/building-with-maven.md | 6 +-
pom.xml | 10 --
repl-bin/pom.xml | 184 -------------------------------
repl-bin/src/deb/bin/run | 57 ----------
repl-bin/src/deb/bin/spark-executor | 22 ----
repl-bin/src/deb/bin/spark-shell | 21 ----
repl-bin/src/deb/control/control | 8 --
7 files changed, 3 insertions(+), 305 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/147a943d/docs/building-with-maven.md
----------------------------------------------------------------------
diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md
index b9ff0af..6a9a8d6 100644
--- a/docs/building-with-maven.md
+++ b/docs/building-with-maven.md
@@ -71,8 +71,8 @@ This setup works fine in IntelliJ IDEA 11.1.4. After opening the project via the
## Building Spark Debian Packages ##
-It includes support for building a Debian package containing a 'fat-jar' which includes the repl, the examples and bagel. This can be created by specifying the following profiles:
+The maven build includes support for building a Debian package containing the assembly 'fat-jar', PySpark, and the necessary scripts and configuration files. This can be created by specifying the following:
- $ mvn -Prepl-bin -Pdeb clean package
+ $ mvn -Pdeb -DskipTests clean package
-The debian package can then be found under repl/target. We added the short commit hash to the file name so that we can distinguish individual packages build for SNAPSHOT versions.
+The debian package can then be found under assembly/target. We added the short commit hash to the file name so that we can distinguish individual packages built for SNAPSHOT versions.
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/147a943d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b25d9d7..54072b0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -754,15 +754,5 @@
</modules>
</profile>
-
- <profile>
- <id>repl-bin</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <modules>
- <module>repl-bin</module>
- </modules>
- </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/147a943d/repl-bin/pom.xml
----------------------------------------------------------------------
diff --git a/repl-bin/pom.xml b/repl-bin/pom.xml
deleted file mode 100644
index 869dbdb..0000000
--- a/repl-bin/pom.xml
+++ /dev/null
@@ -1,184 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-parent</artifactId>
- <version>0.9.0-incubating-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-repl-bin_2.10</artifactId>
- <packaging>pom</packaging>
- <name>Spark Project REPL binary packaging</name>
- <url>http://spark.incubator.apache.org/</url>
-
- <properties>
- <deb.pkg.name>spark</deb.pkg.name>
- <deb.install.path>/usr/share/spark</deb.install.path>
- <deb.user>root</deb.user>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-bagel_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <scope>runtime</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-repl_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <scope>runtime</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <configuration>
- <shadedArtifactAttached>false</shadedArtifactAttached>
- <outputFile>${project.build.directory}/${project.artifactId}-${project.version}-shaded.jar</outputFile>
- <artifactSet>
- <includes>
- <include>*:*</include>
- </includes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
- <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>reference.conf</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>spark.repl.Main</mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <profiles>
- <profile>
- <id>deb</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>buildnumber-maven-plugin</artifactId>
- <version>1.1</version>
- <executions>
- <execution>
- <phase>validate</phase>
- <goals>
- <goal>create</goal>
- </goals>
- <configuration>
- <shortRevisionLength>8</shortRevisionLength>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.vafer</groupId>
- <artifactId>jdeb</artifactId>
- <version>0.11</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>jdeb</goal>
- </goals>
- <configuration>
- <deb>${project.build.directory}/${deb.pkg.name}_${project.version}-${buildNumber}_all.deb</deb>
- <attach>false</attach>
- <compression>gzip</compression>
- <dataSet>
- <data>
- <src>${project.build.directory}/${project.artifactId}-${project.version}-shaded.jar</src>
- <type>file</type>
- <mapper>
- <type>perm</type>
- <user>${deb.user}</user>
- <group>${deb.user}</group>
- <prefix>${deb.install.path}</prefix>
- </mapper>
- </data>
- <data>
- <src>${basedir}/src/deb/bin</src>
- <type>directory</type>
- <mapper>
- <type>perm</type>
- <user>${deb.user}</user>
- <group>${deb.user}</group>
- <prefix>${deb.install.path}</prefix>
- <filemode>744</filemode>
- </mapper>
- </data>
- <data>
- <src>${basedir}/../conf</src>
- <type>directory</type>
- <mapper>
- <type>perm</type>
- <user>${deb.user}</user>
- <group>${deb.user}</group>
- <prefix>${deb.install.path}/conf</prefix>
- <filemode>744</filemode>
- </mapper>
- </data>
- </dataSet>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/147a943d/repl-bin/src/deb/bin/run
----------------------------------------------------------------------
diff --git a/repl-bin/src/deb/bin/run b/repl-bin/src/deb/bin/run
deleted file mode 100755
index 3a6f22f..0000000
--- a/repl-bin/src/deb/bin/run
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-SCALA_VERSION=2.10
-
-# Figure out where the Scala framework is installed
-FWDIR="$(cd `dirname $0`; pwd)"
-
-# Export this as SPARK_HOME
-export SPARK_HOME="$FWDIR"
-
-# Load environment variables from conf/spark-env.sh, if it exists
-if [ -e $FWDIR/conf/spark-env.sh ] ; then
- . $FWDIR/conf/spark-env.sh
-fi
-
-# Figure out how much memory to use per executor and set it as an environment
-# variable so that our process sees it and can report it to Mesos
-if [ -z "$SPARK_MEM" ] ; then
- SPARK_MEM="512m"
-fi
-export SPARK_MEM
-
-# Set JAVA_OPTS to be able to load native libraries and to set heap size
-JAVA_OPTS="$SPARK_JAVA_OPTS"
-JAVA_OPTS+=" -Djava.library.path=$SPARK_LIBRARY_PATH"
-JAVA_OPTS+=" -Xms$SPARK_MEM -Xmx$SPARK_MEM"
-# Load extra JAVA_OPTS from conf/java-opts, if it exists
-if [ -e $FWDIR/conf/java-opts ] ; then
- JAVA_OPTS+=" `cat $FWDIR/conf/java-opts`"
-fi
-export JAVA_OPTS
-
-# Build up classpath
-CLASSPATH=":$FWDIR/conf"
-for jar in `find $FWDIR -name '*jar'`; do
- CLASSPATH+=":$jar"
-done
-export CLASSPATH
-
-exec java -Dscala.usejavacp=true -Djline.shutdownhook=true -cp "$CLASSPATH" $JAVA_OPTS $EXTRA_ARGS "$@"
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/147a943d/repl-bin/src/deb/bin/spark-executor
----------------------------------------------------------------------
diff --git a/repl-bin/src/deb/bin/spark-executor b/repl-bin/src/deb/bin/spark-executor
deleted file mode 100755
index 052d76f..0000000
--- a/repl-bin/src/deb/bin/spark-executor
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-FWDIR="$(cd `dirname $0`; pwd)"
-echo "Running spark-executor with framework dir = $FWDIR"
-exec $FWDIR/run org.apache.spark.executor.MesosExecutorBackend
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/147a943d/repl-bin/src/deb/bin/spark-shell
----------------------------------------------------------------------
diff --git a/repl-bin/src/deb/bin/spark-shell b/repl-bin/src/deb/bin/spark-shell
deleted file mode 100755
index 118349d..0000000
--- a/repl-bin/src/deb/bin/spark-shell
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-FWDIR="$(cd `dirname $0`; pwd)"
-exec $FWDIR/run org.apache.spark.repl.Main "$@"
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/147a943d/repl-bin/src/deb/control/control
----------------------------------------------------------------------
diff --git a/repl-bin/src/deb/control/control b/repl-bin/src/deb/control/control
deleted file mode 100644
index a6b4471..0000000
--- a/repl-bin/src/deb/control/control
+++ /dev/null
@@ -1,8 +0,0 @@
-Package: [[deb.pkg.name]]
-Version: [[version]]-[[buildNumber]]
-Section: misc
-Priority: extra
-Architecture: all
-Maintainer: Matei Zaharia <ma...@gmail.com>
-Description: [[name]]
-Distribution: development