You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ma...@apache.org on 2014/01/21 21:31:04 UTC
[1/3] git commit: BIGTOP-1181. Add pyspark to spark package
Updated Branches:
refs/heads/master 83906bb8e -> d2112eaf1
BIGTOP-1181. Add pyspark to spark package
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/b1425046
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/b1425046
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/b1425046
Branch: refs/heads/master
Commit: b1425046e452f6cc176e36fc89443b19e87e69fc
Parents: 83906bb
Author: Sean Mackrory <ma...@apache.org>
Authored: Tue Jan 14 13:34:40 2014 -0800
Committer: Sean Mackrory <ma...@apache.org>
Committed: Tue Jan 21 12:03:30 2014 -0800
----------------------------------------------------------------------
.../src/common/spark/install_spark.sh | 21 +++++++++++++++++
bigtop-packages/src/deb/spark/control | 6 +++++
bigtop-packages/src/deb/spark/rules | 2 +-
.../src/deb/spark/spark-core.install | 18 +++++++++++++++
.../src/deb/spark/spark-python.install | 4 ++++
bigtop-packages/src/rpm/spark/SPECS/spark.spec | 24 ++++++++++++++++++--
.../package/src/main/resources/package_data.xml | 10 ++++++++
7 files changed, 82 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b1425046/bigtop-packages/src/common/spark/install_spark.sh
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/spark/install_spark.sh b/bigtop-packages/src/common/spark/install_spark.sh
index 75dd337..664c271 100644
--- a/bigtop-packages/src/common/spark/install_spark.sh
+++ b/bigtop-packages/src/common/spark/install_spark.sh
@@ -31,6 +31,7 @@ usage: $0 <options>
--installed-lib-dir=DIR path where lib-dir will end up on target system
--bin-dir=DIR path to install bins [/usr/bin]
--examples-dir=DIR path to install examples [doc-dir/examples]
+ --pyspark-python executable to use for Python interpreter [python]
... [ see source for more similar options ]
"
exit 1
@@ -46,6 +47,7 @@ OPTS=$(getopt \
-l 'bin-dir:' \
-l 'source-dir:' \
-l 'examples-dir:' \
+ -l 'pyspark-python:' \
-l 'build-dir:' -- "$@")
if [ $? != 0 ] ; then
@@ -79,6 +81,9 @@ while true ; do
--examples-dir)
EXAMPLES_DIR=$2 ; shift 2
;;
+ --pyspark-python)
+ PYSPARK_PYTHON=$2 ; shift 2
+ ;;
--)
shift ; break
;;
@@ -114,6 +119,7 @@ EXAMPLES_DIR=${EXAMPLES_DIR:-$DOC_DIR/examples}
BIN_DIR=${BIN_DIR:-/usr/bin}
CONF_DIR=${CONF_DIR:-/etc/spark/conf.dist}
SCALA_HOME=${SCALA_HOME:-/usr/share/scala}
+PYSPARK_PYTHON=${PYSPARK_PYTHON:-python}
install -d -m 0755 $PREFIX/$LIB_DIR
install -d -m 0755 $PREFIX/$LIB_DIR/lib
@@ -190,3 +196,18 @@ export STANDALONE_SPARK_MASTER_HOST=\`hostname\`
EOF
ln -s /var/run/spark/work $PREFIX/$LIB_DIR/work
+
+cp -r ${BUILD_DIR}/python ${PREFIX}/${INSTALLED_LIB_DIR}/
+cp ${BUILD_DIR}/pyspark ${PREFIX}/${INSTALLED_LIB_DIR}/
+cat > $PREFIX/$BIN_DIR/pyspark <<EOF
+#!/bin/bash
+
+# Autodetect JAVA_HOME if not defined
+. /usr/lib/bigtop-utils/bigtop-detect-javahome
+
+export PYSPARK_PYTHON=${PYSPARK_PYTHON}
+
+exec $INSTALLED_LIB_DIR/pyspark "\$@"
+EOF
+chmod 755 $PREFIX/$BIN_DIR/pyspark
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b1425046/bigtop-packages/src/deb/spark/control
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/control b/bigtop-packages/src/deb/spark/control
index fc61489..14e3440 100644
--- a/bigtop-packages/src/deb/spark/control
+++ b/bigtop-packages/src/deb/spark/control
@@ -42,3 +42,9 @@ Architecture: all
Depends: spark-core (= ${source:Version})
Description: Server for Spark worker
Server for Spark worker
+
+Package: spark-python
+Architecture: all
+Depends: spark (= ${source:Version}), python
+Description: Python client for Spark
+ Includes PySpark, an interactive Python shell for Spark, and related libraries
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b1425046/bigtop-packages/src/deb/spark/rules
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/rules b/bigtop-packages/src/deb/spark/rules
index bd0ad6b..f559f49 100644
--- a/bigtop-packages/src/deb/spark/rules
+++ b/bigtop-packages/src/deb/spark/rules
@@ -40,4 +40,4 @@ override_dh_auto_install: $(svcs)
--build-dir=`pwd` \
--doc-dir=/usr/share/doc/spark \
--source-dir=debian \
- --prefix=debian/spark-core
+ --prefix=debian/tmp
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b1425046/bigtop-packages/src/deb/spark/spark-core.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/spark-core.install b/bigtop-packages/src/deb/spark/spark-core.install
new file mode 100644
index 0000000..0c5a6f5
--- /dev/null
+++ b/bigtop-packages/src/deb/spark/spark-core.install
@@ -0,0 +1,18 @@
+/etc/spark
+/usr/bin/spark-executor
+/usr/bin/spark-shell
+/usr/lib/spark/bagel
+/usr/lib/spark/bin
+/usr/lib/spark/core
+/usr/lib/spark/examples
+/usr/lib/spark/lib
+/usr/lib/spark/mllib
+/usr/lib/spark/repl
+/usr/lib/spark/streaming
+/usr/lib/spark/ui-resources
+/usr/lib/spark/conf
+/usr/lib/spark/RELEASE
+/usr/lib/spark/spark-class
+/usr/lib/spark/spark-executor
+/usr/lib/spark/spark-shell
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b1425046/bigtop-packages/src/deb/spark/spark-python.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/spark-python.install b/bigtop-packages/src/deb/spark/spark-python.install
new file mode 100644
index 0000000..b5461bb
--- /dev/null
+++ b/bigtop-packages/src/deb/spark/spark-python.install
@@ -0,0 +1,4 @@
+/usr/bin/pyspark
+/usr/lib/spark/pyspark
+/usr/lib/spark/python
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b1425046/bigtop-packages/src/rpm/spark/SPECS/spark.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
index a8db290..c78fa97 100644
--- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec
+++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
@@ -33,6 +33,9 @@
%define alternatives_cmd alternatives
%endif
+%define pyspark_python python
+
+
# disable repacking jars
%define __os_install_post %{nil}
@@ -90,7 +93,15 @@ Requires: spark-core = %{version}-%{release}
%description worker
Server for Spark worker
-
+
+%package python
+Summary: Python client for Spark
+Group: Development/Libraries
+Requires: spark = %{version}-%{release}, %{pyspark_python}
+
+%description python
+Includes PySpark, an interactive Python shell for Spark, and related libraries
+
%prep
%setup -n %{spark_name}-%{spark_base_version}
@@ -110,7 +121,8 @@ sh $RPM_SOURCE_DIR/install_spark.sh \
--build-dir=`pwd` \
--source-dir=$RPM_SOURCE_DIR \
--prefix=$RPM_BUILD_ROOT \
- --doc-dir=%{doc_spark}
+ --doc-dir=%{doc_spark} \
+ --pyspark-python=%{pyspark_python}
for service in %{spark_services}
do
@@ -146,6 +158,8 @@ done
%config(noreplace) %{config_spark}.dist
%doc %{doc_spark}
%{lib_spark}
+%exclude %{lib_spark}/pyspark
+%exclude %{lib_spark}/python
%{etc_spark}
%attr(0755,spark,spark) %{var_lib_spark}
%attr(0755,spark,spark) %{var_run_spark}
@@ -154,6 +168,12 @@ done
%{bin}/spark-shell
%{bin}/spark-executor
+%files python
+%defattr(-,root,root,755)
+%attr(0755,root,root) %{bin}/pyspark
+%attr(0755,root,root) %{lib_spark}/pyspark
+%{lib_spark}/python
+
%define service_macro() \
%files %1 \
%attr(0755,root,root)/%{initd_dir}/%{spark_name}-%1 \
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b1425046/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml b/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
index 166323c..766dfd2 100644
--- a/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
+++ b/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
@@ -132,6 +132,16 @@ Spark runs on top of the Apache Mesos cluster manager.</description>
<spark>/self</spark>
</deps>
</spark-worker>
+ <spark-python>
+ <metadata>
+ <summary>Python client for Spark</summary>
+ <description>Includes PySpark, an interactive Python shell for Spark, and related libraries</description>
+ <url>http://spark.incubator.apache.org/</url>
+ </metadata>
+ <deps>
+ <spark>/self</spark>
+ </deps>
+ </spark-python>
<whirr>
<metadata>
<summary>Scripts and libraries for running software services on cloud infrastructure.</summary>
[3/3] git commit: BIGTOP-1187. Spark service packages have extraneous
core in the name
Posted by ma...@apache.org.
BIGTOP-1187. Spark service packages have extraneous core in the name
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/d2112eaf
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/d2112eaf
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/d2112eaf
Branch: refs/heads/master
Commit: d2112eaf146350b00da6ec00bdcad03636a13ef2
Parents: f921de4
Author: Sean Mackrory <ma...@apache.org>
Authored: Mon Jan 20 15:31:25 2014 -0800
Committer: Sean Mackrory <ma...@apache.org>
Committed: Tue Jan 21 12:03:48 2014 -0800
----------------------------------------------------------------------
bigtop-packages/src/rpm/spark/SPECS/spark.spec | 38 ++++++++++----------
.../package/src/main/resources/package_data.xml | 4 +--
2 files changed, 21 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/d2112eaf/bigtop-packages/src/rpm/spark/SPECS/spark.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
index c646bad..55aa469 100644
--- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec
+++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
@@ -77,28 +77,28 @@ written in Scala, a high-level language for the JVM, and exposes a clean
language-integrated syntax that makes it easy to write parallel jobs.
Spark runs on top of the Apache Mesos cluster manager.
-%package master
+%package -n spark-master
Summary: Server for Spark master
Group: Development/Libraries
Requires: spark-core = %{version}-%{release}
-%description master
+%description -n spark-master
Server for Spark master
-%package worker
+%package -n spark-worker
Summary: Server for Spark worker
Group: Development/Libraries
Requires: spark-core = %{version}-%{release}
-%description worker
+%description -n spark-worker
Server for Spark worker
-%package python
+%package -n spark-python
Summary: Python client for Spark
Group: Development/Libraries
Requires: spark = %{version}-%{release}, %{pyspark_python}
-%description python
+%description -n spark-python
Includes PySpark, an interactive Python shell for Spark, and related libraries
%prep
@@ -147,7 +147,7 @@ done
#######################
#### FILES SECTION ####
#######################
-%files
+%files
%defattr(-,root,root,755)
%config(noreplace) %{config_spark}.dist
%doc %{doc_spark}
@@ -162,26 +162,26 @@ done
%{bin}/spark-shell
%{bin}/spark-executor
-%files python
+%files -n spark-python
%defattr(-,root,root,755)
%attr(0755,root,root) %{bin}/pyspark
%attr(0755,root,root) %{lib_spark}/pyspark
%{lib_spark}/python
%define service_macro() \
-%files %1 \
-%attr(0755,root,root)/%{initd_dir}/%{spark_name}-%1 \
-%post %1 \
-chkconfig --add %{spark_name}-%1 \
+%files -n %1 \
+%attr(0755,root,root)/%{initd_dir}/%1 \
+%post -n %1 \
+chkconfig --add %1 \
\
-%preun %1 \
+%preun -n %1 \
if [ $1 = 0 ] ; then \
- service %{spark_name}-%1 stop > /dev/null 2>&1 \
- chkconfig --del %{spark_name}-%1 \
+ service %1 stop > /dev/null 2>&1 \
+ chkconfig --del %1 \
fi \
-%postun %1 \
+%postun -n %1 \
if [ $1 -ge 1 ]; then \
- service %{spark_name}-%1 condrestart >/dev/null 2>&1 \
+ service %1 condrestart >/dev/null 2>&1 \
fi
-%service_macro master
-%service_macro worker
+%service_macro spark-master
+%service_macro spark-worker
http://git-wip-us.apache.org/repos/asf/bigtop/blob/d2112eaf/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml b/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
index 766dfd2..3eb24f7 100644
--- a/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
+++ b/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
@@ -90,7 +90,7 @@ Java Servlet and JavaServer Pages technologies.</description>
</giraph-conf>
</alternatives>
</giraph>
- <spark>
+ <spark-core>
<metadata>
<summary>Lightning-Fast Cluster Computing</summary>
<description>Spark is a MapReduce-like cluster computing framework designed to support
@@ -111,7 +111,7 @@ Spark runs on top of the Apache Mesos cluster manager.</description>
<alt>/etc/spark/conf.dist</alt>
</spark-conf>
</alternatives>
- </spark>
+ </spark-core>
<spark-master>
<metadata>
<summary>Server for Spark master</summary>
[2/3] git commit: BIGTOP-1185. Spark service won't start on
Debian-based platforms
Posted by ma...@apache.org.
BIGTOP-1185. Spark service won't start on Debian-based platforms
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/f921de44
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/f921de44
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/f921de44
Branch: refs/heads/master
Commit: f921de44527d2697a690b63cb329c1be70ce2440
Parents: b142504
Author: Sean Mackrory <ma...@apache.org>
Authored: Thu Jan 16 11:35:09 2014 -0800
Committer: Sean Mackrory <ma...@apache.org>
Committed: Tue Jan 21 12:03:39 2014 -0800
----------------------------------------------------------------------
.../src/common/spark/install_spark.sh | 5 ++
.../src/deb/spark/spark-core.install | 5 +-
.../src/deb/spark/spark-core.postinst | 1 +
.../src/deb/spark/spark-core.preinst | 61 ++++++++++++++++++++
bigtop-packages/src/rpm/spark/SPECS/spark.spec | 6 --
5 files changed, 71 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f921de44/bigtop-packages/src/common/spark/install_spark.sh
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/spark/install_spark.sh b/bigtop-packages/src/common/spark/install_spark.sh
index 664c271..d29dfd8 100644
--- a/bigtop-packages/src/common/spark/install_spark.sh
+++ b/bigtop-packages/src/common/spark/install_spark.sh
@@ -126,6 +126,11 @@ install -d -m 0755 $PREFIX/$LIB_DIR/lib
install -d -m 0755 $PREFIX/$SPARK_BIN_DIR
install -d -m 0755 $PREFIX/$DOC_DIR
+install -d -m 0755 $PREFIX/var/lib/spark/
+install -d -m 0755 $PREFIX/var/log/spark/
+install -d -m 0755 $PREFIX/var/run/spark/
+install -d -m 0755 $PREFIX/var/run/spark/work/
+
tar --wildcards -C $PREFIX/$LIB_DIR -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz 'lib/*'
for comp in core repl bagel mllib streaming; do
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f921de44/bigtop-packages/src/deb/spark/spark-core.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/spark-core.install b/bigtop-packages/src/deb/spark/spark-core.install
index 0c5a6f5..a48a222 100644
--- a/bigtop-packages/src/deb/spark/spark-core.install
+++ b/bigtop-packages/src/deb/spark/spark-core.install
@@ -15,4 +15,7 @@
/usr/lib/spark/spark-class
/usr/lib/spark/spark-executor
/usr/lib/spark/spark-shell
-
+/var/lib/spark/
+/var/log/spark/
+/var/run/spark/
+/var/run/spark/work/
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f921de44/bigtop-packages/src/deb/spark/spark-core.postinst
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/spark-core.postinst b/bigtop-packages/src/deb/spark/spark-core.postinst
index 6133296..f6cca0a 100644
--- a/bigtop-packages/src/deb/spark/spark-core.postinst
+++ b/bigtop-packages/src/deb/spark/spark-core.postinst
@@ -20,6 +20,7 @@ case "$1" in
configure)
# Install config alternatives
update-alternatives --install /etc/spark/conf spark-conf /etc/spark/conf.dist 30
+ chown -R spark:spark /var/log/spark /var/lib/spark /var/run/spark
;;
abort-upgrade|abort-remove|abort-deconfigure)
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f921de44/bigtop-packages/src/deb/spark/spark-core.preinst
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/spark-core.preinst b/bigtop-packages/src/deb/spark/spark-core.preinst
new file mode 100644
index 0000000..c8950a4
--- /dev/null
+++ b/bigtop-packages/src/deb/spark/spark-core.preinst
@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# preinst script for spark
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+# * <new-preinst> `install'
+# * <new-preinst> `install' <old-version>
+# * <new-preinst> `upgrade' <old-version>
+# * <old-preinst> `abort-upgrade' <new-version>
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+
+case "$1" in
+ install|upgrade)
+ if ! getent passwd spark >/dev/null; then
+ # Adding system user: spark .
+ adduser \
+ --system \
+ --group \
+ --home /var/lib/spark \
+ --gecos "Spark" \
+ --shell /bin/false \
+ spark >/dev/null
+ fi
+ ;;
+
+ abort-upgrade)
+ ;;
+
+ *)
+ echo "preinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f921de44/bigtop-packages/src/rpm/spark/SPECS/spark.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
index c78fa97..c646bad 100644
--- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec
+++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
@@ -35,7 +35,6 @@
%define pyspark_python python
-
# disable repacking jars
%define __os_install_post %{nil}
@@ -110,11 +109,6 @@ bash $RPM_SOURCE_DIR/do-component-build
%install
%__rm -rf $RPM_BUILD_ROOT
-%__install -d -m 0755 $RPM_BUILD_ROOT/%{bin_spark}/
-%__install -d -m 0755 $RPM_BUILD_ROOT/%{_localstatedir}/lib/%{spark_name}/
-%__install -d -m 0755 $RPM_BUILD_ROOT/%{_localstatedir}/log/%{spark_name}/
-%__install -d -m 0755 $RPM_BUILD_ROOT/%{_localstatedir}/run/%{spark_name}/
-%__install -d -m 0755 $RPM_BUILD_ROOT/%{_localstatedir}/run/%{spark_name}/work/
%__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/
sh $RPM_SOURCE_DIR/install_spark.sh \