You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2013/03/28 22:14:38 UTC
[1/5] git commit: BIGTOP-832. Bump version of Hue to 2.2.0
Updated Branches:
refs/heads/master 569554fc5 -> c74fbe7c3
BIGTOP-832. Bump version of Hue to 2.2.0
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/9eefa559
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/9eefa559
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/9eefa559
Branch: refs/heads/master
Commit: 9eefa5598da3e38f7bb208e40680527326e4257e
Parents: 569554f
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Mon Mar 11 10:33:21 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Thu Mar 28 12:30:01 2013 -0700
----------------------------------------------------------------------
bigtop-deploy/puppet/manifests/cluster.pp | 6 +
bigtop-deploy/puppet/modules/hue/manifests/init.pp | 2 +-
bigtop-deploy/puppet/modules/hue/templates/hue.ini | 96 ++++++++++++---
bigtop-packages/src/common/hue/install_hue.sh | 14 +--
bigtop-packages/src/deb/hue/hue-common.install | 3 +-
bigtop-packages/src/rpm/hue/SPECS/hue.spec | 3 +-
bigtop.mk | 6 +-
7 files changed, 93 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/9eefa559/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index 6675863..041a37a 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -78,6 +78,9 @@ class hadoop_cluster_node {
$hadoop_oozie_url = "http://${hadoop_head_node}:11000/oozie"
$hadoop_httpfs_url = "http://${hadoop_head_node}:14000/webhdfs/v1"
+ $hadoop_rm_url = "http://${hadoop_head_node}:8088"
+ $hadoop_rm_proxy_url = "http://${hadoop_head_node}:8088"
+ $hadoop_history_server_url = "http://${hadoop_head_node}:19888"
$solrcloud_collections = ["collection1"]
$solrcloud_port = "1978"
@@ -211,6 +214,9 @@ class hadoop_head_node inherits hadoop_cluster_node {
}
hue::server { "hue server":
+ rm_url => $hadoop_rm_url,
+ rm_proxy_url => $hadoop_rm_proxy_url,
+ history_server_url => $hadoop_history_server_url,
webhdfs_url => $hadoop_httpfs_url,
rm_host => $hadoop_rm_host,
rm_port => $hadoop_rm_port,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/9eefa559/bigtop-deploy/puppet/modules/hue/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hue/manifests/init.pp b/bigtop-deploy/puppet/modules/hue/manifests/init.pp
index b7fed81..f4a7b57 100644
--- a/bigtop-deploy/puppet/modules/hue/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hue/manifests/init.pp
@@ -14,7 +14,7 @@
# limitations under the License.
class hue {
- define server($webhdfs_url, $rm_host, $rm_port, $oozie_url,
+ define server($webhdfs_url, $rm_host, $rm_port, $oozie_url, $rm_url, $rm_proxy_url, $history_server_url,
$hue_host = "0.0.0.0", $hue_port = "8888", $default_fs = "hdfs://localhost:8020",
$kerberos_realm = "") {
if ($kerberos_realm) {
http://git-wip-us.apache.org/repos/asf/bigtop/blob/9eefa559/bigtop-deploy/puppet/modules/hue/templates/hue.ini
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hue/templates/hue.ini b/bigtop-deploy/puppet/modules/hue/templates/hue.ini
index 1bc3b1c..d9fd161 100644
--- a/bigtop-deploy/puppet/modules/hue/templates/hue.ini
+++ b/bigtop-deploy/puppet/modules/hue/templates/hue.ini
@@ -17,12 +17,12 @@
# ===================================
#
# For complete documentation about the contents of this file, run
-# $ <hue_root>/build/env/bin/hue config_help
+# $ <hue_root>/build/env/bin/hue config_help
#
# All .ini files under the current directory are treated equally. Their
# contents are merged to form the Hue configuration, which can
# can be viewed on the Hue at
-# http://<hue_host>:<port>/dump_config
+# http://<hue_host>:<port>/dump_config
###########################################################################
@@ -48,6 +48,12 @@
# Turn off backtrace for server error
http_500_debug_mode=0
+ # Server email for internal error messages
+ ## django_server_email='hue@localhost.localdomain'
+
+ # Email backend
+ ## django_email_backend=django.core.mail.backends.smtp.EmailBackend
+
# Set to true to use CherryPy as the webserver, set to false
# to use Spawning as the webserver. Defaults to Spawning if
# key is not specified.
@@ -73,6 +79,13 @@
# Default encoding for site data
## default_site_encoding=utf-8
+ # Administrators
+ # ----------------
+ [[django_admins]]
+ ## [[[admin1]]]
+ ## name=john
+ ## email=john@doe.com
+
# UI customizations
# -------------------
[[custom]]
@@ -91,10 +104,22 @@
# (Default. Relies on Django and user manager, after the first login)
# - desktop.auth.backend.LdapBackend
# - desktop.auth.backend.PamBackend
+ # - desktop.auth.backend.SpnegoDjangoBackend
+ # - desktop.auth.backend.RemoteUserDjangoBackend
## backend=desktop.auth.backend.AllowFirstUserDjangoBackend
## pam_service=login
+ # When using the desktop.auth.backend.RemoteUserDjangoBackend, this sets
+ # the normalized name of the header that contains the remote user.
+ # The HTTP header in the request is converted to a key by converting
+ # all characters to uppercase, replacing any hyphens with underscores
+ # and adding an HTTP_ prefix to the name. So, for example, if the header
+ # is called Remote-User that would be configured as HTTP_REMOTE_USER
+ #
+ # Defaults to HTTP_REMOTE_USER
+ ## remote_user_header=HTTP_REMOTE_USER
+
# Configuration options for connecting to LDAP and Active Directory
# -------------------------------------------------------------------
[[ldap]]
@@ -123,6 +148,10 @@
# For use when using LdapBackend for Hue authentication
## ldap_username_pattern="uid=<username>,ou=People,dc=mycompany,dc=com"
+ # Create users in Hue when they try to login with their LDAP credentials
+ # For use when using LdapBackend for Hue authentication
+ ## create_users_on_login = true
+
[[[users]]]
# Base filter for searching for users
@@ -149,7 +178,7 @@
engine=sqlite3
name=/var/lib/hue/desktop.db
# Database engine is typically one of:
- # postgresql, mysql, sqlite3, or oracle
+ # postgresql_psycopg2, mysql, or sqlite3
#
# Note that for sqlite3, 'name', below is a filename;
# for other backends, it is the database name.
@@ -277,10 +306,19 @@
hadoop_mapred_home=/usr/lib/hadoop-mapreduce
# Defaults to $HADOOP_BIN or /usr/bin/hadoop
- ## hadoop_bin=/usr/bin/hadoop
+ hadoop_bin=/usr/bin/hadoop
# Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
- ## hadoop_conf_dir=/etc/hadoop/conf
+ hadoop_conf_dir=/etc/hadoop/conf
+
+ # URL of the ResourceManager API
+ resourcemanager_api_url=<%= rm_url %>
+
+ # URL of the ProxyServer API
+ proxy_api_url=<%= rm_proxy_url %>
+
+ # URL of the HistoryServer API
+ history_server_api_url=<%= history_server_url %>
###########################################################################
@@ -325,13 +363,22 @@
###########################################################################
[beeswax]
- # Configure the port the internal metastore daemon runs on. Used only if
- # hive.metastore.local is true.
- ## beeswax_meta_server_port=8003
- # Configure the port the beeswax thrift server runs on
+ # Host where Beeswax server Thrift daemon is running.
+ # If Kerberos security is enabled, the fully-qualified domain name (FQDN) is
+ # required, even if the Thrift daemon is running on the same host as Hue.
+ ## beeswax_server_host=<FQDN of Beeswax Server>
+
+ # Port where Beeswax Thrift server runs on.
## beeswax_server_port=8002
+ # Host where internal metastore Thrift daemon is running.
+ ## beeswax_meta_server_host=localhost
+
+ # Configure the port the internal metastore daemon runs on.
+ # Used only if hive.metastore.local is true.
+ ## beeswax_meta_server_port=8003
+
# Hive home directory
## hive_home_dir=/usr/lib/hive
@@ -406,8 +453,8 @@
help = "The command-line Flume client interface."
[[[[ environment ]]]]
- # You can specify environment variables for the Flume shell
- # in this section.
+ # You can specify environment variables for the Flume shell
+ # in this section.
# Define and configure a new shell type "pig"
# ------------------------------------------------------------------------
@@ -417,12 +464,23 @@
help = "The command-line interpreter for Pig"
[[[[ environment ]]]]
- # You can specify environment variables for the Pig shell
- # in this section. Note that JAVA_HOME must be configured
- # for the Pig shell to run.
+ # You can specify environment variables for the Pig shell
+ # in this section. Note that JAVA_HOME must be configured
+ # for the Pig shell to run.
+
+ # [[[[[ JAVA_HOME ]]]]]
+ # value = "/usr/lib/jvm/java-6-sun"
- [[[[[ JAVA_HOME ]]]]]
- value = "/usr/lib/jvm/java-6-sun"
+ # Define and configure a new shell type "sqoop2"
+ # ------------------------------------------------------------------------
+ [[[ sqoop2 ]]]
+ nice_name = "Sqoop2 Shell"
+ command = "/usr/bin/sqoop2"
+ help = "The command-line Sqoop2 client."
+
+ [[[[ environment ]]]]
+ # You can configure environment variables for the Sqoop2 shell
+ # in this section.
# Define and configure a new shell type "hbase"
# ------------------------------------------------------------------------
@@ -432,8 +490,8 @@
help = "The command-line HBase client interface."
[[[[ environment ]]]]
- # You can configure environment variables for the HBase shell
- # in this section.
+ # You can configure environment variables for the HBase shell
+ # in this section.
###########################################################################
@@ -442,4 +500,4 @@
[useradmin]
# The name of the default user group that users will be a member of
- ## default_user_group=default_group
+ ## default_user_group=default
http://git-wip-us.apache.org/repos/asf/bigtop/blob/9eefa559/bigtop-packages/src/common/hue/install_hue.sh
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hue/install_hue.sh b/bigtop-packages/src/common/hue/install_hue.sh
index d2ad6b4..b1ff065 100644
--- a/bigtop-packages/src/common/hue/install_hue.sh
+++ b/bigtop-packages/src/common/hue/install_hue.sh
@@ -111,8 +111,7 @@ cp -f $BUILD_DIR/apps/shell/src/shell/build/setuid $PREFIX/$LIB_DIR/apps/shell/s
# Remove Hue database and then recreate it, but with just the "right" apps
rm -f $PREFIX/$LIB_DIR/desktop/desktop.db $PREFIX/$LIB_DIR/app.reg
-# FIXME: jobbrowser HUE-10
-APPS="about filebrowser help jobsub proxy useradmin shell"
+APPS="about filebrowser jobbrowser help jobsub proxy useradmin shell"
export DESKTOP_LOG_DIR=$BUILD_DIR
export DESKTOP_LOGLEVEL=WARN
export ROOT=$PREFIX/$LIB_DIR
@@ -140,14 +139,9 @@ sed -i -e '/\[\[yarn_clusters\]\]/,+20s@## submit_to=False@submit_to=True@' \
# Fix a redirection, since by default beeswax is not installed
sed -i -e '/meta http-equiv="refresh"/s#/beeswax#/about#' \
$PREFIX/$LIB_DIR/desktop/core/src/desktop/templates/index.mako
-# FIXME: HUE-880
-sed -i -e '/"name": *"Hive"/,/description/s#"parameters":.*$#"parameters": "[{\\"name\\":\\"oozie.use.system.libpath\\",\\"value\\":\\"true\\"}]",#' \
- -e '/"name": *"Pig"/,/description/s#"parameters":.*$#"parameters": "[{\\"name\\":\\"oozie.use.system.libpath\\",\\"value\\":\\"true\\"}]",#' \
- -e '/"name": *"Sqoop"/,/description/s#"parameters":.*$#"parameters": "[{\\"name\\":\\"oozie.use.system.libpath\\",\\"value\\":\\"true\\"}]",#' \
- -e '/"name": *"DistCp"/,/description/s#"parameters":.*$#"parameters": "[{\\"name\\":\\"oozie.use.system.libpath\\",\\"value\\":\\"true\\"}]",#' \
- $PREFIX/$LIB_DIR/apps/oozie/src/oozie/fixtures/initial_example_data.json
-sed -i -e '/jobbrowser.views/s#<a href=[^>]*>#<a href=".">#' \
- $PREFIX/$LIB_DIR/apps/oozie/src/oozie/templates/dashboard/list_oozie_workflow.mako
+# FIXME: HUE-1097
+sed -i -e '/impala.conf/d' \
+ $PREFIX/$LIB_DIR/apps/beeswax/src/beeswax/server/dbms.py
# Relink logs subdirectory just in case
install -d -m 0755 $PREFIX/$LOG_DIR
http://git-wip-us.apache.org/repos/asf/bigtop/blob/9eefa559/bigtop-packages/src/deb/hue/hue-common.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hue/hue-common.install b/bigtop-packages/src/deb/hue/hue-common.install
index 24d5753..20dd95e 100644
--- a/bigtop-packages/src/deb/hue/hue-common.install
+++ b/bigtop-packages/src/deb/hue/hue-common.install
@@ -19,8 +19,7 @@
/usr/lib/hue/apps/about
/usr/lib/hue/apps/filebrowser
/usr/lib/hue/apps/help
-# FIXME: HUE-10
-# /usr/lib/hue/apps/jobbrowser
+/usr/lib/hue/apps/jobbrowser
/usr/lib/hue/apps/jobsub
/usr/lib/hue/apps/proxy
/usr/lib/hue/apps/shell
http://git-wip-us.apache.org/repos/asf/bigtop/blob/9eefa559/bigtop-packages/src/rpm/hue/SPECS/hue.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/hue/SPECS/hue.spec b/bigtop-packages/src/rpm/hue/SPECS/hue.spec
index d11828a..7861057 100644
--- a/bigtop-packages/src/rpm/hue/SPECS/hue.spec
+++ b/bigtop-packages/src/rpm/hue/SPECS/hue.spec
@@ -255,8 +255,7 @@ fi
%{about_app_dir}
%{filebrowser_app_dir}
%{help_app_dir}
-# FIXME: HUE-10
-# %{jobbrowser_app_dir}
+%{jobbrowser_app_dir}
%{jobsub_app_dir}
%{proxy_app_dir}
%{useradmin_app_dir}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/9eefa559/bigtop.mk
----------------------------------------------------------------------
diff --git a/bigtop.mk b/bigtop.mk
index 3fda56c..44f6168 100644
--- a/bigtop.mk
+++ b/bigtop.mk
@@ -190,12 +190,12 @@ $(eval $(call PACKAGE,giraph,GIRAPH))
HUE_NAME=hue
HUE_RELNOTES_NAME=Hadoop User Experience
HUE_PKG_NAME=hue
-HUE_BASE_VERSION=2.1.0
-HUE_PKG_VERSION=2.1.0
+HUE_BASE_VERSION=2.2.0
+HUE_PKG_VERSION=2.2.0
HUE_RELEASE_VERSION=1
HUE_TARBALL_DST=hue-$(HUE_BASE_VERSION).tar.gz
HUE_TARBALL_SRC=hue-$(HUE_BASE_VERSION).tgz
-HUE_SITE=https://github.com/downloads/cloudera/hue
+HUE_SITE=http://cloudera.github.com/hue/releases/$(HUE_BASE_VERSION)
HUE_ARCHIVE=$(HUE_SITE)
$(eval $(call PACKAGE,hue,HUE))
[4/5] git commit: BIGTOP-893. migrate puppet code to use init-hdfs.sh
Posted by rv...@apache.org.
BIGTOP-893. migrate puppet code to use init-hdfs.sh
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/5b640023
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/5b640023
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/5b640023
Branch: refs/heads/master
Commit: 5b64002342a1df10927d4a6fdeb42f9c956e68ad
Parents: 4d3290d
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Thu Mar 28 12:49:35 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Thu Mar 28 12:49:35 2013 -0700
----------------------------------------------------------------------
bigtop-deploy/puppet/manifests/cluster.pp | 49 +++++--------------
bigtop-packages/src/common/hadoop/init-hdfs.sh | 3 +
2 files changed, 16 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/5b640023/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index 041a37a..de1b8bf 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -232,48 +232,25 @@ class hadoop_head_node inherits hadoop_cluster_node {
kerberos_realm => $kerberos_realm,
}
- hadoop::create_hdfs_dirs { [ "/tmp", "/var", "/var/log", "/hbase", "/benchmarks", "/user", "/user/history", "/user/jenkins", "/user/hive", "/user/root", "/user/hue", "/user/oozie" ]:
- auth => $hadoop_security_authentication,
- hdfs_dirs_meta => { "/tmp" => { perm =>"1777", user => "hdfs" },
- "/var" => { perm => "755", user => "hdfs" },
- "/var/log" => { perm =>"1775", user => "yarn:mapred" },
- "/hbase" => { perm => "755", user => "hbase" },
- "/benchmarks" => { perm => "777", user => "hdfs" },
- "/user" => { perm => "755", user => "hdfs" },
- "/user/history" => { perm => "775", user => "mapred" },
- "/user/jenkins" => { perm => "777", user => "jenkins"},
- "/user/hive" => { perm => "777", user => "hive" },
- "/user/root" => { perm => "777", user => "root" },
- "/user/hue" => { perm => "777", user => "hue" },
- "/user/oozie" => { perm => "777", user => "oozie" },
- },
- }
- Hadoop::Create_hdfs_dirs<||> -> Hadoop-hbase::Master<||>
- Hadoop::Create_hdfs_dirs<||> -> Hadoop::Resourcemanager<||>
- Hadoop::Create_hdfs_dirs<||> -> Hadoop::Historyserver<||>
- Hadoop::Create_hdfs_dirs<||> -> Hadoop::Httpfs<||>
- Hadoop::Create_hdfs_dirs<||> -> Hadoop::Rsync_hdfs<||>
-
- hadoop::rsync_hdfs { [ "/user/oozie/share/lib/hive",
- "/user/oozie/share/lib/mapreduce-streaming",
- "/user/oozie/share/lib/distcp",
- "/user/oozie/share/lib/pig",
- "/user/oozie/share/lib/sqoop" ]:
- auth => $hadoop_security_authentication,
- files => { "/user/oozie/share/lib/hive" => "/usr/lib/hive/lib/*.jar",
- "/user/oozie/share/lib/mapreduce-streaming" => "/usr/lib/hadoop-mapreduce/hadoop-streaming*.jar",
- "/user/oozie/share/lib/distcp" => "/usr/lib/hadoop-mapreduce/hadoop-distcp*.jar",
- "/user/oozie/share/lib/pig" => "/usr/lib/pig/{lib/,}*.jar",
- "/user/oozie/share/lib/sqoop" => "/usr/lib/sqoop/{lib/,}*.jar" },
- }
- Hadoop::Rsync_hdfs<||> -> Hadoop-oozie::Server<||>
-
solr::server { "solrcloud server":
collections => $solrcloud_collections,
port => $solrcloud_port,
port_admin => $solrcloud_port_admin,
zk => $solrcloud_zk,
}
+
+ exec { "init hdfs":
+ path => ['/bin','/sbin','/usr/bin','/usr/sbin'],
+ command => 'bash -x /usr/lib/hadoop/libexec/init-hdfs.sh',
+ require => Package['hadoop-hdfs']
+ }
+
+ Exec<| title == "init hdfs" |> -> Hadoop-hbase::Master<||>
+ Exec<| title == "init hdfs" |> -> Hadoop::Resourcemanager<||>
+ Exec<| title == "init hdfs" |> -> Hadoop::Historyserver<||>
+ Exec<| title == "init hdfs" |> -> Hadoop::Httpfs<||>
+ Exec<| title == "init hdfs" |> -> Hadoop::Rsync_hdfs<||>
+ Exec<| title == "init hdfs" |> -> Hadoop-oozie::Server<||>
}
class standby_head_node inherits hadoop_cluster_node {
http://git-wip-us.apache.org/repos/asf/bigtop/blob/5b640023/bigtop-packages/src/common/hadoop/init-hdfs.sh
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hadoop/init-hdfs.sh b/bigtop-packages/src/common/hadoop/init-hdfs.sh
index de43564..80cb212 100755
--- a/bigtop-packages/src/common/hadoop/init-hdfs.sh
+++ b/bigtop-packages/src/common/hadoop/init-hdfs.sh
@@ -26,6 +26,9 @@ su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /var'
su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /var/log'
su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 1775 /var/log'
su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown yarn:mapred /var/log'
+su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir -p /var/log/hadoop-yarn/apps'
+su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 1777 /var/log/hadoop-yarn/apps'
+su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown yarn:mapred /var/log/hadoop-yarn/apps'
su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /hbase'
su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hbase /hbase'
su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /benchmarks'
[2/5] git commit: BIGTOP-822. Bump version of Hadoop to 2.0.4
Posted by rv...@apache.org.
BIGTOP-822. Bump version of Hadoop to 2.0.4
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/d821cdf0
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/d821cdf0
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/d821cdf0
Branch: refs/heads/master
Commit: d821cdf0c93987ca96191b9ac42c24c0f9e0998c
Parents: 9eefa55
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Thu Mar 28 12:35:02 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Thu Mar 28 12:35:02 2013 -0700
----------------------------------------------------------------------
bigtop-packages/src/deb/hadoop/docs | 3 ---
bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec | 3 ++-
bigtop.mk | 15 +++++++++------
3 files changed, 11 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/d821cdf0/bigtop-packages/src/deb/hadoop/docs
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hadoop/docs b/bigtop-packages/src/deb/hadoop/docs
deleted file mode 100644
index b5a470a..0000000
--- a/bigtop-packages/src/deb/hadoop/docs
+++ /dev/null
@@ -1,3 +0,0 @@
-LICENSE.txt
-NOTICE.txt
-README.txt
http://git-wip-us.apache.org/repos/asf/bigtop/blob/d821cdf0/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
index 7fa4dbd..bf93f52 100644
--- a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
+++ b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
@@ -421,7 +421,8 @@ These projects (enumerated below) allow HDFS to be mounted (on most flavors of U
%prep
-%setup -n %{name}-%{hadoop_base_version}-src
+# %setup -n %{name}-%{hadoop_base_version}-src
+%setup -n apache-hadoop-common-55bb321
%build
http://git-wip-us.apache.org/repos/asf/bigtop/blob/d821cdf0/bigtop.mk
----------------------------------------------------------------------
diff --git a/bigtop.mk b/bigtop.mk
index 44f6168..4bea43d 100644
--- a/bigtop.mk
+++ b/bigtop.mk
@@ -18,14 +18,17 @@ BIGTOP_VERSION=0.6.0
# Hadoop 0.20.0-based hadoop package
HADOOP_NAME=hadoop
HADOOP_RELNOTES_NAME=Apache Hadoop
-HADOOP_BASE_VERSION=2.0.2-alpha
-HADOOP_PKG_VERSION=2.0.2
+HADOOP_BASE_VERSION=2.0.4-SNAPSHOT
+HADOOP_PKG_VERSION=2.0.4
HADOOP_RELEASE_VERSION=1
HADOOP_TARBALL_DST=$(HADOOP_NAME)-$(HADOOP_BASE_VERSION).tar.gz
-HADOOP_TARBALL_SRC=$(HADOOP_NAME)-$(HADOOP_BASE_VERSION)-src.tar.gz
-HADOOP_DOWNLOAD_PATH=/hadoop/common/$(HADOOP_NAME)-$(HADOOP_BASE_VERSION)
-HADOOP_SITE=$(APACHE_MIRROR)$(HADOOP_DOWNLOAD_PATH)
-HADOOP_ARCHIVE=$(APACHE_ARCHIVE)$(HADOOP_DOWNLOAD_PATH)
+#HADOOP_TARBALL_SRC=$(HADOOP_NAME)-$(HADOOP_BASE_VERSION)-src.tar.gz
+#HADOOP_DOWNLOAD_PATH=/hadoop/common/$(HADOOP_NAME)-$(HADOOP_BASE_VERSION)
+#HADOOP_SITE=$(APACHE_MIRROR)$(HADOOP_DOWNLOAD_PATH)
+#HADOOP_ARCHIVE=$(APACHE_ARCHIVE)$(HADOOP_DOWNLOAD_PATH)
+HADOOP_TARBALL_SRC=55bb321
+HADOOP_SITE=https://github.com/apache/hadoop-common/tarball
+HADOOP_ARCHIVE=$(HADOOP_SITE)
$(eval $(call PACKAGE,hadoop,HADOOP))
# ZooKeeper
[5/5] git commit: BIGTOP-763. Add hdfs-journalnode to hadoop.spec
Posted by rv...@apache.org.
BIGTOP-763. Add hdfs-journalnode to hadoop.spec
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c74fbe7c
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c74fbe7c
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c74fbe7c
Branch: refs/heads/master
Commit: c74fbe7c354a39cd96be328f6cb90bf78173e82b
Parents: 5b64002
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Sun Feb 10 14:51:32 2013 -0800
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Thu Mar 28 13:02:05 2013 -0700
----------------------------------------------------------------------
.../src/common/hadoop/hadoop-hdfs-journalnode.svc | 24 +++++++++++++++
bigtop-packages/src/common/hadoop/hadoop.1 | 5 ++-
bigtop-packages/src/deb/hadoop/control | 9 +++++
bigtop-packages/src/deb/hadoop/rules | 2 +-
bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec | 15 ++++++++-
5 files changed, 52 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c74fbe7c/bigtop-packages/src/common/hadoop/hadoop-hdfs-journalnode.svc
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hadoop/hadoop-hdfs-journalnode.svc b/bigtop-packages/src/common/hadoop/hadoop-hdfs-journalnode.svc
new file mode 100644
index 0000000..2afafb8
--- /dev/null
+++ b/bigtop-packages/src/common/hadoop/hadoop-hdfs-journalnode.svc
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+HADOOP_NAME="journalnode"
+DAEMON="hadoop-hdfs-$HADOOP_NAME"
+DESC="Hadoop $HADOOP_NAME"
+EXEC_PATH="/usr/lib/hadoop/sbin/hadoop-daemon.sh"
+SVC_USER="hdfs"
+DAEMON_FLAGS="$HADOOP_NAME"
+CONF_DIR="/etc/hadoop/conf"
+PIDFILE="/var/run/hadoop-hdfs/hadoop-$SVC_USER-$HADOOP_NAME.pid"
+LOCKDIR="/var/lock/subsys"
+LOCKFILE="hadoop-$HADOOP_NAME"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c74fbe7c/bigtop-packages/src/common/hadoop/hadoop.1
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hadoop/hadoop.1 b/bigtop-packages/src/common/hadoop/hadoop.1
index 1d9de16..6a2f4c8 100644
--- a/bigtop-packages/src/common/hadoop/hadoop.1
+++ b/bigtop-packages/src/common/hadoop/hadoop.1
@@ -274,8 +274,11 @@ Service script for starting and stopping the Hadoop DataNode
.IX Item "/etc/init.d/hadoop-secondarynamenode"
Service script for starting and stopping the Hadoop Secondary NameNode
.IP "/etc/init.d/hadoop\-hdfs\-zkfc" 4
-.IX Item "/etc/init.d/hadoop-secondarynamenode"
+.IX Item "/etc/init.d/hadoop-hdfs-zkfc"
Service script for starting and stopping the Hadoop HDFS failover controller
+.IP "/etc/init.d/hadoop\-hdfs\-journalnode" 4
+.IX Item "/etc/init.d/hadoop-hdfs-journalnode"
+Service script for starting and stopping the Hadoop HDFS JournalNode
.IP "/etc/init.d/hadoop\-yarn\-resourcemanager" 4
.IX Item "/etc/init.d/hadoop-yarn-resourcemanager"
Service script for starting and stopping the Hadoop YARN Resource Manager
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c74fbe7c/bigtop-packages/src/deb/hadoop/control
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hadoop/control b/bigtop-packages/src/deb/hadoop/control
index 82cf345..6fab677 100644
--- a/bigtop-packages/src/deb/hadoop/control
+++ b/bigtop-packages/src/deb/hadoop/control
@@ -154,6 +154,15 @@ Description: Hadoop HDFS failover controller
for: Health monitoring, ZooKeeper session management, ZooKeeper-based
election.
+Package: hadoop-hdfs-journalnode
+Provides: hadoop-hdfs-journalnode
+Architecture: all
+Depends: hadoop-hdfs (= ${source:Version})
+Description: Hadoop HDFS JournalNode
+ The HDFS JournalNode is responsible for persisting NameNode edit logs.
+ In a typical deployment the JournalNode daemon runs on at least three
+ separate machines in the cluster.
+
Package: hadoop-hdfs-datanode
Architecture: all
Depends: hadoop-hdfs (= ${source:Version})
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c74fbe7c/bigtop-packages/src/deb/hadoop/rules
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hadoop/rules b/bigtop-packages/src/deb/hadoop/rules
index deb7ae7..a27c205 100755
--- a/bigtop-packages/src/deb/hadoop/rules
+++ b/bigtop-packages/src/deb/hadoop/rules
@@ -39,7 +39,7 @@ override_dh_auto_build:
env HADOOP_VERSION=${hadoop_version} HADOOP_ARCH=${native_dir} \
bash debian/do-component-build -Divy.home=${HOME}/.ivy2
-hadoop_svcs=hdfs-namenode hdfs-secondarynamenode hdfs-datanode hdfs-zkfc \
+hadoop_svcs=hdfs-namenode hdfs-secondarynamenode hdfs-datanode hdfs-zkfc hdfs-journalnode \
yarn-resourcemanager yarn-nodemanager yarn-proxyserver \
mapreduce-historyserver httpfs
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c74fbe7c/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
index cd5bed8..ec86023 100644
--- a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
+++ b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
@@ -56,7 +56,7 @@
%define doc_hadoop %{_docdir}/%{name}-%{hadoop_version}
%define httpfs_services httpfs
%define mapreduce_services mapreduce-historyserver
-%define hdfs_services hdfs-namenode hdfs-secondarynamenode hdfs-datanode hdfs-zkfc
+%define hdfs_services hdfs-namenode hdfs-secondarynamenode hdfs-datanode hdfs-zkfc hdfs-journalnode
%define yarn_services yarn-resourcemanager yarn-nodemanager yarn-proxyserver
%define hadoop_services %{hdfs_services} %{mapreduce_services} %{yarn_services} %{httpfs_services}
# Hadoop outputs built binaries into %{hadoop_build}
@@ -161,6 +161,7 @@ Source20: hdfs.default
Source21: yarn.default
Source22: hadoop-layout.sh
Source23: hadoop-hdfs-zkfc.svc
+Source24: hadoop-hdfs-journalnode.svc
Buildroot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id} -u -n)
BuildRequires: fuse-devel, fuse, cmake
Requires: coreutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service, bigtop-utils, zookeeper >= 3.4.0
@@ -291,6 +292,17 @@ which runs a NameNode also runs a ZKFC, and that ZKFC is responsible
for: Health monitoring, ZooKeeper session management, ZooKeeper-based
election.
+%package hdfs-journalnode
+Summary: Hadoop HDFS JournalNode
+Group: System/Daemons
+Requires: %{name}-hdfs = %{version}-%{release}
+Requires(pre): %{name} = %{version}-%{release}
+
+%description hdfs-journalnode
+The HDFS JournalNode is responsible for persisting NameNode edit logs.
+In a typical deployment the JournalNode daemon runs on at least three
+separate machines in the cluster.
+
%package hdfs-datanode
Summary: Hadoop Data Node
Group: System/Daemons
@@ -663,6 +675,7 @@ fi
%service_macro hdfs-namenode
%service_macro hdfs-secondarynamenode
%service_macro hdfs-zkfc
+%service_macro hdfs-journalnode
%service_macro hdfs-datanode
%service_macro yarn-resourcemanager
%service_macro yarn-nodemanager
[3/5] git commit: BIGTOP-892. hadoop packages misses
capacity-scheduler configuration
Posted by rv...@apache.org.
BIGTOP-892. hadoop packages misses capacity-scheduler configuration
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/4d3290d9
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/4d3290d9
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/4d3290d9
Branch: refs/heads/master
Commit: 4d3290d978beadccdd1567e1b2630f404e034bed
Parents: d821cdf
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Thu Mar 28 12:42:03 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Thu Mar 28 12:42:03 2013 -0700
----------------------------------------------------------------------
.../src/deb/hadoop/hadoop-mapreduce.install | 3 +++
bigtop-packages/src/deb/hadoop/hadoop-yarn.install | 2 ++
bigtop-packages/src/deb/hadoop/hadoop.install | 3 +++
bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec | 8 ++++++++
4 files changed, 16 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4d3290d9/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.install b/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.install
index ddfce49..34a3796 100644
--- a/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.install
+++ b/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.install
@@ -1,5 +1,8 @@
/etc/security/limits.d/mapreduce.conf
/etc/hadoop/conf.empty/mapred-site.xml
+/etc/hadoop/conf.empty/mapred-env.sh
+/etc/hadoop/conf.empty/mapred-queues.xml.template
+/etc/hadoop/conf.empty/mapred-site.xml.template
/usr/lib/hadoop-mapreduce
/usr/lib/hadoop/libexec/mapred-config.sh
/usr/bin/mapred
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4d3290d9/bigtop-packages/src/deb/hadoop/hadoop-yarn.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hadoop/hadoop-yarn.install b/bigtop-packages/src/deb/hadoop/hadoop-yarn.install
index 0151948..e9c5074 100644
--- a/bigtop-packages/src/deb/hadoop/hadoop-yarn.install
+++ b/bigtop-packages/src/deb/hadoop/hadoop-yarn.install
@@ -1,6 +1,8 @@
/etc/security/limits.d/yarn.conf
/etc/hadoop/conf.empty/yarn-env.sh
/etc/hadoop/conf.empty/yarn-site.xml
+/etc/hadoop/conf.empty/capacity-scheduler.xml
+/etc/hadoop/conf.empty/container-executor.cfg
/usr/lib/hadoop-yarn
/usr/lib/hadoop/libexec/yarn-config.sh
/usr/bin/yarn
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4d3290d9/bigtop-packages/src/deb/hadoop/hadoop.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hadoop/hadoop.install b/bigtop-packages/src/deb/hadoop/hadoop.install
index 20254fe..38b6286 100644
--- a/bigtop-packages/src/deb/hadoop/hadoop.install
+++ b/bigtop-packages/src/deb/hadoop/hadoop.install
@@ -5,6 +5,9 @@
/etc/hadoop/conf.empty/ssl-client.xml.example
/etc/hadoop/conf.empty/ssl-server.xml.example
/etc/hadoop/conf.empty/core-site.xml
+/etc/hadoop/conf.empty/configuration.xsl
+/etc/hadoop/conf.empty/hadoop-env.sh
+/etc/hadoop/conf.empty/hadoop-policy.xml
/etc/default/hadoop
/etc/bash_completion.d/hadoop
/usr/lib/hadoop/etc
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4d3290d9/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
index bf93f52..cd5bed8 100644
--- a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
+++ b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
@@ -561,6 +561,8 @@ fi
%defattr(-,root,root)
%config(noreplace) %{etc_hadoop}/conf.empty/yarn-env.sh
%config(noreplace) %{etc_hadoop}/conf.empty/yarn-site.xml
+%config(noreplace) %{etc_hadoop}/conf.empty/capacity-scheduler.xml
+%config(noreplace) %{etc_hadoop}/conf.empty/container-executor.cfg
%config(noreplace) /etc/security/limits.d/yarn.conf
%{lib_hadoop}/libexec/yarn-config.sh
%{lib_yarn}
@@ -588,6 +590,9 @@ fi
%files mapreduce
%defattr(-,root,root)
%config(noreplace) %{etc_hadoop}/conf.empty/mapred-site.xml
+%config(noreplace) %{etc_hadoop}/conf.empty/mapred-env.sh
+%config(noreplace) %{etc_hadoop}/conf.empty/mapred-queues.xml.template
+%config(noreplace) %{etc_hadoop}/conf.empty/mapred-site.xml.template
%config(noreplace) /etc/security/limits.d/mapreduce.conf
%{lib_mapreduce}
%{lib_hadoop}/libexec/mapred-config.sh
@@ -607,6 +612,9 @@ fi
%config(noreplace) %{etc_hadoop}/conf.empty/slaves
%config(noreplace) %{etc_hadoop}/conf.empty/ssl-client.xml.example
%config(noreplace) %{etc_hadoop}/conf.empty/ssl-server.xml.example
+%config(noreplace) %{etc_hadoop}/conf.empty/configuration.xsl
+%config(noreplace) %{etc_hadoop}/conf.empty/hadoop-env.sh
+%config(noreplace) %{etc_hadoop}/conf.empty/hadoop-policy.xml
%config(noreplace) /etc/default/hadoop
/etc/bash_completion.d/hadoop
%{lib_hadoop}/*.jar