You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by co...@apache.org on 2014/12/10 06:03:10 UTC
[1/2] bigtop git commit: BIGTOP-1566. Puppet README file needs to
reflect multiple modules directoty requirement
Repository: bigtop
Updated Branches:
refs/heads/master 6f720d06c -> 6bae4a32b
BIGTOP-1566. Puppet README file needs to reflect multiple modules directoty requirement
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/6bae4a32
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/6bae4a32
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/6bae4a32
Branch: refs/heads/master
Commit: 6bae4a32ba94598bd0f9a7015bc5e9af3e6180ef
Parents: 5d7a513
Author: Konstantin Boudnik <co...@apache.org>
Authored: Mon Dec 8 17:29:01 2014 -0800
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Tue Dec 9 14:40:41 2014 -0800
----------------------------------------------------------------------
bigtop-deploy/puppet/README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6bae4a32/bigtop-deploy/puppet/README.md
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/README.md b/bigtop-deploy/puppet/README.md
index b518d61..c76185a 100644
--- a/bigtop-deploy/puppet/README.md
+++ b/bigtop-deploy/puppet/README.md
@@ -95,5 +95,5 @@ And run the following on those nodes:
# hadoop_storage_dirs,/data/1,/data/2
# bigtop_yumrepo_uri,http://mirror.example.com/path/to/mirror/
# EOF
-# puppet apply -d --modulepath=bigtop-deploy/puppet/modules bigtop-deploy/puppet/manifests/site.pp
+# puppet apply -d --modulepath="bigtop-deploy/puppet/modules:/etc/puppet/modules" bigtop-deploy/puppet/manifests/site.pp
</pre>
[2/2] bigtop git commit: BIGTOP-1563. Puppet deployment needs to
setup user hdfs keys for password-less logins
Posted by co...@apache.org.
BIGTOP-1563. Puppet deployment needs to setup user hdfs keys for password-less logins
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/5d7a513f
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/5d7a513f
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/5d7a513f
Branch: refs/heads/master
Commit: 5d7a513f5d96e649cf2ee31c030a955aecef0865
Parents: 6f720d0
Author: Konstantin Boudnik <co...@apache.org>
Authored: Mon Dec 8 17:00:49 2014 -0800
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Tue Dec 9 14:40:41 2014 -0800
----------------------------------------------------------------------
bigtop-deploy/puppet/README.md | 4 +++
bigtop-deploy/puppet/config/hdfs/id_hdfsuser | 27 +++++++++++++++
.../puppet/config/hdfs/id_hdfsuser.pub | 1 +
bigtop-deploy/puppet/config/site.csv.example | 3 ++
.../puppet/modules/hadoop/manifests/init.pp | 36 ++++++++++++++++++++
5 files changed, 71 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/5d7a513f/bigtop-deploy/puppet/README.md
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/README.md b/bigtop-deploy/puppet/README.md
index e9e5a7b..b518d61 100644
--- a/bigtop-deploy/puppet/README.md
+++ b/bigtop-deploy/puppet/README.md
@@ -73,6 +73,10 @@ $confdir is the directory that puppet will look into for its configuration. On
this will be either /etc/puppet/ or /etc/puppetlabs/puppet/. You may override this value by
specifying --confdir=path/to/config/dir on the puppet command line.
+You can instruct the recipes to install ssh-keys for user hdfs to enable passwordless login
+across the cluster. This is for test purposes only, so by default the option is turned off.
+Refer to bigtop-deploy/puppet/config/site.csv.example for more details.
+
For other options that may be set here, look for calls to extlookup() in manifests/cluster.pp.
Note that if hadoop\_storage\_dirs is left unset, puppet will attempt to guess which directories
to use.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/5d7a513f/bigtop-deploy/puppet/config/hdfs/id_hdfsuser
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/config/hdfs/id_hdfsuser b/bigtop-deploy/puppet/config/hdfs/id_hdfsuser
new file mode 100644
index 0000000..2a7c35f
--- /dev/null
+++ b/bigtop-deploy/puppet/config/hdfs/id_hdfsuser
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAt6fOtyNM1yCgrdKIXzmZ4k00F2u/Fi5CARiZN8A2qD8vfRHa
+0sjyRRRuIJzTQArNiFS6TWJaTW7C19McCiaNGHHGAFeaxpzNjS56B8z+EfCG9vUP
+828i/81ED/+inOGSzRSBnivpxOgYNzaRwY/PxmjZ/9DRD6g6/2lqJu59j+gmXQqO
+rdyJFPPiGPQjEBJ2Yrg/hSdaq6/1YReh+C6AKWgIFdgv5CKrlutMM/OXGW9HhujX
+s0Hg7ewNeCcfGhPeatchTF0R8p7EoHLGrLwn+FtgTBvElo/NQN+pdwL8B7qXOKbr
+mNRY8xjdIrItBa4wa6V0B3JKo4YMnSCx+W68EwIDAQABAoIBACxhW727xQzTcKCw
+mI3SYJJkL6tdOS8F/KNBK31icJLVugcSMnvvXUQ+PKKa4x29vo8sUlzy4HmHw4Pe
+Gw+E42G91Kgug1NtfuqYCZsq+Vaidirla3ZbK+z+0GzBTK12gUAXrX7og7Q2let5
+v100AHSkz9FWx75t29vuwF0qEqOcgBKLNKHUAmd0i4fUE0wYiMOGO0Q5HOWv142V
+eFLSWZ3AG6F7uXyzhd24c8b5nHiQVrxOwIHoUSAQVsbbXnAnyeSQ0MaqJkfEsIGf
+CFIWFUKSqPVhMASBVazEO8WPNJHoO6sunVpLkk+Hjjw3QIt4U8HYB56unzuCbBMA
+50+HsrkCgYEA4QY/IwwQCKWZsosnIRgeFlCFH4Mz817E5lJq3vvuoyeCCGol1sCj
+tJW149que7MXt0lo0+R7TK3US7TlmJy9dbNP/fmlEAo6ne4hQ1+aosKANbvf+VmH
+Plvohox2YoihgtgbQhr60EVql8PJ4w2rVT5JsyGmzM0an9GNv2bqZiUCgYEA0O+9
+hBaxAHfGYEzwNlsDs6AX5BPrdvefFLln4q34Hg1tpYK+7L9jUGhZrFB0NCUjBf1N
+X7m/kCeQQD+HE8KqBzSyGSGxSKlySnKjFK86U9lY0maPBPeqREbPyGnc0qME4W9+
+SQWy7EBQDQY8envFge3iFzScJYcfDblO6+fNN9cCgYAH1dnY9JCSXIHPj7zbECXs
+DP21rTHf5YrhojkKbqdOkw+aD2gl5CtPHG9q8bBlqkTHIw09Y5VPhDYCNNhRyhNm
+x8mfRFQVDOYhnasW13tg83jXZiVUX2OwLi8YkEeHAs7s6W15jcrx+ZjOlAixmUY/
+/509WYyfUPz+V6Oh6Il86QKBgQCJOVvsPAcGeqQUrZN6763vtK7cQmT83sdhkh/z
+1jTGcCVadhO1ZEHzFoYG0VRwxw9IqMO/ojx5jjKEr/Skes63Ktnp/aH1SizhTodj
+V27kkTRwA6HdLlhqvWNMipLY2O6krVhw10hbOCYbV8Qy4cWnfzhl60y+rQ8tMU7v
+S3ASQwKBgFd+MdpochiSNbu+51RCVgnOyPahJmwstVq88p4PzdBNw7qlo5lWt30W
+EejN+fUR/dIZPy4M3nxH9W7n4QWqa8UiuNC1hFEQm/tnj7hKXu8P5jPXTcIJFusf
+d26rhw3MAbnwoqtRiyXxA6Teecq7RJH4I1+6RB1Y+jUeVO9rlmyA
+-----END RSA PRIVATE KEY-----
http://git-wip-us.apache.org/repos/asf/bigtop/blob/5d7a513f/bigtop-deploy/puppet/config/hdfs/id_hdfsuser.pub
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/config/hdfs/id_hdfsuser.pub b/bigtop-deploy/puppet/config/hdfs/id_hdfsuser.pub
new file mode 100644
index 0000000..8e279db
--- /dev/null
+++ b/bigtop-deploy/puppet/config/hdfs/id_hdfsuser.pub
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC3p863I0zXIKCt0ohfOZniTTQXa78WLkIBGJk3wDaoPy99EdrSyPJFFG4gnNNACs2IVLpNYlpNbsLX0xwKJo0YccYAV5rGnM2NLnoHzP4R8Ib29Q/zbyL/zUQP/6Kc4ZLNFIGeK+nE6Bg3NpHBj8/GaNn/0NEPqDr/aWom7n2P6CZdCo6t3IkU8+IY9CMQEnZiuD+FJ1qrr/VhF6H4LoApaAgV2C/kIquW60wz85cZb0eG6NezQeDt7A14Jx8aE95q1yFMXRHynsSgcsasvCf4W2BMG8SWj81A36l3AvwHupc4puuY1FjzGN0isi0FrjBrpXQHckqjhgydILH5brwT hdfs dev ssh key
http://git-wip-us.apache.org/repos/asf/bigtop/blob/5d7a513f/bigtop-deploy/puppet/config/site.csv.example
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/config/site.csv.example b/bigtop-deploy/puppet/config/site.csv.example
index f8e5774..820934e 100644
--- a/bigtop-deploy/puppet/config/site.csv.example
+++ b/bigtop-deploy/puppet/config/site.csv.example
@@ -22,4 +22,7 @@ bigtop_yumrepo_uri,http://mirror.example.com/path/to/mirror/
# hue,mahout,giraph,crunch,pig,hive,zookeeper
# Example (to deploy only HDFS and YARN server and gateway parts)
#components,hadoop,yarn
+# Test-only variable controls if user hdfs' sshkeys should be installed to allow
+# for passwordless login across the cluster. Required by some integration tests
+#testonly_hdfs_sshkeys=no
http://git-wip-us.apache.org/repos/asf/bigtop/blob/5d7a513f/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
index 0562f3f..ef4ffa8 100644
--- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
@@ -77,6 +77,42 @@ class hadoop {
}
class common-hdfs inherits common {
+ # Check if test mode is enforced, so we can install hdfs ssh-keys for passwordless
+ $testonly = extlookup("testonly_hdfs_sshkeys", 'no')
+ if ($testonly == "yes") {
+ notify{"WARNING: provided hdfs ssh keys are for testing purposes only.\n
+ They shouldn't be used in production cluster": }
+ $ssh_user = "hdfs"
+ $ssh_user_home = "/var/lib/hadoop-hdfs"
+ $ssh_user_keydir = "$ssh_user_home/.ssh"
+ $ssh_keypath = "$ssh_user_keydir/id_hdfsuser"
+ $ssh_privkey = "$extlookup_datadir/hdfs/id_hdfsuser"
+ $ssh_pubkey = "$extlookup_datadir/hdfs/id_hdfsuser.pub"
+
+ file { $ssh_user_keydir:
+ ensure => directory,
+ owner => 'hdfs',
+ group => 'hdfs',
+ mode => '0700',
+ require => Package["hadoop-hdfs"],
+ }
+
+ file { $ssh_keypath:
+ source => $ssh_privkey,
+ owner => 'hdfs',
+ group => 'hdfs',
+ mode => '0600',
+ require => File[$ssh_user_keydir],
+ }
+
+ file { "$ssh_user_keydir/authorized_keys":
+ source => $ssh_pubkey,
+ owner => 'hdfs',
+ group => 'hdfs',
+ mode => '0600',
+ require => File[$ssh_user_keydir],
+ }
+ }
if ($auth == "kerberos" and $ha != "disabled") {
fail("High-availability secure clusters are not currently supported")
}