You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2013/12/04 04:02:13 UTC
[1/5] git commit: ACCUMULO-1960 Fall back onto sudo if the user
doesn't run it as root and isn't running processes as their user for
agitation.
Updated Branches:
refs/heads/master 5a8a28af9 -> 5b84dc2a1
ACCUMULO-1960 Fall back onto sudo if the user doesn't run it as root and isn't running processes as their user for agitation.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/35ff0e50
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/35ff0e50
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/35ff0e50
Branch: refs/heads/master
Commit: 35ff0e50ad7d8678e66d193cc53be40f6a72956f
Parents: cccdb8c
Author: Josh Elser <el...@apache.org>
Authored: Tue Dec 3 21:17:03 2013 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Tue Dec 3 21:17:03 2013 -0500
----------------------------------------------------------------------
test/system/continuous/agitator.pl | 71 ++++++++++++++++----
.../system/continuous/continuous-env.sh.example | 3 +-
test/system/continuous/start-agitator.sh | 14 +++-
test/system/continuous/stop-agitator.sh | 8 ++-
4 files changed, 77 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/35ff0e50/test/system/continuous/agitator.pl
----------------------------------------------------------------------
diff --git a/test/system/continuous/agitator.pl b/test/system/continuous/agitator.pl
index f1abcd5..faeb231 100755
--- a/test/system/continuous/agitator.pl
+++ b/test/system/continuous/agitator.pl
@@ -19,21 +19,22 @@
use POSIX qw(strftime);
use Cwd qw();
-if(scalar(@ARGV) != 4 && scalar(@ARGV) != 2){
- print "Usage : agitator.pl <min sleep before kill in minutes>[:max sleep before kill in minutes] <min sleep before tup in minutes>[:max sleep before tup in minutes] [<min kill> <max kill>]\n";
+if(scalar(@ARGV) != 6 && scalar(@ARGV) != 4){
+ print "Usage : agitator.pl <min sleep before kill in minutes>[:max sleep before kill in minutes] <min sleep before tup in minutes>[:max sleep before tup in minutes] hdfs_user accumulo_user [<min kill> <max kill>]\n";
exit(1);
}
-$ACCUMULO_USER='accumulo';
-$HDFS_USER='hdfs';
+$myself=`whoami`;
+chomp($myself);
+$am_root=($myself eq 'root');
$cwd=Cwd::cwd();
$ACCUMULO_HOME=$cwd . '/../../..';
$HADOOP_PREFIX=$ENV{"HADOOP_PREFIX"};
-print STDERR "Current directory: $cwd\n";
-print STDERR "ACCUMULO_HOME=$ACCUMULO_HOME\n";
-print STDERR "HADOOP_PREFIX=$HADOOP_PREFIX\n";
+print "Current directory: $cwd\n";
+print "ACCUMULO_HOME=$ACCUMULO_HOME\n";
+print "HADOOP_PREFIX=$HADOOP_PREFIX\n";
@sleeprange1 = split(/:/, $ARGV[0]);
$sleep1 = $sleeprange1[0];
@@ -67,9 +68,15 @@ if(defined $ENV{'ACCUMULO_CONF_DIR'}){
$ACCUMULO_CONF_DIR = $ACCUMULO_HOME . '/conf';
}
-if(scalar(@ARGV) == 4){
- $minKill = $ARGV[2];
- $maxKill = $ARGV[3];
+$HDFS_USER=$ARGV[2];
+$ACCUMULO_USER=$ARGV[3];
+
+$am_hdfs_user=($HDFS_USER eq $myself);
+$am_accumulo_user=($ACCUMULO_USER eq $myself);
+
+if(scalar(@ARGV) == 6){
+ $minKill = $ARGV[4];
+ $maxKill = $ARGV[5];
}else{
$minKill = 1;
$maxKill = 1;
@@ -133,11 +140,29 @@ while(1){
print STDERR "$t Killing $server $kill_tserver $kill_datanode\n";
if ($kill_tserver) {
- system("su -c '$ACCUMULO_HOME/bin/stop-server.sh $server \"accumulo-start.jar\" tserver KILL' - $ACCUMULO_USER");
+ if ($am_root) {
+ # We're root, switch to the Accumulo user and try to stop gracefully
+ system("su -c '$ACCUMULO_HOME/bin/stop-server.sh $server \"accumulo-start.jar\" tserver KILL' - $ACCUMULO_USER");
+ } elsif ($am_accumulo_user) {
+ # We're the accumulo user, just run the commandj
+ system("$ACCUMULO_HOME/bin/stop-server.sh $server 'accumulo-start.jar' tserver KILL");
+ } else {
+ # We're not the accumulo user, try to use sudo
+ system("sudo -u $ACCUMULO_USER $ACCUMULO_HOME/bin/stop-server.sh $server accumulo-start.jar tserver KILL");
+ }
}
if ($kill_datanode) {
- system("ssh $server pkill -9 -f [p]roc_datanode");
+ if ($am_root) {
+ # We're root, switch to HDFS to ssh and kill the process
+ system("su -c 'ssh $server pkill -9 -f [p]roc_datanode' - $HDFS_USER");
+ } elsif ($am_hdfs_user) {
+ # We're the HDFS user, just kill the process
+ system("ssh $server \"pkill -9 -f '[p]roc_datanode'\"");
+ } else {
+ # We're not the hdfs user, try to use sudo
+ system("ssh $server 'sudo -u $HDFS_USER pkill -9 -f \'[p]roc_datanode\''");
+ }
}
}
@@ -145,11 +170,29 @@ while(1){
sleep($nextsleep2 * 60);
$t = strftime "%Y%m%d %H:%M:%S", localtime;
print STDERR "$t Running tup\n";
- system("su -c $ACCUMULO_HOME/bin/tup.sh - $ACCUMULO_USER");
+ if ($am_root) {
+ # Running as root, su to the accumulo user
+ system("su -c $ACCUMULO_HOME/bin/tup.sh - $ACCUMULO_USER");
+ } elsif ($am_accumulo_user) {
+ # restart the as them as the accumulo user
+ system("$ACCUMULO_HOME/bin/tup.sh");
+ } else {
+ # Not the accumulo user, try to sudo to the accumulo user
+ system("sudo -u $ACCUMULO_USER $ACCUMULO_HOME/bin/tup.sh");
+ }
if ($kill_datanode) {
print STDERR "$t Starting datanode on $server\n";
- system("ssh $server 'su -c \"$HADOOP_PREFIX/sbin/hadoop-daemon.sh start datanode\" - $HDFS_USER'");
+ if ($am_root) {
+ # We're root, switch to the HDFS user
+ system("ssh $server 'su -c \"$HADOOP_PREFIX/sbin/hadoop-daemon.sh start datanode\" - $HDFS_USER 2>/dev/null 1>/dev/null'");
+ } elsif ($am_hdfs_user) {
+ # We can just start as we're the HDFS user
+ system("ssh $server '$HADOOP_PREFIX/sbin/hadoop-daemon.sh start datanode'");
+ } else {
+ # Not the HDFS user, have to try sudo
+ system("ssh $server 'sudo -u $HDFS_USER $HADOOP_PREFIX/sbin/hadoop-daemon.sh start datanode'");
+ }
}
$nextsleep1 = int(rand($sleep1max - $sleep1)) + $sleep1;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/35ff0e50/test/system/continuous/continuous-env.sh.example
----------------------------------------------------------------------
diff --git a/test/system/continuous/continuous-env.sh.example b/test/system/continuous/continuous-env.sh.example
index af80081..7c23edc 100644
--- a/test/system/continuous/continuous-env.sh.example
+++ b/test/system/continuous/continuous-env.sh.example
@@ -24,7 +24,8 @@ ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/opt/zookeeper}
CONTINUOUS_LOG_DIR=$ACCUMULO_HOME/test/system/continuous/logs
INSTANCE_NAME=instance
ZOO_KEEPERS=zhost1,zhost2
-ACCUMULO_USER=accumulo
+ACCUMULO_USER=`whoami`
+HDFS_USER=`whoami`
USER=user
PASS=pass
TABLE=ci
http://git-wip-us.apache.org/repos/asf/accumulo/blob/35ff0e50/test/system/continuous/start-agitator.sh
----------------------------------------------------------------------
diff --git a/test/system/continuous/start-agitator.sh b/test/system/continuous/start-agitator.sh
index fd43a14..8c2bafb 100755
--- a/test/system/continuous/start-agitator.sh
+++ b/test/system/continuous/start-agitator.sh
@@ -21,6 +21,16 @@ export HADOOP_PREFIX
mkdir -p $CONTINUOUS_LOG_DIR
-nohup ./agitator.pl $KILL_SLEEP_TIME $TUP_SLEEP_TIME $MIN_KILL $MAX_KILL >$CONTINUOUS_LOG_DIR/`date +%Y%m%d%H%M%S`_`hostname`_agitator.out 2>$CONTINUOUS_LOG_DIR/`date +%Y%m%d%H%M%S`_`hostname`_agitator.err &
+# Agitator needs to handle HDFS and Accumulo - can't switch to a single user and expect it to work
+nohup ./agitator.pl $KILL_SLEEP_TIME $TUP_SLEEP_TIME $HDFS_USER $ACCUMULO_USER $MIN_KILL $MAX_KILL >$CONTINUOUS_LOG_DIR/`date +%Y%m%d%H%M%S`_`hostname`_agitator.out 2>$CONTINUOUS_LOG_DIR/`date +%Y%m%d%H%M%S`_`hostname`_agitator.err &
-su -c "nohup $CONTINUOUS_CONF_DIR/magitator.pl $MASTER_KILL_SLEEP_TIME $MASTER_RESTART_SLEEP_TIME >$CONTINUOUS_LOG_DIR/`date +%Y%m%d%H%M%S`_`hostname`_magitator.out 2>$CONTINUOUS_LOG_DIR/`date +%Y%m%d%H%M%S`_`hostname`_magitator.err &" -m - $ACCUMULO_USER
+if [[ "`whoami`" == "root" ]]; then
+ # Change to the correct user if started as root
+ su -c "nohup $CONTINUOUS_CONF_DIR/magitator.pl $MASTER_KILL_SLEEP_TIME $MASTER_RESTART_SLEEP_TIME >$CONTINUOUS_LOG_DIR/`date +%Y%m%d%H%M%S`_`hostname`_magitator.out 2>$CONTINUOUS_LOG_DIR/`date +%Y%m%d%H%M%S`_`hostname`_magitator.err &" -m - $ACCUMULO_USER
+elif [[ "`whoami`" == $ACCUMULO_USER ]]; then
+ # Just run the magitator if we're the accumulo user
+ nohup $CONTINUOUS_CONF_DIR/magitator.pl $MASTER_KILL_SLEEP_TIME $MASTER_RESTART_SLEEP_TIME >$CONTINUOUS_LOG_DIR/`date +%Y%m%d%H%M%S`_`hostname`_magitator.out 2>$CONTINUOUS_LOG_DIR/`date +%Y%m%d%H%M%S`_`hostname`_magitator.err &
+else
+ # Not root, and not the accumulo user, hope you can sudo to it
+ sudo -m -u $ACCUMULO_USER "nohup $CONTINUOUS_CONF_DIR/magitator.pl $MASTER_KILL_SLEEP_TIME $MASTER_RESTART_SLEEP_TIME >$CONTINUOUS_LOG_DIR/`date +%Y%m%d%H%M%S`_`hostname`_magitator.out 2>$CONTINUOUS_LOG_DIR/`date +%Y%m%d%H%M%S`_`hostname`_magitator.err &"
+fi
http://git-wip-us.apache.org/repos/asf/accumulo/blob/35ff0e50/test/system/continuous/stop-agitator.sh
----------------------------------------------------------------------
diff --git a/test/system/continuous/stop-agitator.sh b/test/system/continuous/stop-agitator.sh
index b853a55..8ce448e 100755
--- a/test/system/continuous/stop-agitator.sh
+++ b/test/system/continuous/stop-agitator.sh
@@ -1,5 +1,4 @@
#! /usr/bin/env bash
-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -18,5 +17,10 @@
CONTINUOUS_CONF_DIR=${CONTINUOUS_CONF_DIR:-$ACCUMULO_HOME/test/system/continuous/}
. $CONTINUOUS_CONF_DIR/continuous-env.sh
-pkill -f agitator.pl
+# Try to use sudo when we wouldn't normally be able to kill the processes
+if [[ ("`whoami`" != "root") && ("`whoami`" != $ACCUMULO_USER) ]]; then
+ sudo -u $ACCUMULO_USER pkill -f agitator.pl
+else
+ pkill -f agitator.pl
+fi
[3/5] git commit: ACCUMULO-1960 Update README for continuous ingest
for agitator improvements.
Posted by el...@apache.org.
ACCUMULO-1960 Update README for continuous ingest for agitator improvements.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0817cdc7
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0817cdc7
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0817cdc7
Branch: refs/heads/master
Commit: 0817cdc73d3dcfd5cfef53b57476a89f42f25490
Parents: 57af813
Author: Josh Elser <el...@apache.org>
Authored: Tue Dec 3 21:53:32 2013 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Tue Dec 3 21:53:32 2013 -0500
----------------------------------------------------------------------
test/system/continuous/README | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0817cdc7/test/system/continuous/README
----------------------------------------------------------------------
diff --git a/test/system/continuous/README b/test/system/continuous/README
index 62af7e3..713c18f 100644
--- a/test/system/continuous/README
+++ b/test/system/continuous/README
@@ -35,7 +35,13 @@ service that continually collect statistics about accumulo and HDFS.
stop-stats.sh
Optionally, start the agitator to periodically kill the tabletserver and/or datanode
-process(es) on random nodes. This must be run as root.
+process(es) on random nodes. You can run this script as root and it will properly start
+processes as the user you configured in continuous-env.sh (HDFS_USER for the Datanode and
+ACCUMULO_USER for Accumulo processes). If you run it as yourself and the HDFS_USER and
+ACCUMULO_USER values are the same as your user, the agitator will not change users. In
+the case where you run the agitator as a non-privileged user which isn't the same as HDFS_USER
+nor ACCUMULO_USER, the agitator will attempt to `sudo` to these users which relies on correct
+configuration of sudo.
start-agitator.sh
stop-agitator.sh
[4/5] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5312aea5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5312aea5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5312aea5
Branch: refs/heads/master
Commit: 5312aea508e15df61e9a1f1e4c77b771e63a9a7d
Parents: 8a34937 0817cdc
Author: Josh Elser <el...@apache.org>
Authored: Tue Dec 3 21:57:53 2013 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Tue Dec 3 21:57:53 2013 -0500
----------------------------------------------------------------------
test/system/continuous/README | 8 ++-
test/system/continuous/agitator.pl | 71 ++++++++++++++++----
.../system/continuous/continuous-env.sh.example | 3 +-
test/system/continuous/magitator.pl | 2 +-
test/system/continuous/start-agitator.sh | 14 +++-
test/system/continuous/stop-agitator.sh | 8 ++-
6 files changed, 85 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5312aea5/test/system/continuous/start-agitator.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5312aea5/test/system/continuous/stop-agitator.sh
----------------------------------------------------------------------
diff --cc test/system/continuous/stop-agitator.sh
index 72d9980,8ce448e..aa132c2
--- a/test/system/continuous/stop-agitator.sh
+++ b/test/system/continuous/stop-agitator.sh
@@@ -15,19 -14,13 +14,24 @@@
# See the License for the specific language governing permissions and
# limitations under the License.
-CONTINUOUS_CONF_DIR=${CONTINUOUS_CONF_DIR:-$ACCUMULO_HOME/test/system/continuous/}
+# Start: Resolve Script Directory
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "${SOURCE}" ]; do # resolve $SOURCE until the file is no longer a symlink
+ bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )"
+ SOURCE="$(readlink "${SOURCE}")"
+ [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )"
+script=$( basename "${SOURCE}" )
+# Stop: Resolve Script Directory
+
+CONTINUOUS_CONF_DIR=${CONTINUOUS_CONF_DIR:-${bin}}
. $CONTINUOUS_CONF_DIR/continuous-env.sh
- pkill -f agitator.pl
+ # Try to use sudo when we wouldn't normally be able to kill the processes
+ if [[ ("`whoami`" != "root") && ("`whoami`" != $ACCUMULO_USER) ]]; then
+ sudo -u $ACCUMULO_USER pkill -f agitator.pl
+ else
+ pkill -f agitator.pl
+ fi
[5/5] git commit: Merge branch '1.6.0-SNAPSHOT'
Posted by el...@apache.org.
Merge branch '1.6.0-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5b84dc2a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5b84dc2a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5b84dc2a
Branch: refs/heads/master
Commit: 5b84dc2a1d67299f3f97236e7476d404c6c4e988
Parents: 5a8a28a 5312aea
Author: Josh Elser <el...@apache.org>
Authored: Tue Dec 3 22:01:55 2013 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Tue Dec 3 22:01:55 2013 -0500
----------------------------------------------------------------------
test/system/continuous/README | 8 ++-
test/system/continuous/agitator.pl | 71 ++++++++++++++++----
.../system/continuous/continuous-env.sh.example | 3 +-
test/system/continuous/magitator.pl | 2 +-
test/system/continuous/start-agitator.sh | 14 +++-
test/system/continuous/stop-agitator.sh | 8 ++-
6 files changed, 85 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
[2/5] git commit: ACCUMULO-1960 Use $ACCUMULO_CONF_DIR/gc to kill the
gc, not $ACCUMULO_CONF_DIR/masters
Posted by el...@apache.org.
ACCUMULO-1960 Use $ACCUMULO_CONF_DIR/gc to kill the gc, not $ACCUMULO_CONF_DIR/masters
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/57af8138
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/57af8138
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/57af8138
Branch: refs/heads/master
Commit: 57af8138a88c8753693de3a9ce9ba24dbf6a6be6
Parents: 35ff0e5
Author: Josh Elser <el...@apache.org>
Authored: Tue Dec 3 21:18:29 2013 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Tue Dec 3 21:18:29 2013 -0500
----------------------------------------------------------------------
test/system/continuous/magitator.pl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/57af8138/test/system/continuous/magitator.pl
----------------------------------------------------------------------
diff --git a/test/system/continuous/magitator.pl b/test/system/continuous/magitator.pl
index 921c4d1..e26d143 100755
--- a/test/system/continuous/magitator.pl
+++ b/test/system/continuous/magitator.pl
@@ -60,7 +60,7 @@ while(1){
$cmd = "pssh -h $ACCUMULO_CONF_DIR/masters \"pkill -f '[ ]org.apache.accumulo.start.*master'\" < /dev/null";
print "$t $cmd\n";
system($cmd);
- $cmd = "pssh -h $ACCUMULO_CONF_DIR/masters \"pkill -f '[ ]org.apache.accumulo.start.*gc'\" < /dev/null";
+ $cmd = "pssh -h $ACCUMULO_CONF_DIR/gc \"pkill -f '[ ]org.apache.accumulo.start.*gc'\" < /dev/null";
print "$t $cmd\n";
system($cmd);
}