You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mw...@apache.org on 2017/02/17 21:58:39 UTC
accumulo-testing git commit: ACCUMULO-4581 Fixing several bugs in
agitator scripts
Repository: accumulo-testing
Updated Branches:
refs/heads/master 28ca75a36 -> 7495ba604
ACCUMULO-4581 Fixing several bugs in agitator scripts
* Updated how perl scripts are called in bash
* Removed analyze-missing.pl as it relies on centralized logging
* Renamed accumulo home and conf dir variables names in agitator
perl scripts
Project: http://git-wip-us.apache.org/repos/asf/accumulo-testing/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-testing/commit/7495ba60
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-testing/tree/7495ba60
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-testing/diff/7495ba60
Branch: refs/heads/master
Commit: 7495ba6048ead216e578e8b6cdcc86d4ae8b1b47
Parents: 28ca75a
Author: Mike Walch <mw...@apache.org>
Authored: Wed Feb 1 15:53:47 2017 -0500
Committer: Mike Walch <mw...@apache.org>
Committed: Mon Feb 6 15:55:26 2017 -0500
----------------------------------------------------------------------
bin/accumulo-testing | 28 ++++---
conf/accumulo-testing-env.sh.example | 12 ++-
libexec/analyze-missing.pl | 127 ------------------------------
libexec/datanode-agitator.pl | 18 ++---
libexec/master-agitator.pl | 25 +++---
libexec/tserver-agitator.pl | 22 +++---
6 files changed, 49 insertions(+), 183 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/7495ba60/bin/accumulo-testing
----------------------------------------------------------------------
diff --git a/bin/accumulo-testing b/bin/accumulo-testing
index 7cf7206..84a1d40 100755
--- a/bin/accumulo-testing
+++ b/bin/accumulo-testing
@@ -116,6 +116,8 @@ function determine_mapred_main() {
}
function start_agitator() {
+ hash pssh 2>/dev/null || { echo >&2 "The agitator requires pssh to be installed. Aborting."; exit 1; }
+
mkdir -p "${at_home}/logs"
log_base="${at_home}/logs/$(date +%Y%m%d%H%M%S)_$(hostname)"
libexec="${at_home}/libexec"
@@ -132,17 +134,23 @@ function start_agitator() {
su -c "$master_cmd >${master_log}.out 2>${master_log}.err" & -m - "$AGTR_ACCUMULO_USER"
su -c "$tserver_cmd >${tserver_log}.out 2>${tserver_log}.err" & -m - "$AGTR_ACCUMULO_USER"
su -c "$datanode_cmd >${datanode_log}.out 2>${datanode_log}.err" & -m - "$AGTR_HDFS_USER"
- elif [[ $AGITATOR_USER == "$AGTR_ACCUMULO_USER" ]]; then
- echo "Running master-agitator and tserver-agitator as $AGITATOR_USER Running datanode-agitator as $AGTR_HDFS_USER using sudo."
- $master_cmd > "${master_log}.out" 2> "${master_log}.err" &
- $tserver_cmd > "${tserver_log}.out" 2> "${tserver_log}.err" &
- sudo -u "$AGTR_HDFS_USER" "$datanode_cmd" > "${datanode_log}.out" 2> "${datanode_log}.err" &
else
- echo "Running master-agitator and tserver-agitator as $AGTR_ACCUMULO_USER using sudo. Running datanode-agitator as $AGTR_HDFS_USER using sudo."
- # Not root, and not the accumulo user, hope you can sudo to it
- sudo -u "$AGTR_ACCUMULO_USER" "$master_cmd" > "${master_log}.out" 2> "${master_log}.err" &
- sudo -u "$AGTR_ACCUMULO_USER" "$tserver_cmd" > "${tserver_log}.out" 2> "${tserver_log}.err" &
- sudo -u "$AGTR_HDFS_USER" "$datanode_cmd" > "${datanode_log}.out" 2> "${datanode_log}.err" & -m - "$AGTR_HDFS_USER"
+ if [[ $AGITATOR_USER == "$AGTR_ACCUMULO_USER" ]]; then
+ echo "Running master-agitator and tserver-agitator as $AGITATOR_USER"
+ $master_cmd > "${master_log}.out" 2> "${master_log}.err" &
+ $tserver_cmd > "${tserver_log}.out" 2> "${tserver_log}.err" &
+ else
+ echo "Running master-agitator and tserver-agitator as $AGTR_ACCUMULO_USER using sudo."
+ sudo -u "$AGTR_ACCUMULO_USER" $master_cmd > "${master_log}.out" 2> "${master_log}.err" &
+ sudo -u "$AGTR_ACCUMULO_USER" $tserver_cmd > "${tserver_log}.out" 2> "${tserver_log}.err" &
+ fi
+ if [[ $AGITATOR_USER == "$AGTR_HDFS_USER" ]]; then
+ echo "Running datanode-agitator as $AGITATOR_USER"
+ $datanode_cmd > "${datanode_log}.out" 2> "${datanode_log}.err" &
+ else
+ echo "Running datanode-agitator as $AGTR_HDFS_USER using sudo."
+ sudo -u "$AGTR_HDFS_USER" $datanode_cmd > "${datanode_log}.out" 2> "${datanode_log}.err" &
+ fi
fi
if ${AGTR_HDFS:-false} ; then
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/7495ba60/conf/accumulo-testing-env.sh.example
----------------------------------------------------------------------
diff --git a/conf/accumulo-testing-env.sh.example b/conf/accumulo-testing-env.sh.example
index 3f71463..e03c973 100644
--- a/conf/accumulo-testing-env.sh.example
+++ b/conf/accumulo-testing-env.sh.example
@@ -13,20 +13,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Hadoop
-# ======
+# General
+# =======
test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
-test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR=/path/to/hadoop/etc/hadoop
-
-# Accumulo
-# ========
-# Set the Accumulo version that should be included in the shaded jar
+# Versions set below will be what is included in the shaded jar
export ACCUMULO_VERSION=`accumulo version`
export HADOOP_VERSION=`hadoop version | head -n1 | awk '{print $2}'`
export ZOOKEEPER_VERSION=3.4.9
# Agitator
# ========
+# Agitator needs know where Accumulo is installed
+test -z "$ACCUMULO_HOME" && export ACCUMULO_HOME=/path/to/accumulo
# Accumulo user
AGTR_ACCUMULO_USER=$(whoami)
# Time (in minutes) between killing Accumulo masters
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/7495ba60/libexec/analyze-missing.pl
----------------------------------------------------------------------
diff --git a/libexec/analyze-missing.pl b/libexec/analyze-missing.pl
deleted file mode 100755
index 5cce1b1..0000000
--- a/libexec/analyze-missing.pl
+++ /dev/null
@@ -1,127 +0,0 @@
-#! /usr/bin/env perl
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-use POSIX qw(strftime);
-
-if(scalar(@ARGV) != 4){
- print "Usage : analyze-missing.pl <accumulo home> <continuous log dir> <user> <pass> \n";
- exit(1);
-}
-
-$ACCUMULO_HOME=$ARGV[0];
-$CONTINUOUS_LOG_DIR=$ARGV[1];
-$USER=$ARGV[2];
-$PASS=$ARGV[3];
-
-
-@missing = `grep MIS $CONTINUOUS_LOG_DIR/*.err`;
-
-
-
-for $miss (@missing) {
- chomp($miss);
- ($file, $type, $time, $row) = split(/[: ]/, $miss);
-
- substr($file, -3, 3, "out");
-
- $prevRowLine = `grep -B 1 $row $file | grep SRQ | grep -v $row`;
-
- @prla = split(/\s+/, $prevRowLine);
- $prevRow = $prla[2];
-# print $prevRow."\n";
-
- $aScript = `mktemp /tmp/miss_script.XXXXXXXXXX`;
- chomp($aScript);
- open(AS, ">$aScript") || die;
-
- print AS "table ci\n";
- print AS "scan -b $prevRow -e $prevRow\n";
- print AS "scan -b $row -e $row\n";
- print AS "quit\n";
- close(AS);
-
- $exist = 0;
- $ingestIDSame = 0;
- $ingestId = "";
- $count = 0;
-
- @entries = `$ACCUMULO_HOME/bin/accumulo shell -u $USER -p $PASS -f $aScript | grep $row`;
- system("rm $aScript");
-
- for $entry (@entries){
- chomp($entry);
- @entryA = split(/[: ]+/, $entry);
- if($entryA[0] eq $row){
- $exist = 1;
-
- if($entryA[4] eq $ingestId){
- $ingestIDSame = 1;
- }
- }else{
- $ingestId = $entryA[4];
- $count = hex($entryA[5]);
- }
- }
-
-
- #look in ingest logs
- @ingestLogs = `ls $CONTINUOUS_LOG_DIR/*ingest*.out`;
- @flushTimes = ();
- chomp(@ingestLogs);
- for $ingestLog (@ingestLogs){
- open(IL, "<$ingestLog") || die;
-
-
- while($firstLine = <IL>){
- chomp($firstLine);
- if($firstLine =~ /UUID.*/){
- last;
- }
- }
-
- @iinfo = split(/\s+/,$firstLine);
- if($iinfo[2] eq $ingestId){
- while($line = <IL>){
- if($line =~ /FLUSH (\d+) \d+ \d+ (\d+) \d+/){
- push(@flushTimes, $1);
- if(scalar(@flushTimes) > 3){
- shift(@flushTimes);
- }
- if($count < $2){
- last;
- }
- }
- }
- }
-
-
-
- close(IL);
-
- if(scalar(@flushTimes) > 0){
- last;
- }
- }
-
- $its0 = strftime "%m/%d/%Y_%H:%M:%S", gmtime($flushTimes[0]/1000);
- $its1 = strftime "%m/%d/%Y_%H:%M:%S", gmtime($flushTimes[1]/1000);
- $mts = strftime "%m/%d/%Y_%H:%M:%S", gmtime($time/1000);
-
- print "$row $exist $ingestIDSame $prevRow $ingestId $its0 $its1 $mts\n";
-}
-
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/7495ba60/libexec/datanode-agitator.pl
----------------------------------------------------------------------
diff --git a/libexec/datanode-agitator.pl b/libexec/datanode-agitator.pl
index a98bb66..bb0b38e 100755
--- a/libexec/datanode-agitator.pl
+++ b/libexec/datanode-agitator.pl
@@ -24,16 +24,16 @@ if(scalar(@ARGV) != 5 && scalar(@ARGV) != 3){
exit(1);
}
-my $ACCUMULO_HOME;
+my $accumuloHome;
if( defined $ENV{'ACCUMULO_HOME'} ){
- $ACCUMULO_HOME = $ENV{'ACCUMULO_HOME'};
+ $accumuloHome = $ENV{'ACCUMULO_HOME'};
} else {
- $cwd=Cwd::cwd();
- $ACCUMULO_HOME=$cwd . '/../../..';
+ print "ERROR: ACCUMULO_HOME needs to be set!";
+ exit(1);
}
$HADOOP_PREFIX=$ARGV[2];
-print "ACCUMULO_HOME=$ACCUMULO_HOME\n";
+print "ACCUMULO_HOME=$accumuloHome\n";
print "HADOOP_PREFIX=$HADOOP_PREFIX\n";
@sleeprange1 = split(/:/, $ARGV[0]);
@@ -62,11 +62,7 @@ if($sleep2 > $sleep2max){
die("sleep2 > sleep2max $sleep2 > $sleep2max");
}
-if(defined $ENV{'ACCUMULO_CONF_DIR'}){
- $ACCUMULO_CONF_DIR = $ENV{'ACCUMULO_CONF_DIR'};
-}else{
- $ACCUMULO_CONF_DIR = $ACCUMULO_HOME . '/conf';
-}
+$accumuloConfDir = $accumuloHome . '/conf';
if(scalar(@ARGV) == 5){
$minKill = $ARGV[3];
@@ -80,7 +76,7 @@ if($minKill > $maxKill){
die("minKill > maxKill $minKill > $maxKill");
}
-@tserversRaw = `cat $ACCUMULO_CONF_DIR/tservers`;
+@tserversRaw = `cat $accumuloConfDir/tservers`;
chomp(@tserversRaw);
for $tserver (@tserversRaw){
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/7495ba60/libexec/master-agitator.pl
----------------------------------------------------------------------
diff --git a/libexec/master-agitator.pl b/libexec/master-agitator.pl
index d87f17e..949cfbb 100755
--- a/libexec/master-agitator.pl
+++ b/libexec/master-agitator.pl
@@ -15,7 +15,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
use POSIX qw(strftime);
use Cwd qw();
@@ -24,24 +23,20 @@ if(scalar(@ARGV) != 2){
exit(1);
}
-my $ACCUMULO_HOME;
+my $accumuloHome;
if( defined $ENV{'ACCUMULO_HOME'} ){
- $ACCUMULO_HOME = $ENV{'ACCUMULO_HOME'};
+ $accumuloHome = $ENV{'ACCUMULO_HOME'};
} else {
- $cwd=Cwd::cwd();
- $ACCUMULO_HOME=$cwd . '/../../..';
+ print "ERROR: ACCUMULO_HOME needs to be set!";
+ exit(1);
}
-if(defined $ENV{'ACCUMULO_CONF_DIR'}){
- $ACCUMULO_CONF_DIR = $ENV{'ACCUMULO_CONF_DIR'};
-}else{
- $ACCUMULO_CONF_DIR = $ACCUMULO_HOME . '/conf';
-}
+$accumuloConfDir = $accumuloHome . '/conf';
$sleep1 = $ARGV[0];
$sleep2 = $ARGV[1];
-@mastersRaw = `cat $ACCUMULO_CONF_DIR/masters`;
+@mastersRaw = `cat $accumuloConfDir/masters`;
chomp(@mastersRaw);
for $master (@mastersRaw){
@@ -64,18 +59,18 @@ while(1){
system($cmd);
}else{
print STDERR "$t Killing all masters\n";
- $cmd = "pssh -h $ACCUMULO_CONF_DIR/masters \"pkill -f '[ ]org.apache.accumulo.start.*master'\" < /dev/null";
+ $cmd = "pssh -h $accumuloConfDir/masters \"pkill -f '[ ]org.apache.accumulo.start.*master'\" < /dev/null";
print "$t $cmd\n";
system($cmd);
$file = '';
- if (-e "$ACCUMULO_CONF_DIR/gc") {
+ if (-e "$accumuloConfDir/gc") {
$file = 'gc';
} else {
$file = 'masters';
}
- $cmd = "pssh -h $ACCUMULO_CONF_DIR/$file \"pkill -f '[ ]org.apache.accumulo.start.*gc'\" < /dev/null";
+ $cmd = "pssh -h $accumuloConfDir/$file \"pkill -f '[ ]org.apache.accumulo.start.*gc'\" < /dev/null";
print "$t $cmd\n";
system($cmd);
}
@@ -84,7 +79,7 @@ while(1){
$t = strftime "%Y%m%d %H:%M:%S", localtime;
print STDERR "$t Running start-all\n";
- $cmd = "pssh -h $ACCUMULO_CONF_DIR/masters \"$ACCUMULO_HOME/bin/accumulo-service master start\" < /dev/null";
+ $cmd = "pssh -h $accumuloConfDir/masters \"$accumuloHome/bin/accumulo-service master start\" < /dev/null";
print "$t $cmd\n";
system($cmd);
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/7495ba60/libexec/tserver-agitator.pl
----------------------------------------------------------------------
diff --git a/libexec/tserver-agitator.pl b/libexec/tserver-agitator.pl
index de29e3a..160824e 100755
--- a/libexec/tserver-agitator.pl
+++ b/libexec/tserver-agitator.pl
@@ -24,15 +24,15 @@ if(scalar(@ARGV) != 4 && scalar(@ARGV) != 2){
exit(1);
}
-my $ACCUMULO_HOME;
+my $accumuloHome;
if( defined $ENV{'ACCUMULO_HOME'} ){
- $ACCUMULO_HOME = $ENV{'ACCUMULO_HOME'};
+ $accumuloHome = $ENV{'ACCUMULO_HOME'};
} else {
- $cwd=Cwd::cwd();
- $ACCUMULO_HOME=$cwd . '/../../..';
+ print "ERROR: ACCUMULO_HOME needs to be set!";
+ exit(1);
}
-print "ACCUMULO_HOME=$ACCUMULO_HOME\n";
+print "ACCUMULO_HOME=$accumuloHome\n";
@sleeprange1 = split(/:/, $ARGV[0]);
$sleep1 = $sleeprange1[0];
@@ -60,11 +60,7 @@ if($sleep2 > $sleep2max){
die("sleep2 > sleep2max $sleep2 > $sleep2max");
}
-if(defined $ENV{'ACCUMULO_CONF_DIR'}){
- $ACCUMULO_CONF_DIR = $ENV{'ACCUMULO_CONF_DIR'};
-}else{
- $ACCUMULO_CONF_DIR = $ACCUMULO_HOME . '/conf';
-}
+$accumuloConfDir = $accumuloHome . '/conf';
if(scalar(@ARGV) == 4){
$minKill = $ARGV[2];
@@ -78,7 +74,7 @@ if($minKill > $maxKill){
die("minKill > maxKill $minKill > $maxKill");
}
-@tserversRaw = `cat $ACCUMULO_CONF_DIR/tservers`;
+@tserversRaw = `cat $accumuloConfDir/tservers`;
chomp(@tserversRaw);
for $tserver (@tserversRaw){
@@ -118,7 +114,7 @@ while(1){
print STDERR "$t Killing tserver on $server\n";
# We're the accumulo user, just run the commandj
- system("ssh $server '$ACCUMULO_HOME/bin/accumulo-service tserver stop'");
+ system("ssh $server '$accumuloHome/bin/accumulo-service tserver kill'");
}
$nextsleep2 = int(rand($sleep2max - $sleep2)) + $sleep2;
@@ -126,7 +122,7 @@ while(1){
$t = strftime "%Y%m%d %H:%M:%S", localtime;
print STDERR "$t Running tup\n";
# restart the as them as the accumulo user
- system("$ACCUMULO_HOME/libexec/cluster.sh start-tservers");
+ system("$accumuloHome/bin/accumulo-cluster start-tservers");
$nextsleep1 = int(rand($sleep1max - $sleep1)) + $sleep1;
sleep($nextsleep1 * 60);