You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by vi...@apache.org on 2013/11/08 16:53:40 UTC
[1/2] git commit: ACCUMULO-1658 System integration tests should
default ACCUMULO_CONF_DIR to ACCUMULO_HOME/conf
Updated Branches:
refs/heads/1.5.1-SNAPSHOT 060188a74 -> 422aaaa7c
ACCUMULO-1658 System integration tests should default ACCUMULO_CONF_DIR to ACCUMULO_HOME/conf
Signed-off-by: John Vines <jv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6648056c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6648056c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6648056c
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 6648056cac0a3d271744ab322e8fede221c1f792
Parents: 579c274
Author: ofishel <ry...@cloudera.com>
Authored: Wed Oct 30 08:57:32 2013 -0700
Committer: John Vines <jv...@gmail.com>
Committed: Fri Nov 8 10:49:42 2013 -0500
----------------------------------------------------------------------
test/system/auto/TestUtils.py | 20 +++++++++++++-------
test/system/continuous/agitator.pl | 6 ++++++
.../system/continuous/continuous-env.sh.example | 1 +
test/system/continuous/magitator.pl | 7 ++++++-
test/system/continuous/mapred-setup.sh | 4 ++++
test/system/continuous/start-stats.sh | 4 ++++
test/system/randomwalk/bin/reset-cluster.sh | 4 ++++
test/system/randomwalk/bin/start-all.sh | 4 ++++
test/system/randomwalk/bin/start-local.sh | 4 ++++
test/system/scalability/run.py | 3 +++
10 files changed, 49 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6648056c/test/system/auto/TestUtils.py
----------------------------------------------------------------------
diff --git a/test/system/auto/TestUtils.py b/test/system/auto/TestUtils.py
index f953bca..c10c090 100755
--- a/test/system/auto/TestUtils.py
+++ b/test/system/auto/TestUtils.py
@@ -42,12 +42,18 @@ ACCUMULO_HOME = os.path.dirname(__file__)
ACCUMULO_HOME = os.path.join(ACCUMULO_HOME, *(os.path.pardir,)*3)
ACCUMULO_HOME = os.path.realpath(ACCUMULO_HOME)
ACCUMULO_DIR = "/user/" + os.getenv('LOGNAME') + "/accumulo-" + ID
+if os.getenv('ACCUMULO_CONF_DIR'):
+ ACCUMULO_CONF_DIR = os.getenv('ACCUMULO_CONF_DIR')
+else:
+ ACCUMULO_CONF_DIR = os.path.join(ACCUMULO_HOME, 'conf')
SITE = "test-" + ID
WALOG = os.path.join(ACCUMULO_HOME, 'walogs', ID)
-LOG_PROPERTIES= os.path.join(ACCUMULO_HOME, 'conf', 'log4j.properties')
-LOG_GENERIC = os.path.join(ACCUMULO_HOME, 'conf', 'generic_logger.xml')
-LOG_MONITOR = os.path.join(ACCUMULO_HOME, 'conf', 'monitor_logger.xml')
+
+LOG_PROPERTIES= os.path.join(ACCUMULO_CONF_DIR, 'log4j.properties')
+LOG_GENERIC = os.path.join(ACCUMULO_CONF_DIR, 'generic_logger.xml')
+LOG_MONITOR = os.path.join(ACCUMULO_CONF_DIR, 'conf', 'monitor_logger.xml')
+
General_CLASSPATH = ("$ACCUMULO_CONF_DIR,$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar, $ACCUMULO_HOME/lib/[^.].*.jar, $ZOOKEEPER_HOME/zookeeper[^.].*.jar,"
"$HADOOP_HOME/conf,$HADOOP_HOME/[^.].*.jar, $HADOOP_HOME/lib/[^.].*.jar")
@@ -58,7 +64,8 @@ ROOT_PASSWORD = 'secret'
INSTANCE_NAME=ID
ZOOKEEPERS = socket.getfqdn()
-accumulo_site = os.path.join(ACCUMULO_HOME, 'conf', 'accumulo-site.xml')
+accumulo_site = os.path.join(ACCUMULO_CONF_DIR, 'accumulo-site.xml')
+
if os.path.exists(accumulo_site):
import config
ZOOKEEPERS = config.parse(accumulo_site).get('instance.zookeeper.host', ZOOKEEPERS)
@@ -238,9 +245,8 @@ class TestUtilsMixin:
self.pkill(host, 'org.apache.accumulo.start', signal)
def create_config_file(self, settings):
- fp = open(os.path.join(ACCUMULO_HOME, 'conf', SITE),
- 'w')
- fp.write('<configuration>\n')
+ fp = open(os.path.join(ACCUMULO_CONF_DIR, SITE), 'w')
+ fp.write('<configuration>\n')
settings = self.settings.copy()
settings.update({ 'instance.zookeeper.host': ZOOKEEPERS,
'instance.dfs.dir': ACCUMULO_DIR,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6648056c/test/system/continuous/agitator.pl
----------------------------------------------------------------------
diff --git a/test/system/continuous/agitator.pl b/test/system/continuous/agitator.pl
index c074665..e838bc6 100755
--- a/test/system/continuous/agitator.pl
+++ b/test/system/continuous/agitator.pl
@@ -25,6 +25,12 @@ if(scalar(@ARGV) != 4 && scalar(@ARGV) != 2){
$ACCUMULO_HOME="../../..";
+if(defined $ENV{'ACCUMULO_CONF_DIR'}){
+ $ACCUMULO_CONF_DIR = $ENV{'ACCUMULO_CONF_DIR'};
+}else{
+ $ACCUMULO_CONF_DIR = $ACCUMULO_HOME . '/conf';
+}
+
$sleep1 = $ARGV[0];
$sleep2 = $ARGV[1];
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6648056c/test/system/continuous/continuous-env.sh.example
----------------------------------------------------------------------
diff --git a/test/system/continuous/continuous-env.sh.example b/test/system/continuous/continuous-env.sh.example
index 76f2632..830ae86 100644
--- a/test/system/continuous/continuous-env.sh.example
+++ b/test/system/continuous/continuous-env.sh.example
@@ -18,6 +18,7 @@
# Inherit values from environment if they are already set.
HADOOP_HOME=${HADOOP_HOME:-/opt/hadoop}
ACCUMULO_HOME=${ACCUMULO_HOME:-/opt/accumulo}
+ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR:-$ACCUMULO_HOME/conf}
JAVA_HOME=${JAVA_HOME:-/opt/java}
ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/opt/zookeeper}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6648056c/test/system/continuous/magitator.pl
----------------------------------------------------------------------
diff --git a/test/system/continuous/magitator.pl b/test/system/continuous/magitator.pl
index a0cab60..921c4d1 100755
--- a/test/system/continuous/magitator.pl
+++ b/test/system/continuous/magitator.pl
@@ -23,9 +23,14 @@ if(scalar(@ARGV) != 2){
exit(1);
}
-
$ACCUMULO_HOME="../../..";
+if(defined $ENV{'ACCUMULO_CONF_DIR'}){
+ $ACCUMULO_CONF_DIR = $ENV{'ACCUMULO_CONF_DIR'};
+}else{
+ $ACCUMULO_CONF_DIR = $ACCUMULO_HOME . '/conf';
+}
+
$sleep1 = $ARGV[0];
$sleep2 = $ARGV[1];
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6648056c/test/system/continuous/mapred-setup.sh
----------------------------------------------------------------------
diff --git a/test/system/continuous/mapred-setup.sh b/test/system/continuous/mapred-setup.sh
index e14d221..f4f1be0 100755
--- a/test/system/continuous/mapred-setup.sh
+++ b/test/system/continuous/mapred-setup.sh
@@ -18,6 +18,10 @@
# a little helper script that other scripts can source to setup
# for running a map reduce job
+if [ -z "$ACCUMULO_CONF_DIR" ]; then
+ ACCUMULO_CONF_DIR=${ACCUMULO_HOME}/conf
+fi
+
CONTINUOUS_CONF_DIR=${CONTINUOUS_CONF_DIR:-$ACCUMULO_HOME/test/system/continuous/}
. $CONTINUOUS_CONF_DIR/continuous-env.sh
. $ACCUMULO_CONF_DIR/accumulo-env.sh
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6648056c/test/system/continuous/start-stats.sh
----------------------------------------------------------------------
diff --git a/test/system/continuous/start-stats.sh b/test/system/continuous/start-stats.sh
index 5ece7f5..d089f70 100755
--- a/test/system/continuous/start-stats.sh
+++ b/test/system/continuous/start-stats.sh
@@ -15,6 +15,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+if [ -z "${ACCUMULO_CONF_DIR}" ]; then
+ ACCUMULO_CONF_DIR=${ACCUMULO_HOME}/conf
+fi
+
CONTINUOUS_CONF_DIR=${CONTINUOUS_CONF_DIR:-$ACCUMULO_HOME/test/system/continuous/}
. $CONTINUOUS_CONF_DIR/continuous-env.sh
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6648056c/test/system/randomwalk/bin/reset-cluster.sh
----------------------------------------------------------------------
diff --git a/test/system/randomwalk/bin/reset-cluster.sh b/test/system/randomwalk/bin/reset-cluster.sh
index 83dfcee..69a13b9 100755
--- a/test/system/randomwalk/bin/reset-cluster.sh
+++ b/test/system/randomwalk/bin/reset-cluster.sh
@@ -34,6 +34,10 @@ this="$bin/$script"
ACCUMULO_HOME=`dirname "$this"`/../../../..
export ACCUMULO_HOME=`cd $ACCUMULO_HOME; pwd`
+if [ -z "$ACCUMULO_CONF_DIR" ] ; then
+ $ACCUMULO_CONF_DIR=${ACCUMULO_HOME}/conf
+fi
+
if [ -f $ACCUMULO_CONF_DIR/accumulo-env.sh ] ; then
. $ACCUMULO_CONF_DIR/accumulo-env.sh
fi
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6648056c/test/system/randomwalk/bin/start-all.sh
----------------------------------------------------------------------
diff --git a/test/system/randomwalk/bin/start-all.sh b/test/system/randomwalk/bin/start-all.sh
index 67296de..24c29ad 100755
--- a/test/system/randomwalk/bin/start-all.sh
+++ b/test/system/randomwalk/bin/start-all.sh
@@ -34,6 +34,10 @@ this="$bin/$script"
ACCUMULO_HOME=`dirname "$this"`/../../../..
export ACCUMULO_HOME=`cd $ACCUMULO_HOME; pwd`
+if [ -z "${ACCUMULO_CONF_DIR}" ]; then
+ export ACCUMULO_CONF_DIR=${ACCUMULO_HOME}/conf
+fi
+
if [ -f $ACCUMULO_CONF_DIR/accumulo-env.sh ] ; then
. $ACCUMULO_CONF_DIR/accumulo-env.sh
fi
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6648056c/test/system/randomwalk/bin/start-local.sh
----------------------------------------------------------------------
diff --git a/test/system/randomwalk/bin/start-local.sh b/test/system/randomwalk/bin/start-local.sh
index c02b942..9e59966 100755
--- a/test/system/randomwalk/bin/start-local.sh
+++ b/test/system/randomwalk/bin/start-local.sh
@@ -35,6 +35,10 @@ this="$bin/$script"
ACCUMULO_HOME=`dirname "$this"`/../../../..
export ACCUMULO_HOME=`cd $ACCUMULO_HOME; pwd`
+if [ -z "$ACCUMULO_CONF_DIR" ] ; then
+ export ACCUMULO_CONF_DIR=${ACCUMULO_HOME}/conf
+fi
+
if [ -f $ACCUMULO_CONF_DIR/accumulo-env.sh ] ; then
. $ACCUMULO_CONF_DIR/accumulo-env.sh
fi
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6648056c/test/system/scalability/run.py
----------------------------------------------------------------------
diff --git a/test/system/scalability/run.py b/test/system/scalability/run.py
index d0f9f51..ac45aa2 100755
--- a/test/system/scalability/run.py
+++ b/test/system/scalability/run.py
@@ -173,6 +173,9 @@ def main():
if not os.getenv('ACCUMULO_HOME'):
raise 'ACCUMULO_HOME needs to be set!'
+ if not os.getenv('ACCUMULO_CONF_DIR'):
+ os.environ['ACCUMULO_CONF_DIR'] = os.path.join(os.getenv('ACCUMULO_HOME'), 'conf')
+
if not os.getenv('HADOOP_HOME'):
raise 'HADOOP_HOME needs to be set!'
[2/2] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Posted by vi...@apache.org.
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
test/system/auto/TestUtils.py
test/system/continuous/agitator.pl
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/422aaaa7
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/422aaaa7
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/422aaaa7
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 422aaaa7c35de07c280d496e489fbab541edeea3
Parents: 060188a 6648056
Author: John Vines <jv...@gmail.com>
Authored: Fri Nov 8 10:53:24 2013 -0500
Committer: John Vines <jv...@gmail.com>
Committed: Fri Nov 8 10:53:24 2013 -0500
----------------------------------------------------------------------
test/system/auto/TestUtils.py | 20 +++++++++++++-------
test/system/continuous/agitator.pl | 6 ++++++
.../system/continuous/continuous-env.sh.example | 1 +
test/system/continuous/magitator.pl | 7 ++++++-
test/system/continuous/mapred-setup.sh | 4 ++++
test/system/continuous/start-stats.sh | 4 ++++
test/system/randomwalk/bin/reset-cluster.sh | 4 ++++
test/system/randomwalk/bin/start-all.sh | 4 ++++
test/system/randomwalk/bin/start-local.sh | 4 ++++
test/system/scalability/run.py | 3 +++
10 files changed, 49 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/422aaaa7/test/system/auto/TestUtils.py
----------------------------------------------------------------------
diff --cc test/system/auto/TestUtils.py
index c70baf1,c10c090..df112c5
--- a/test/system/auto/TestUtils.py
+++ b/test/system/auto/TestUtils.py
@@@ -42,33 -42,21 +42,39 @@@ ACCUMULO_HOME = os.path.dirname(__file_
ACCUMULO_HOME = os.path.join(ACCUMULO_HOME, *(os.path.pardir,)*3)
ACCUMULO_HOME = os.path.realpath(ACCUMULO_HOME)
ACCUMULO_DIR = "/user/" + os.getenv('LOGNAME') + "/accumulo-" + ID
+ if os.getenv('ACCUMULO_CONF_DIR'):
+ ACCUMULO_CONF_DIR = os.getenv('ACCUMULO_CONF_DIR')
+ else:
+ ACCUMULO_CONF_DIR = os.path.join(ACCUMULO_HOME, 'conf')
SITE = "test-" + ID
- LOG_PROPERTIES= os.path.join(ACCUMULO_HOME, 'conf', 'log4j.properties')
- LOG_GENERIC = os.path.join(ACCUMULO_HOME, 'conf', 'generic_logger.xml')
- LOG_MONITOR = os.path.join(ACCUMULO_HOME, 'conf', 'monitor_logger.xml')
-WALOG = os.path.join(ACCUMULO_HOME, 'walogs', ID)
-
+ LOG_PROPERTIES= os.path.join(ACCUMULO_CONF_DIR, 'log4j.properties')
+ LOG_GENERIC = os.path.join(ACCUMULO_CONF_DIR, 'generic_logger.xml')
+ LOG_MONITOR = os.path.join(ACCUMULO_CONF_DIR, 'conf', 'monitor_logger.xml')
+General_CLASSPATH = """
++$ACCUMULO_CONF_DIR,
+$ACCUMULO_HOME/server/target/classes/,
+ $ACCUMULO_HOME/core/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-core.jar,
+ $ACCUMULO_HOME/start/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-start.jar,
+ $ACCUMULO_HOME/fate/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-fate.jar,
+ $ACCUMULO_HOME/examples/simple/target/classes,
+ $ACCUMULO_HOME/lib/accumulo-examples-simple.jar,
+ $ACCUMULO_HOME/lib/[^.].*.jar,
+ $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/yarn/.*.jar,
+"""
-General_CLASSPATH = ("$ACCUMULO_CONF_DIR,$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar, $ACCUMULO_HOME/lib/[^.].*.jar, $ZOOKEEPER_HOME/zookeeper[^.].*.jar,"
-"$HADOOP_HOME/conf,$HADOOP_HOME/[^.].*.jar, $HADOOP_HOME/lib/[^.].*.jar")
+
log = logging.getLogger('test.auto')
ROOT = 'root'
@@@ -245,12 -242,11 +252,11 @@@ class TestUtilsMixin
def stop_accumulo(self, signal=signal.SIGHUP):
log.info('killing accumulo processes everywhere')
for host in self.hosts:
- self.pkill(host, 'org.apache.accumulo.start', signal)
+ self.pkill(host, 'accumulo.config.file', signal)
def create_config_file(self, settings):
- fp = open(os.path.join(ACCUMULO_HOME, 'conf', SITE),
- 'w')
- fp.write('<configuration>\n')
+ fp = open(os.path.join(ACCUMULO_CONF_DIR, SITE), 'w')
+ fp.write('<configuration>\n')
settings = self.settings.copy()
settings.update({ 'instance.zookeeper.host': ZOOKEEPERS,
'instance.dfs.dir': ACCUMULO_DIR,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/422aaaa7/test/system/continuous/agitator.pl
----------------------------------------------------------------------
diff --cc test/system/continuous/agitator.pl
index d9c5395,e838bc6..c779983
--- a/test/system/continuous/agitator.pl
+++ b/test/system/continuous/agitator.pl
@@@ -24,34 -24,16 +24,40 @@@ if(scalar(@ARGV) != 4 && scalar(@ARGV)
}
$ACCUMULO_HOME="../../..";
+$HADOOP_PREFIX=$ENV{"HADOOP_PREFIX"};
+
+@sleeprange1 = split(/:/, $ARGV[0]);
+$sleep1 = $sleeprange1[0];
+
+@sleeprange2 = split(/:/, $ARGV[1]);
+$sleep2 = $sleeprange2[0];
+
+if(scalar(@sleeprange1) > 1){
+ $sleep1max = $sleeprange1[1] + 1;
+}else{
+ $sleep1max = $sleep1;
+}
+
+if($sleep1 > $sleep1max){
+ die("sleep1 > sleep1max $sleep1 > $sleep1max");
+}
+
+if(scalar(@sleeprange2) > 1){
+ $sleep2max = $sleeprange2[1] + 1;
+}else{
+ $sleep2max = $sleep2;
+}
+
+if($sleep2 > $sleep2max){
+ die("sleep2 > sleep2max $sleep2 > $sleep2max");
+}
+ if(defined $ENV{'ACCUMULO_CONF_DIR'}){
- $ACCUMULO_CONF_DIR = $ENV{'ACCUMULO_CONF_DIR'};
++ $ACCUMULO_CONF_DIR = $ENV{'ACCUMULO_CONF_DIR'};
+ }else{
+ $ACCUMULO_CONF_DIR = $ACCUMULO_HOME . '/conf';
+ }
+
-$sleep1 = $ARGV[0];
-$sleep2 = $ARGV[1];
-
if(scalar(@ARGV) == 4){
$minKill = $ARGV[2];
$maxKill = $ARGV[3];
http://git-wip-us.apache.org/repos/asf/accumulo/blob/422aaaa7/test/system/continuous/continuous-env.sh.example
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/422aaaa7/test/system/continuous/mapred-setup.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/422aaaa7/test/system/continuous/start-stats.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/422aaaa7/test/system/randomwalk/bin/reset-cluster.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/422aaaa7/test/system/randomwalk/bin/start-all.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/422aaaa7/test/system/randomwalk/bin/start-local.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/422aaaa7/test/system/scalability/run.py
----------------------------------------------------------------------