You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by zz...@apache.org on 2013/05/02 03:26:46 UTC
[3/3] git commit: remove test scripts for helix-agent. disable
helix-agent
remove test scripts for helix-agent. disable helix-agent
Project: http://git-wip-us.apache.org/repos/asf/incubator-helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-helix/commit/4ebc0fad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-helix/tree/4ebc0fad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-helix/diff/4ebc0fad
Branch: refs/heads/master
Commit: 4ebc0fad9b9f3135c64c266d7cdae9b65177ce61
Parents: d5289ae
Author: zzhang <zz...@uci.edu>
Authored: Wed May 1 18:26:36 2013 -0700
Committer: zzhang <zz...@uci.edu>
Committed: Wed May 1 18:26:36 2013 -0700
----------------------------------------------------------------------
.../integration-test/config/log4j-info.properties | 23 -
.../integration-test/config/log4j.properties | 25 -
.../config/zookeeper-log4j2file.properties | 26 -
.../integration-test/lib/dds_test_infra.tar.gz | Bin 69424 -> 0 bytes
.../main/scripts/integration-test/log4j.properties | 24 -
.../scripts/integration-test/script/cm_driver.py | 71 -
.../scripts/integration-test/script/dds_driver.py | 1098 ---------
.../integration-test/script/driver_cmd_dict.py | 312 ---
.../scripts/integration-test/script/pexpect.py | 1864 ---------------
.../scripts/integration-test/script/pexpect.pyc | Bin 68256 -> 0 bytes
.../scripts/integration-test/script/utility.py | 813 -------
.../scripts/integration-test/script/utility.pyc | Bin 41258 -> 0 bytes
.../main/scripts/integration-test/setup_env.inc | 37 -
.../scripts/integration-test/testcases/foo_test.py | 38 -
.../testcases/report_pass_fail.inc | 40 -
.../integration-test/testcases/setup_env.inc | 60 -
.../org/apache/helix/agent/TestHelixAgent.java | 3 +-
17 files changed, 2 insertions(+), 4432 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/4ebc0fad/helix-agent/src/main/scripts/integration-test/config/log4j-info.properties
----------------------------------------------------------------------
diff --git a/helix-agent/src/main/scripts/integration-test/config/log4j-info.properties b/helix-agent/src/main/scripts/integration-test/config/log4j-info.properties
deleted file mode 100644
index cca0ae9..0000000
--- a/helix-agent/src/main/scripts/integration-test/config/log4j-info.properties
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=INFO, ConsoleAppender
-log4j.appender.ConsoleAppender=org.apache.log4j.ConsoleAppender
-log4j.appender.ConsoleAppender.layout=org.apache.log4j.PatternLayout
-log4j.appender.ConsoleAppender.layout.ConversionPattern=%d{ISO8601} +%r [%t] (%p) {%c{1}:%M} (%F:%L) %m%n
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/4ebc0fad/helix-agent/src/main/scripts/integration-test/config/log4j.properties
----------------------------------------------------------------------
diff --git a/helix-agent/src/main/scripts/integration-test/config/log4j.properties b/helix-agent/src/main/scripts/integration-test/config/log4j.properties
deleted file mode 100644
index 7e49982..0000000
--- a/helix-agent/src/main/scripts/integration-test/config/log4j.properties
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=CONSOLE, R
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.R=com.linkedin.clustermanager.tools.CLMLogFileAppender
-log4j.appender.R.layout=org.apache.log4j.PatternLayout
-log4j.appender.R.layout.ConversionPattern=%5p [%C:%M] (%F:%L) - %m%n
-log4j.appender.R.File=${user.home}/EspressoLogs/log.txt
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/4ebc0fad/helix-agent/src/main/scripts/integration-test/config/zookeeper-log4j2file.properties
----------------------------------------------------------------------
diff --git a/helix-agent/src/main/scripts/integration-test/config/zookeeper-log4j2file.properties b/helix-agent/src/main/scripts/integration-test/config/zookeeper-log4j2file.properties
deleted file mode 100644
index 2ce940f..0000000
--- a/helix-agent/src/main/scripts/integration-test/config/zookeeper-log4j2file.properties
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-log4j.rootLogger=INFO, ConsoleAppender
-
-log4j.appender.ConsoleAppender=org.apache.log4j.ConsoleAppender
-
-log4j.appender.ConsoleAppender.layout=org.apache.log4j.PatternLayout
-log4j.appender.ConsoleAppender.layout.ConversionPattern=%d{yyyy/MM/dd HH:mm:ss.SSS} %t %p [%c] %m%n
-
-#log4j.logger.org.apache=WARN
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/4ebc0fad/helix-agent/src/main/scripts/integration-test/lib/dds_test_infra.tar.gz
----------------------------------------------------------------------
diff --git a/helix-agent/src/main/scripts/integration-test/lib/dds_test_infra.tar.gz b/helix-agent/src/main/scripts/integration-test/lib/dds_test_infra.tar.gz
deleted file mode 100644
index cc621e0..0000000
Binary files a/helix-agent/src/main/scripts/integration-test/lib/dds_test_infra.tar.gz and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/4ebc0fad/helix-agent/src/main/scripts/integration-test/log4j.properties
----------------------------------------------------------------------
diff --git a/helix-agent/src/main/scripts/integration-test/log4j.properties b/helix-agent/src/main/scripts/integration-test/log4j.properties
deleted file mode 100644
index 562afe0..0000000
--- a/helix-agent/src/main/scripts/integration-test/log4j.properties
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-log4j.rootLogger=CONSOLE, R
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.R=com.linkedin.espresso.cm.tools.CLMLogFileAppender
-log4j.appender.R.layout=org.apache.log4j.PatternLayout
-log4j.appender.R.layout.ConversionPattern=%5p [%C:%M] (%F:%L) - %m%n
-log4j.appender.R.File=${user.home}/EspressoLogs/log.txt
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/4ebc0fad/helix-agent/src/main/scripts/integration-test/script/cm_driver.py
----------------------------------------------------------------------
diff --git a/helix-agent/src/main/scripts/integration-test/script/cm_driver.py b/helix-agent/src/main/scripts/integration-test/script/cm_driver.py
deleted file mode 100755
index 5884976..0000000
--- a/helix-agent/src/main/scripts/integration-test/script/cm_driver.py
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-'''
- Load the dds driver and support file if needed
-
-'''
-__version__ = "$Revision: 0.1 $"
-__date__ = "$Date: 2011/6/27 $"
-
-import os, pdb
-
-#pdb.set_trace()
-
-# Global varaibles
-meta_data_file=".metadata_infra"
-dds_test_infra_tarball="dds_test_infra.tar.gz"
-
-this_file_full_path=os.path.abspath(__file__)
-this_file_dirname=os.path.dirname(this_file_full_path)
-meta_data_file_full_path=os.path.join(this_file_dirname, meta_data_file)
-dds_test_infra_tarball_full_path="%s/../lib/%s" % (this_file_dirname,dds_test_infra_tarball)
-
-need_reload=False
-file_change_time = str(os.path.getmtime(dds_test_infra_tarball_full_path))
-view_root= os.path.abspath("%s/../../../../../" % this_file_dirname) # script dir is 5 levels lower
-if not os.path.exists(os.path.join(view_root,"integration-test")):
- view_root= os.path.abspath("%s/../../" % this_file_dirname) # script dir is 5 levels lower
-if not os.path.exists(os.path.join(view_root,"integration-test")):
- print "VIEW_ROOT %s is not correct" % view_root
- assert False, "VIEW_ROOT %s is not correct" % view_root
-
-if not os.path.exists(meta_data_file_full_path): need_reload = True
-else:
- last_change_time = open(meta_data_file_full_path).readlines()[0].split("=")[-1]
- if file_change_time != last_change_time:
- need_reload = True
-if need_reload:
- open(meta_data_file_full_path,"w").write("change time of %s=%s" % (meta_data_file_full_path, file_change_time))
- # specific to the cm
- os.system("tar zxf %s > /dev/null" % dds_test_infra_tarball_full_path)
- #os.system("tar zxf %s " % dds_test_infra_tarball_full_path)
- integ_java_dir=os.path.join(view_root,"src/test")
- config_dir=os.path.join(this_file_dirname,"../config")
- os.system("cp -rf integ/java %s" % integ_java_dir)
- os.system("rm -rf integ")
- os.system("cp script/* %s" % this_file_dirname)
- os.system("rm -rf script")
- os.system("cp config/* %s" % config_dir)
- os.system("rm -rf config")
-
-os.environ["VIEW_ROOT"]=view_root
-execfile(os.path.join(this_file_dirname,"dds_driver.py"))
-
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/4ebc0fad/helix-agent/src/main/scripts/integration-test/script/dds_driver.py
----------------------------------------------------------------------
diff --git a/helix-agent/src/main/scripts/integration-test/script/dds_driver.py b/helix-agent/src/main/scripts/integration-test/script/dds_driver.py
deleted file mode 100755
index 886c769..0000000
--- a/helix-agent/src/main/scripts/integration-test/script/dds_driver.py
+++ /dev/null
@@ -1,1098 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-'''
- Start and stop dbus2 servers, consumers
- Will handle remote run in the future
-
- bootstrap_relay start/stop
- bootstrap_producer start/stop
- bootstrap_server start/stop
- bootstrap_consumer start/stop, stop_scn, stop_after_secs
- profile_relay
- profile_consumer
-
- zookeeper start/stop/wait_exist/wait_no_exist/wait_value/cmd
-$SCRIPT_DIR/dbus2_driver.py -c zookeeper -o start --zookeeper_server_ports=${zookeeper_server_ports} --cmdline_props="tickTime=2000;initLimit=5;syncLimit=2" --zookeeper_cmds=<semicolon separate list of command> --zookeeper_path= zookeeper_value=
- -. start, parse the port, generate the local file path in var/work/zookeeper_data/1, start, port default from 2181, generate log4j file
- -. stop, find the process id, id is port - 2181 + 1, will stop all the processes
- -. wait, query client and get the status
- -. execute the cmd
-
-'''
-__version__ = "$Revision: 0.1 $"
-__date__ = "$Date: 2010/11/16 $"
-
-import sys, os, fcntl
-import pdb
-import time, copy, re
-from optparse import OptionParser, OptionGroup
-import logging
-import threading
-import pexpect
-from utility import *
-import distutils.dir_util
-
-# Global varaibles
-options=None
-server_host="localhost"
-server_port="8080"
-consumer_host="localhost"
-consumer_port=8081
-consumer_http_start_port=8081 # may need to be changed?
-consumer_jmx_service_start_port=10000 # may need to be changed?
-rmi_registry_port="1099"
-log_file_pattern="%s_%s_%s_%s.%s.log" # testname, component, oper, time, pid
-#stats_cmd_pattern='''jps | grep %%s | awk '{printf "open "$1"\\nbean com.linkedin.databus2:relayId=1408230481,type=OutboundTrafficTotalStats\\nget *"}' | java -jar %s/../lib/jmxterm-1.0-alpha-4-uber.jar -i -n''' % get_this_file_dirname()
-stats_cmd_pattern='''jps -J-Xms5M -J-Xmx5M | grep %%s | awk '{printf "open "$1"\\nbean com.linkedin.databus2:relayId=1408230481,type=OutboundTrafficTotalStats\\nget *"}' | java -jar %s/../lib/jmxterm-1.0-alpha-4-uber.jar -i -n''' % get_this_file_dirname()
-#config_sub_cmd='''dbus2_config_sub.py''' % get_this_file_dirname()
-jmx_cli = None
-
-def zookeeper_opers(oper):
- if options.zookeeper_reset: zookeeper_opers_stop()
- zookeeper_setup(oper)
- globals()["zookeeper_opers_%s" % oper]()
-
-def conf_and_deploy(ant_file):
- ''' to deploy a service only, substitue the cmd_line ops
- explored-war build-app-conf change the conf deploy.only
- '''
- conf_and_deploy_1(ant_file)
-
-def get_stats(pattern):
- ''' called to get stats for a process '''
- pids = [x for x in sys_pipe_call_1("jps | grep %s" % pattern) if x]
- if not pids: my_error("pid for component '%s' ('%s') is not find" % (options.component, pattern))
- pid = pids[0].split()[0]
- get_stats_1(pid, options.jmx_bean, options.jmx_attr)
-
-def wait_event(func, option=None):
- ''' called to wait for '''
- wait_event_1(func(), option)
-
-def producer_wait_event(name, func):
- ''' called to wait for '''
- producer_wait_event_1(name, func())
-
-def shutdown(oper="normal"):
- pid = send_shutdown(server_host, options.http_port or server_port, oper == "force")
- dbg_print("shutdown pid = %s" % (pid))
- ret = wait_for_condition('not process_exist(%s)' % (pid), 120)
-
-def get_wait_timeout():
- if options.timeout: return options.timeout
- else: return 10
-
-def pause_resume_consumer(oper):
- global consumer_port
- if options.component_id: consumer_port=find_open_port(consumer_host, consumer_http_start_port, options.component_id)
- url = "http://%s:%s/pauseConsumer/%s" % (consumer_host, consumer_port, oper)
- out = send_url(url).split("\n")[1]
- dbg_print("out = %s" % out)
- time.sleep(0.1)
-
-def get_bootstrap_db_conn_info():
- return ("bootstrap", "bootstrap", "bootstrap")
-
-lock_tab_sql_file = tempfile.mkstemp()[1]
-def producer_lock_tab(oper):
- dbname, user, passwd = get_bootstrap_db_conn_info()
- if oper == "lock" or oper == "save_file":
- qry = '''
-drop table if exists lock_stat_tab_1;
-CREATE TABLE lock_stat_tab_1 (session_id int) ENGINE=InnoDB;
-drop procedure if exists my_session_wait;
-delimiter $$
-create procedure my_session_wait()
-begin
- declare tmp int;
- LOOP
- select sleep(3600) into tmp;
- END LOOP;
-end$$
-delimiter ;
-
-set @cid = connection_id();
-insert into lock_stat_tab_1 values (@cid);
-commit;
-lock table tab_1 read local;
-call my_session_wait();
-unlock tables;
-'''
- if oper == "save_file": open(lock_tab_sql_file, "w").write(qry)
- else:
- ret = mysql_exec_sql(qry, dbname, user, passwd)
- print ret
- #ret = cmd_call(cmd, options.timeout, "ERROR 2013", get_outf())
- else:
- ret = mysql_exec_sql_one_row("select session_id from lock_stat_tab_1", dbname, user, passwd)
- dbg_print(" ret = %s" % ret)
- if not ret: my_error("No lock yet")
- session_id = ret[0]
- qry = "kill %s" % session_id
- ret = mysql_exec_sql(qry, dbname, user, passwd)
-
-def producer_purge_log():
- ''' this one is deprecated. Use the cleaner instead '''
- dbname, user, passwd = get_bootstrap_db_conn_info()
- ret = mysql_exec_sql("select id from bootstrap_sources", dbname, user, passwd, None, True)
- for srcid in [x[0] for x in ret]: # for each source
- dbg_print("srcid = %s" % srcid)
- applied_logid = mysql_exec_sql_one_row("select logid from bootstrap_applier_state", dbname, user, passwd)[0]
- qry = "select logid from bootstrap_loginfo where srcid=%s and logid<%s order by logid limit %s" % (srcid, applied_logid, options.producer_log_purge_limit)
- ret = mysql_exec_sql(qry, dbname, user, passwd, None, True)
- logids_to_purge = [x[0] for x in ret]
- qry = ""
- for logid in logids_to_purge: qry += "drop table if exists log_%s_%s;" % (srcid, logid)
- mysql_exec_sql(qry, dbname, user, passwd)
- dbg_print("logids_to_purge = %s" % logids_to_purge)
- mysql_exec_sql("delete from bootstrap_loginfo where srcid=%s and logid in (%s); commit" % (srcid, ",".join(logids_to_purge)), dbname, user, passwd)
-
-# load the command dictionary
-parser = OptionParser(usage="usage: %prog [options]")
-execfile(os.path.join(get_this_file_dirname(),"driver_cmd_dict.py"))
-
-allowed_opers=[]
-for cmd in cmd_dict: allowed_opers.extend(cmd_dict[cmd].keys())
-allowed_opers=[x for x in list(set(allowed_opers)) if x!="default"]
-
-ct=None # global variale of the cmd thread, use to access subprocess
-def is_starting_component():
- return options.operation != "default" and "%s_%s" % (options.component, options.operation) in cmd_ret_pattern
-
-# need to check pid to determine if process is dead
-# Thread and objects
-class cmd_thread(threading.Thread):
- ''' execute one cmd in parallel, check output. there should be a timer. '''
- def __init__ (self, cmd, ret_pattern=None, outf=None):
- threading.Thread.__init__(self)
- self.daemon=True # make it daemon, does not matter if use sys.exit()
- self.cmd = cmd
- self.ret_pattern = ret_pattern
- self.outf = sys.stdout
- if outf: self.outf = outf
- self.thread_wait_end=False
- self.thread_ret_ok=False
- self.subp=None
- self.ok_to_run=True
- def run(self):
- self.subp = subprocess_call_1(self.cmd)
- if not self.subp:
- self.thread_wait_end=True
- return
- # capture java call here
- if options.capture_java_call: cmd_call_capture_java_call() # test only remote
- # print the pid
- if is_starting_component():
- java_pid_str = "## java process pid = %s\n## hostname = %s\n" % (find_java_pid(self.subp.pid), host_name_global)
- if java_pid_str: open(options.logfile,"a").write(java_pid_str)
- self.outf.write(java_pid_str)
- # no block
- fd = self.subp.stdout.fileno()
- fl = fcntl.fcntl(fd, fcntl.F_GETFL)
- fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
- while (self.ok_to_run): # for timeout case, must terminate the thread, need non block read
- try: line = self.subp.stdout.readline()
- except IOError, e:
- time.sleep(0.1)
- #dbg_print("IOError %s" % e)
- continue
- dbg_print("line = %s" % line)
- if not line: break
- self.outf.write("%s" % line)
- if self.ret_pattern and self.ret_pattern.search(line):
- self.thread_ret_ok=True
- break
- if not self.ret_pattern: self.thread_ret_ok=True # no pattern ok
- self.thread_wait_end=True
- # has pattern but not find, then not ok
- #while (1): # read the rest and close the pipe
- # try: line = self.subp.stdout.readline()
- # except IOError, e:
- # break
- self.subp.stdout.close()
- # close all the file descriptors
- #os.close(1) # stdin
- #os.close(2) # stdout
- #os.close(3) # stderr
- dbg_print("end of thread run")
-
-def cmd_call_capture_java_call():
- ''' this one depends on the ivy path and ps length. may not work for all '''
- if options.capture_java_call!="auto":
- short_class_name=options.capture_java_call
- else:
- short_class_name=cmd_dict[options.component]["stop"].split("grep ")[-1].split(" ")[0]
- ret = wait_for_condition('sys_pipe_call("ps -ef | grep java | grep -v grep | grep %s")' % short_class_name, 20)
- java_ps_call = sys_pipe_call('ps -ef | grep "/java -d64" | grep -v grep | grep -v capture_java_call| grep %s' % short_class_name)
- #java_ps_call = tmp_str
- ivy_dir=get_ivy_dir() # espresso has different ivy
- dbg_print("ivy_dir = %s, java_ps_call=%s" % (ivy_dir,java_ps_call))
- view_root=get_view_root()
- class_path_list = []
- #pdb.set_trace()
- for jar_path in java_ps_call.split("-classpath ")[-1].split(" com.linkedin")[0].split(":"): # classpath
- if not jar_path: continue
- if not re.search("(%s|%s)" % (ivy_dir,view_root),jar_path):
- class_path_list.append(jar_path)
- continue
- if re.search(ivy_dir,jar_path):
- sub_dir= ivy_dir
- sub_str = "IVY_DIR"
- if re.search(view_root,jar_path):
- sub_dir= view_root
- sub_str = "VIEW_ROOT"
- class_path_list.append('\"%s\"' % re.sub(sub_dir,sub_str,jar_path))
- class_path_list.sort()
- class_path = "[\n %s\n]" % "\n ,".join(class_path_list)
- class_name = java_ps_call.split(short_class_name)[0].split(" ")[-1] + short_class_name
-#cmd_direct_call={
- print '''
- ,"%s":
- {
- "class_path":%s
- ,"class_name":"%s"
- }
-''' % (options.component, class_path, class_name)
-#}
-
- #dbg_print("class_path = %s, class_name = %s" % (class_path, class_name))
- #sys.exit(0)
-
-def cmd_call(cmd, timeout, ret_pattern=None, outf=None):
- ''' return False if timed out. timeout is in secs '''
- #if options.capture_java_call: cmd_call_capture_java_call() # test only remote
- if options.operation=="stop" and options.component_id:
- process_info = get_process_info()
- key=get_process_info_key(options.component, options.component_id)
- if key in process_info:
- kill_cmd="kill -9"
- if "stop" in cmd_dict[options.component]:
- kill_cmd = cmd_dict[options.component]["stop"]
- m = re.search("^.*(kill.*)\s*$",kill_cmd)
- if m: kill_cmd = m.group(1)
- sys_call("%s %s" % (kill_cmd, process_info[key]["pid"]))
- return RetCode.OK
- global ct
- ct = cmd_thread(cmd, ret_pattern, outf)
- ct.start()
- sleep_cnt = 0
- sleep_interval = 0.5
- ret = RetCode.TIMEOUT
- while (sleep_cnt * sleep_interval < timeout):
- if ct.thread_wait_end or (ct.subp and not process_exist(ct.subp.pid)):
- print "end"
- if ct.thread_ret_ok: ret = RetCode.OK # include find pattern or no pattern given
- else: ret= RetCode.ERROR
- if options.save_process_id:
- id = options.component_id and options.component_id or 0
- save_process_info(options.component, str(id), None, options.logfile) # no port of cm
- #if options.capture_java_call: cmd_call_capture_java_call()
- break # done
- time.sleep(sleep_interval)
- sleep_cnt += 1
- while (not ct.thread_wait_end):
- ct.ok_to_run = False # terminate the thread in timeout case
- time.sleep(0.1)
- return ret
-
-remote_component=None
-remote_cmd_template='''ssh %s "bash -c 'source /export/home/eng/dzhang/bin/jdk6_env; cd %s; %s'"'''
-def run_cmd_remote_setup():
- print "!!! REMOTE RUN ENABLED !!!"
- global remote_component
- component_cnt = 0
- # find the one in the cfg file, so multiple consumers must be in sequence
- for section in remote_run_config:
- if re.search(options.component, section):
- remote_component=section
- component_cnt +=1
- if not options.component_id or compnent_cnt == options.component_id: break
- if not remote_component: my_error("No section for component %s, id %s" % (options.component, options.component_id))
- remote_component_properties = remote_run_config[remote_component]
- set_remote_view_root(remote_component_properties["view_root"])
- # create the remote var/work dir, may not be needed as the current view have them
- #sys_call("ssh %s mkdir -p %s %s" % remote_run_config[remote_component]["host"], get_remote_work_dir(), get_remote_var_dir()
-
-def run_cmd_remote(cmd):
- ret = remote_cmd_template % (remote_run_config[remote_component]["host"], get_remote_view_root(), cmd)
- return ret
-
-
-run_cmd_added_options=[]
-def run_cmd_add_option(cmd, option_name, value=None, check_exist=False):
- global direct_java_call_jvm_args
- dbg_print("option_name = %s, value = %s" % (option_name, value))
- #option_name = option_name.split(".")[-1] # get rid of the options., which is for readability only
- if option_name not in dir(options): my_error("invalid option name %s" % option_name)
- global run_cmd_added_options
- run_cmd_added_options.append(option_name)
- if not getattr(options, option_name): return cmd # not such option
- if not value: value = getattr(options,option_name)
- dbg_print("after option_name = %s, value = %s" % (option_name, value))
- #pdb.set_trace()
- if check_exist:
- full_path = file_exists(value)
- if not full_path: my_error("File does not exists! %s" % value)
- value=full_path
- is_jvm_option = re.search("jvm_",option_name)
- if isinstance(value, str) and value[0]!='"' and not (option_name in ["cmdline_args"] or is_jvm_option) and options.enable_direct_java_call: # do not quote the cmdline args
- #value = value.replace(' ','\\ ') # escape the white space
- value = '"%s"' % value # quote it
- if options.enable_direct_java_call:
- option_mapping = direct_java_call_option_mapping
- option_prefix = ""
- option_assign = ""
- if is_jvm_option or option_name in direct_java_call_jvm_args: # must start with jvm
- #pdb.set_trace()
- direct_java_call_jvm_args[option_name][1]=value # overide the default value
- dbg_print("direct_java_call_jvm_args[%s]=%s" % (option_name,direct_java_call_jvm_args[option_name]))
- return cmd
- else:
- option_mapping = ant_call_option_mapping
- option_prefix = "-D"
- option_assign = "="
- option_mapping_name = option_name # default same as the option name
- if option_name in option_mapping: option_mapping_name = option_mapping[option_name]
- option_str = option_prefix + option_mapping_name + option_assign + value
- dbg_print("option_str = %s" % (option_str))
- if not option_str: return cmd
- cmd_split=cmd.split()
- if options.enable_direct_java_call: # add option to the end
- cmd += " %s" % option_str
- else:
- cmd_split.insert(len(cmd_split)-1,option_str) # here it handles insert before the last one
- cmd = " ".join(cmd_split)
- dbg_print("cmd = %s" % cmd)
- return cmd
-
-def run_cmd_add_log_file(cmd):
- global options
- if options.logfile: log_file = options.logfile
- else: log_file= log_file_pattern % (options.testname, options.component, options.operation, time.strftime('%y%m%d_%H%M%S'), os.getpid())
- #log_file = os.path.join(remote_run and get_remote_log_dir() or get_log_dir(), log_file)
- # TODO: maybe we want to put the logs in the remote host
- log_file = os.path.join(get_log_dir(), log_file)
- dbg_print("log_file = %s" % log_file)
- options.logfile = log_file
- open(log_file,"w").write("TEST_NAME=%s\n" % options.testname)
- # logging for all the command
- cmd += " 2>&1 | tee -a %s" % log_file
- return cmd
-
-def run_cmd_get_return_pattern():
- ret_pattern = None
- pattern_key = "%s_%s" % (options.component, options.operation)
- if pattern_key in cmd_ret_pattern: ret_pattern = cmd_ret_pattern[pattern_key]
- if options.wait_pattern: ret_pattern = re.compile(options.wait_pattern)
- dbg_print("ret_pattern = %s" % ret_pattern)
- return ret_pattern
-
-def run_cmd_setup():
- if re.search("_consumer",options.component):
- global consumer_host
- if remote_run: consumer_host = remote_component_properties["host"]
- else: consumer_host = "localhost"
- dbg_print("consumer_host= %s" % consumer_host)
-
-# need to remove from ant_call_option_mapping and run_cmd_add_option to avoid invalid option name
-def run_cmd_add_config(cmd):
- if options.operation in ["start","clean_log","default"]:
- if options.enable_direct_java_call:
- pass_down_options=direct_java_call_option_mapping.keys()
- pass_down_options.extend(direct_java_call_jvm_args.keys())
- #pass_down_options.extend(direct_java_call_jvm_args_ordered)
- else:
- pass_down_options=ant_call_option_mapping.keys()
- #option_mapping = options.enable_direct_java_call and direct_java_call_option_mapping or ant_call_option_mapping
- #if options.enable_direct_java_call: pass_down_options.append("jvm_args")
- if options.config:
- if not remote_run:
- cmd = run_cmd_add_option(cmd, "config", options.config, check_exist=True) # check exist will figure out
- else:
- cmd = run_cmd_add_option(cmd, "config", os.path.join(get_remote_view_root(), options.config), check_exist=False)
- run_cmd_view_root = remote_run and get_remote_view_root() or get_view_root()
- #cmd = run_cmd_add_option(cmd, "dump_file", options.dump_file and os.path.join(run_cmd_view_root, options.dump_file) or None)
- #cmd = run_cmd_add_option(cmd, "value_file", options.value_file and os.path.join(run_cmd_view_root, options.value_file) or None)
- #cmd = run_cmd_add_option(cmd, "log4j_file", options.log4j_file and os.path.join(run_cmd_view_root, options.log4j_file) or None)
- #cmd = run_cmd_add_option(cmd, "jvm_direct_memory_size")
- #cmd = run_cmd_add_option(cmd, "jvm_max_heap_size")
- #cmd = run_cmd_add_option(cmd, "jvm_gc_log")
- #cmd = run_cmd_add_option(cmd, "jvm_args")
- #cmd = run_cmd_add_option(cmd, "db_config_file")
- #cmd = run_cmd_add_option(cmd, "cmdline_props")
-# cmd = run_cmd_add_option(cmd, "filter_conf_file")
-
- if options.checkpoint_dir:
- if options.checkpoint_dir == "auto":
- checkpoint_dir = os.path.join(get_work_dir(), "databus2_checkpoint_%s_%s" % time.strftime('%y%m%d_%H%M%S'), os.getpid())
- else:
- checkpoint_dir = options.checkpoint_dir
- checkpoint_dir = os.path.join(run_cmd_view_root(), checkpoint_dir)
- cmd = run_cmd_add_option(cmd, "checkpoint_dir", checkpoint_dir)
- # clear up the directory
- if not options.checkpoint_keep and os.path.exists(checkpoint_dir): distutils.dir_util.remove_tree(checkpoint_dir)
-
- # options can be changed during remote run
- if remote_run:
- remote_component_properties = remote_run_config[remote_component]
- if not options.relay_host and "relay_host" in remote_component_properties: options.relay_host = remote_component_properties["relay_host"]
- if not options.relay_port and "relay_port" in remote_component_properties: options.relay_port = remote_component_properties["relay_port"]
- if not options.bootstrap_host and "bootstrap_host" in remote_component_properties: options.bootstrap_host = remote_component_properties["bootstrap_host"]
- if not options.bootstrap_port and "bootstrap_port" in remote_component_properties: options.bootstrap_port = remote_component_properties["bootstrap_port"]
- #cmd = run_cmd_add_option(cmd, "relay_host")
- #cmd = run_cmd_add_option(cmd, "relay_port")
- #cmd = run_cmd_add_option(cmd, "bootstrap_host")
- #cmd = run_cmd_add_option(cmd, "bootstrap_port")
- #cmd = run_cmd_add_option(cmd, "consumer_event_pattern")
- if re.search("_consumer",options.component):
- # next available port
- if options.http_port: http_port = options.http_port
- else: http_port = next_available_port(consumer_host, consumer_http_start_port)
- #cmd = run_cmd_add_option(cmd, "http_port", http_port)
- #cmd = run_cmd_add_option(cmd, "jmx_service_port", next_available_port(consumer_host, consumer_jmx_service_start_port))
- # this will take care of the passdown, no need for run_cmd_add_directly
- for option in [x for x in pass_down_options if x not in run_cmd_added_options]:
- cmd = run_cmd_add_option(cmd, option)
-
-
- if options.component=="espresso-relay": cmd+= " -d " # temp hack. TODO: remove
-
- if options.enable_direct_java_call:
- #cmd = re.sub("java -classpath","java -d64 -ea %s -classpath" % " ".join([x[0]+x[1] for x in [direct_java_call_jvm_args[y] for y in direct_java_call_jvm_args_ordered] if x[1]]) ,cmd) # d64 here
- cmd = re.sub("java -classpath","java -d64 -ea %s -classpath" % " ".join([x[0]+x[1] for x in direct_java_call_jvm_args.values() if x[1]]) ,cmd) # d64 here
- dbg_print("cmd = %s" % cmd)
- return cmd
-
-def run_cmd_add_ant_debug(cmd):
- if re.search("^ant", cmd): cmd = re.sub("^ant","ant -d", cmd)
- dbg_print("cmd = %s" % cmd)
- return cmd
-
-def run_cmd_save_cmd(cmd):
- if not options.logfile: return
- re_suffix = re.compile("\.\w+$")
- if re_suffix.search(options.logfile): command_file = re_suffix.sub(".sh", options.logfile)
- else: command_file = "%s.sh" % options.logfile
- dbg_print("command_file = %s" % command_file)
- open(command_file,"w").write("%s\n" % cmd)
-
-def run_cmd_restart(cmd):
- ''' restart using a previous .sh file '''
- if not options.logfile: return cmd
- previous_run_sh_pattern = "%s_*.sh" % "_".join(options.logfile.split("_")[:-3])
- import glob
- previous_run_sh = glob.glob(previous_run_sh_pattern)
- my_warning("No previous run files. Cannot restart. Start with new options.")
- if not previous_run_sh: return cmd
- previous_run_sh.sort()
- run_sh = previous_run_sh[-1]
- print "Use previous run file %s" % run_sh
- lines = open(run_sh).readlines()
- cmd = lines[0].split("2>&1")[0]
- return cmd
-
-def run_cmd_direct_java_call(cmd, component):
- ''' this needs to be consistent with adding option
- currently ant -f ; will mess up if there are options
- '''
-
- if not component in cmd_direct_call:
- options.enable_direct_java_call = False # disable direct java call if classpath not given
- return cmd
- #if re.search("^ant", cmd): # only component in has class path given will be
- #if True: # every thing
- if re.search("ant ", cmd): # only component in has class path given will be
- ivy_dir = get_ivy_dir()
- view_root = get_view_root()
- class_path_list=[]
- for class_path in cmd_direct_call[component]["class_path"]:
- if re.search("IVY_DIR",class_path):
- class_path_list.append(re.sub("IVY_DIR", ivy_dir,class_path))
- continue
- if re.search("VIEW_ROOT",class_path):
- class_path_list.append(re.sub("VIEW_ROOT", view_root,class_path))
- if not os.path.exists(class_path_list[-1]): # some jars not in VIEW_ROOT, trigger before command
- if "before_cmd" in cmd_direct_call[component]:
- before_cmd = "%s; " % cmd_direct_call[component]["before_cmd"]
- sys_call(before_cmd)
- continue
- class_path_list.append(class_path)
- if options.check_class_path:
- for jar_file in class_path_list:
- if not os.path.exists(jar_file):
- print "==WARNING NOT EXISTS: " + jar_file
- new_jar_path = sys_pipe_call("find %s -name %s" % (ivy_dir, os.path.basename(jar_file))).split("\n")[0]
- if new_jar_path:
- print "==found " + new_jar_path
- class_path_list[class_path_list.index(jar_file)] = new_jar_path
- direct_call_cmd = "java -classpath %s %s" % (":".join(class_path_list), cmd_direct_call[component]["class_name"])
- if re.search("ant .*;",cmd): cmd = re.sub("ant .*;","%s" % direct_call_cmd, cmd)
- else: cmd = re.sub("ant .*$",direct_call_cmd, cmd)
- dbg_print("cmd = %s" % cmd)
- return cmd
-
-def run_cmd():
- if (options.component=="bootstrap_dbreset"): setup_rmi("stop")
- if (not options.operation): options.operation="default"
- if (not options.testname):
- options.testname = "TEST_NAME" in os.environ and os.environ["TEST_NAME"] or "default"
- if (options.operation not in cmd_dict[options.component]):
- my_error("%s is not one of the command for %s. Valid values are %s " % (options.operation, options.component, cmd_dict[options.component].keys()))
- # handle the different connetion string for hudson
- if (options.component=="db_relay" and options.db_config_file):
- options.db_config_file = db_config_change(options.db_config_file)
- if (options.component=="test_bootstrap_producer" and options.operation=="lock_tab"):
- producer_lock_tab("save_file")
- cmd = cmd_dict[options.component][options.operation]
- # cmd can be a funciton call
- if isinstance(cmd, list):
- if not callable(cmd[0]): my_error("First element should be function")
- cmd[0](*tuple(cmd[1:])) # call the function
- return
- if options.enable_direct_java_call: cmd = run_cmd_direct_java_call(cmd, options.component)
- if remote_run: run_cmd_remote_setup()
- if options.ant_debug: cmd = run_cmd_add_ant_debug(cmd) # need ant debug call or not
- cmd = run_cmd_add_config(cmd) # handle config file
- if remote_run: cmd = run_cmd_remote(cmd)
- ret_pattern = run_cmd_get_return_pattern()
- if options.restart: cmd = run_cmd_restart(cmd)
- cmd = run_cmd_add_log_file(cmd)
- if is_starting_component(): run_cmd_save_cmd(cmd)
- ret = cmd_call(cmd, options.timeout, ret_pattern, get_outf())
- if options.operation == "stop": time.sleep(0.1)
- return ret
-
-def setup_rmi_cond(oper):
- rmi_up = isOpen(server_host, rmi_registry_port)
- dbg_print("rmi_up = %s" % rmi_up)
- if oper=="start": return rmi_up
- if oper=="stop": return not rmi_up
-
-def setup_rmi(oper="start"):
- ''' start rmi registry if not alreay started '''
- ret = RetCode.OK
- dbg_print("oper = %s" % oper)
- rmi_up = isOpen(server_host, rmi_registry_port)
- rmi_str = "ant -f sitetools/rmiscripts/build.xml; ./rmiservers/bin/rmiregistry%s" % oper
- if oper=="stop": sys_call(kill_cmd_template % "RegistryImpl") # make sure it stops
- if (oper=="start" and not rmi_up) or (oper=="stop" and rmi_up):
- sys_call(rmi_str)
- # wait for rmi
- ret = wait_for_condition('setup_rmi_cond("%s")' % oper)
-
-def setup_env():
- #setup_rmi()
- pass
-
-def get_outf():
- outf = sys.stdout
- if options.output: outf = open(options.output,"w")
- return outf
-
-def start_jmx_cli():
- global jmx_cli
- if not jmx_cli:
- jmx_cli = pexpect.spawn("java -jar %s/../lib/jmxterm-1.0-alpha-4-uber.jar" % get_this_file_dirname())
- jmx_cli.expect("\$>")
-
-def stop_jmx_cli():
- global jmx_cli
- if jmx_cli:
- jmx_cli.sendline("quit")
- jmx_cli.expect(pexpect.EOF)
- jmx_cli = None
-
-def jmx_cli_cmd(cmd):
- if not jmx_cli: start_jmx_cli()
- dbg_print("jmx cmd = %s" % cmd)
- jmx_cli.sendline(cmd)
- jmx_cli.expect("\$>")
- ret = jmx_cli.before.split("\r\n")[1:]
- dbg_print("jmx cmd ret = %s" % ret)
- return ret
-
-def get_stats_1(pid, jmx_bean, jmx_attr):
- outf = get_outf()
- start_jmx_cli()
- jmx_cli_cmd("open %s" % pid)
- ret = jmx_cli_cmd("beans")
- if jmx_bean=="list":
- stat_re = re.compile("^com.linkedin.databus2:")
- stats = [x for x in ret if stat_re.search(x)]
- outf.write("%s\n" % "\n".join(stats))
- return
- stat_re = re.compile("^com.linkedin.databus2:.*%s$" % jmx_bean)
- stats = [x for x in ret if stat_re.search(x)]
- if not stats: # stats not find
- stat_re = re.compile("^com.linkedin.databus2:")
- stats = [x.split("=")[-1].rstrip() for x in ret if stat_re.search(x)]
- my_error("Possible beans are %s" % stats)
- full_jmx_bean = stats[0]
- jmx_cli_cmd("bean %s" % full_jmx_bean)
- if jmx_attr == "all": jmx_attr = "*"
- ret = jmx_cli_cmd("get %s" % jmx_attr)
- outf.write("%s\n" % "\n".join(ret))
- stop_jmx_cli()
-
-def run_testcase(testcase):
- dbg_print("testcase = %s" % testcase)
- os.chdir(get_testcase_dir())
- if not re.search("\.test$", testcase): testcase += ".test"
- if not os.path.exists(testcase):
- my_error("Test case %s does not exist" % testcase)
- dbg_print("testcase = %s" % testcase)
- ret = sys_call("/bin/bash %s" % testcase)
- os.chdir(view_root)
- return ret
-
-def get_ebuf_inbound_total_maxStreamWinScn(host, port, option=None):
- url_template = "http://%s:%s/containerStats/inbound/events/total"
- if option == "bootstrap":
- url_template = "http://%s:%s/clientStats/bootstrap/events/total"
- return http_get_field(url_template, host, port, "maxSeenWinScn")
-
-def consumer_reach_maxStreamWinScn(maxWinScn, host, port, option=None):
- consumerMaxWinScn = get_ebuf_inbound_total_maxStreamWinScn(host, port, option)
- dbg_print("consumerMaxWinScn = %s, maxWinScn = %s" % (consumerMaxWinScn, maxWinScn))
- return consumerMaxWinScn >= maxWinScn
-
-def producer_reach_maxStreamWinScn(name, maxWinScn):
- ''' select max of all the sources '''
- dbname, user, passwd = get_bootstrap_db_conn_info()
- tab_name = (name == "producer") and "bootstrap_producer_state" or "bootstrap_applier_state"
- qry = "select max(windowscn) from %s " % tab_name
- ret = mysql_exec_sql_one_row(qry, dbname, user, passwd)
- producerMaxWinScn = ret and ret[0] or 0 # 0 if no rows
- dbg_print("producerMaxWinScn = %s, maxWinScn = %s" % (producerMaxWinScn, maxWinScn))
- return producerMaxWinScn >= maxWinScn
-
-def wait_for_condition(cond, timeout=60, sleep_interval = 0.1):
- ''' wait for a certain cond. cond could be a function.
- This cannot be in utility. Because it needs to see the cond function '''
- dbg_print("cond = %s" % cond)
- sleep_cnt = 0
- ret = RetCode.TIMEOUT
- while (sleep_cnt * sleep_interval < timeout):
- if eval(cond):
- ret = RetCode.OK
- break
- time.sleep(sleep_interval)
- sleep_cnt += 1
- return ret
-
-def producer_wait_event_1(name, timeout):
- ''' options.relay_host should be set for remote_run '''
- relay_host = options.relay_host and options.relay_host or server_host
- relay_port = options.relay_port and options.relay_port or server_port
- if options.sleep_before_wait: time.sleep(options.sleep_before_wait)
- maxWinScn = get_ebuf_inbound_total_maxStreamWinScn(relay_host, relay_port)
- dbg_print("maxWinScn = %s, timeout = %s" % (maxWinScn, timeout))
- ret = wait_for_condition('producer_reach_maxStreamWinScn("%s", %s)' % (name,maxWinScn), timeout)
- if ret == RetCode.TIMEOUT: print "Timed out waiting consumer to reach maxWinScn %s" % maxWinScn
- return ret
-
-def send_shutdown(host, port, force=False):
- ''' use kill which is much faster '''
- #url_template = "http://%s:%s/operation/shutdown"
- url_template = "http://%s:%s/operation/getpid"
- pid = http_get_field(url_template, host, port, "pid")
- force_str = force and "-9" or ""
- sys_call("kill %s %s" % (force_str,pid))
- return pid
-
-def wait_event_1(timeout, option=None):
- relay_host = options.relay_host and options.relay_host or server_host
- relay_port = options.relay_port and options.relay_port or server_port
- maxWinScn = get_ebuf_inbound_total_maxStreamWinScn(relay_host, relay_port)
- print "Wait maxWinScn:%s" % maxWinScn
- dbg_print("maxWinScn = %s, timeout = %s" % (maxWinScn, timeout))
- # consumer host is defined already
- global consumer_port
- if options.component_id: consumer_port=find_open_port(consumer_host, consumer_http_start_port, options.component_id)
- if options.http_port: consumer_port = options.http_port
- ret = wait_for_condition('consumer_reach_maxStreamWinScn(%s, "%s", %s, "%s")' % (maxWinScn, consumer_host, consumer_port, option and option or ""), timeout)
- if ret == RetCode.TIMEOUT: print "Timed out waiting consumer to reach maxWinScn %s" % maxWinScn
- if options.sleep_after_wait: time.sleep(options.sleep_after_wait)
- return ret
-
-def conf_and_deploy_1_find_dir_name(ant_target, screen_out):
- found_target = False
- copy_file_re = re.compile("\[copy\] Copying 1 file to (.*)")
- for line in screen_out:
- if not found_target and line == ant_target: found_target = True
- if found_target:
- dbg_print("line = %s" % line)
- m = copy_file_re.search(line)
- if m: return m.group(1)
- return None
-
-def conf_and_deploy_1_find_extservice(dir_name):
- extservice_re = re.compile("extservices.*\.springconfig")
- flist = os.listdir(dir_name)
- flist.sort(reverse=True)
- for fname in flist:
- if extservice_re.search(fname): return os.path.join(dir_name, fname)
- return None
-
-def conf_and_deploy_1_find_extservice_name(ant_target, screen_out):
- found_target = False
- copy_file_re = re.compile("\[copy\] Copying (\S*) to ")
- for line in screen_out:
- if not found_target and line == ant_target: found_target = True
- if found_target:
- dbg_print("line = %s" % line)
- m = copy_file_re.search(line)
- if m: return m.group(1)
- return None
-
-
-from xml.dom.minidom import parse
-from xml.dom.minidom import Element
-def conf_and_deploy_1_add_conf(file_name):
- dom1 = parse(file_name)
- map_element=[x for x in dom1.getElementsByTagName("map")][0]
- for prop in options.extservice_props:
- #props = prop.split(";")
- props = prop.split("=")
- len_props = len(props)
- if len_props not in (2,3):
- print "WARNING: prop %s is not a valid setting. IGNORED" % prop
- continue
- is_top_level= (len_props == 2)
- find_keys=[x for x in dom1.getElementsByTagName("entry") if x.attributes["key"].value == props[0]]
- dbg_print("find_keys = %s" % find_keys)
- if not find_keys:
- print "WARNING: prop %s part %s is not in file %s. " % (prop, props[0], file_name)
- if is_top_level: # only add when is top level
- print "WARNING: prop %s part %s is added to file %s. " % (prop, props[0], file_name)
- new_entry=Element("entry")
- new_entry.setAttribute("key", props[0])
- new_entry.setAttribute("value", props[1])
- map_element.appendChild(new_entry)
- continue
- keyNode = find_keys[0]
- if is_top_level:
- keyNode.attributes["value"].value=props[-1]
- continue
- find_props= [x for x in keyNode.getElementsByTagName("prop") if x.attributes["key"].value == props[1]]
- dbg_print("find_props = %s" % find_props)
- if not find_props:
- print "WARNING: prop %s part %s is not in file %s. IGNORED" % (prop, props[1], file_name)
- continue
- find_props[0].childNodes[0].nodeValue=props[-1]
- open(file_name,"w").write(dom1.toxml())
-
-def conf_and_deploy_1(ant_file):
- ''' to deploy a service only, do exploded-war first,
- then build-app-conf substitute the extservice_props into the extservice file
- the deploy.only.noconf to deploy the service using the new conf
- '''
- #pdb.set_trace()
- #out = sys_pipe_call("ant -f %s build-app-conf" % (ant_file))
- #dir_name = conf_and_deploy_1_find_dir_name("build-app-conf:", out.split("\n"))
- tmp_file = tempfile.mkstemp()[1]
- cmd = "ant -f %s exploded-war 2>&1 | tee %s" % (ant_file, tmp_file)
- ret = cmd_call(cmd, 60, re.compile("BUILD SUCCESSFUL"))
- cmd = "ant -f %s build-app-conf 2>&1 | tee %s" % (ant_file, tmp_file)
- ret = cmd_call(cmd, 5, re.compile("BUILD SUCCESSFUL"))
- dir_name = conf_and_deploy_1_find_dir_name("build-app-conf:", [x.rstrip() for x in open(tmp_file).readlines()])
- dbg_print("dir_name = %s" % dir_name)
- if dir_name: extservice_file_name = conf_and_deploy_1_find_extservice(dir_name)
- if not dir_name or not extservice_file_name: my_error("No extservice file in dir %s" % dir_name)
- #out = sys_pipe_call("ant -f %s -d build-app-conf" % (ant_file))
- #extservice_file_name = conf_and_deploy_1_find_extservice_name("build-app-conf:", out.split("\n"))
- dbg_print("extservice_file_name = %s" % extservice_file_name)
- if options.extservice_props:
- tmp_files = [extservice_file_name]
- tmp_files = save_copy([extservice_file_name])
- dbg_print("new_files = %s" % tmp_files)
- conf_and_deploy_1_add_conf(extservice_file_name)
- #shutil.copy(tmp_files[0], extservice_file_name)
- # do the deploy
- #pdb.set_trace()
- cmd = "ant -f %s deploy.only.noconf 2>&1 | tee %s" % (ant_file, tmp_file)
- ret = cmd_call(cmd, 60, re.compile("BUILD SUCCESSFUL"))
-
-zookeeper_cmd=None
-zookeeper_server_ports=None
-zookeeper_server_dir=None
-zookeeper_server_ids=None
-
-#possible_ivy_dir=[os.path.join(os.environ["HOME"],".ivy2/lin-cache/ivy-cache"),os.path.join(os.environ["HOME"],".ivy2/lin-cache"),"/ivy/.ivy2/ivy-cache","/ivy/.ivy2"]
-#possible_ivy_dir=[os.path.join(os.environ["HOME"],".m2/repository"), os.path.join(os.environ["HOME"],".ivy2/lin-cache/"),"/ivy/.ivy2"]
-def get_ivy_dir():
- for ivy_dir in possible_ivy_dir:
- if os.path.exists(ivy_dir): break
- if not os.path.exists(ivy_dir): raise
- return ivy_dir
-
-def zookeeper_setup(oper):
- ''' may need to do a find later. find $HOME/.ivy2/lin-cache -name zookeeper-3.3.0.jar '''
- global zookeeper_cmd, zookeeper_server_ports, zookeeper_server_dir, zookeeper_server_ids, zookeeper_classpath
- #possible_ivy_home_dir=[os.path.join(os.environ["HOME"],".ivy2/lin-cache/"),"/ivy/.ivy2"]
- possible_ivy_home_dir=[os.path.join(os.environ["HOME"],".m2/repository/"), os.path.join(os.environ["HOME"],".ivy2/lin-cache/"),"/ivy/.ivy2"]
- ivy_dir = get_ivy_dir()
- zookeeper_class= (oper=="start") and "org.apache.zookeeper.server.quorum.QuorumPeerMain" or "org.apache.zookeeper.ZooKeeperMain"
- log4j_file=os.path.join(get_view_root(),"integration-test/config/zookeeper-log4j2file.properties")
- dbg_print("zookeeper_classpath = %s" % zookeeper_classpath)
- if not "zookeeper_classpath" in globals():
- zookeeper_classpath="IVY_DIR/org/apache/zookeeper/zookeeper/3.3.0/zookeeper-3.3.0.jar:IVY_DIR/log4j/log4j/1.2.15/log4j-1.2.15.jar"
- if re.search("IVY_DIR",zookeeper_classpath): zookeeper_classpath=re.sub("IVY_DIR", ivy_dir,zookeeper_classpath)
- if re.search("VIEW_ROOT",zookeeper_classpath): zookeeper_classpath=re.sub("VIEW_ROOT", view_root,zookeeper_classpath)
- run_cmd_add_option("", "config", options.config, check_exist=True) # just add the jvm args
- zookeeper_cmd="java -d64 -Xmx512m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=%%s -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dlog4j.configuration=file://%s %s -cp %s %s" % (log4j_file, " ".join([x[0]+x[1] for x in direct_java_call_jvm_args.values() if x[1]]), zookeeper_classpath, zookeeper_class)
- dbg_print("zookeeper_cmd=%s" % (zookeeper_cmd))
- zookeeper_server_ports= options.zookeeper_server_ports and options.zookeeper_server_ports or "localhost:2181"
- zookeeper_server_dir=os.path.join(get_work_dir(),"zookeeper_data")
- dbg_print("zookeeper_server_dir=%s" % (zookeeper_server_dir))
- #zookeeper_server_ids= options.zookeeper_server_ids and [int(x) for x in options.zookeeper_server_ids.split(",")] or range(1,len(zookeeper_server_ports.split(","))+1)
- zookeeper_server_ids= options.zookeeper_server_ids and [int(x) for x in options.zookeeper_server_ids.split(",")] or range(len(zookeeper_server_ports.split(",")))
- dbg_print("zookeeper_server_ids=%s" % (zookeeper_server_ids))
-
-def zookeeper_opers_start_create_conf(zookeeper_server_ports_split):
- zookeeper_num_servers = len(zookeeper_server_ports_split)
- zookeeper_server_conf_files=[]
- zookeeper_internal_port_1_start = 2800
- zookeeper_internal_port_2_start = 3800
- # overide the default config
- server_conf={"tickTime":2000,"initLimit":5,"syncLimit":2,"maxClientCnxns":0}
- if options.cmdline_props:
- for pair in options.cmdline_props.split(";"):
- (k, v) = pair.split("=")
- if k in server_conf: server_conf[k] = v
- # get the server
- zookeeper_internal_conf=""
- for k in server_conf: zookeeper_internal_conf+="%s=%s\n" % (k, server_conf[k])
- dbg_print("zookeeper_internal_conf = %s" % zookeeper_internal_conf)
- #for server_id in range(1,zookeeper_num_servers+1):
- for server_id in range(zookeeper_num_servers):
- zookeeper_host = zookeeper_server_ports_split[server_id].split(":")[0]
- zookeeper_internal_port_1 = zookeeper_internal_port_1_start + server_id
- zookeeper_internal_port_2 = zookeeper_internal_port_2_start + server_id
- if zookeeper_num_servers>1:
- zookeeper_internal_conf += "server.%s=%s:%s:%s\n" % (server_id, zookeeper_host, zookeeper_internal_port_1, zookeeper_internal_port_2)
- dbg_print("zookeeper_internal_conf = %s" % zookeeper_internal_conf)
-
- #for server_id in range(1,zookeeper_num_servers+1):
- for server_id in range(zookeeper_num_servers):
- if server_id not in zookeeper_server_ids: continue
- conf_file = os.path.join(zookeeper_server_dir,"conf_%s" % server_id)
- dataDir=os.path.join(zookeeper_server_dir,str(server_id))
- zookeeper_port = zookeeper_server_ports_split[server_id].split(":")[1]
- conf_file_p = open(conf_file, "w")
- conf_file_p.write("clientPort=%s\n" % zookeeper_port)
- conf_file_p.write("dataDir=%s\n" % dataDir)
- conf_file_p.write("%s\n" % zookeeper_internal_conf)
- conf_file_p.close()
- dbg_print("==conf file %s: \n %s" % (conf_file, open(conf_file).readlines()))
- zookeeper_server_conf_files.append(conf_file)
- return zookeeper_server_conf_files
-
-def zookeeper_opers_start_create_dirs(zookeeper_server_ports_split):
- #for server_id in range(1,len(zookeeper_server_ports_split)+1):
- for server_id in range(len(zookeeper_server_ports_split)):
- if server_id not in zookeeper_server_ids: continue
- current_server_dir=os.path.join(zookeeper_server_dir,str(server_id))
- dbg_print("current_server_dir = %s" % current_server_dir)
- if os.path.exists(current_server_dir):
- if not options.zookeeper_reset: continue
- distutils.dir_util.remove_tree(current_server_dir)
- try: distutils.dir_util.mkpath(current_server_dir)
- except Exception as e: print ("ERROR: Exception = %s" % e)
- my_id_file=os.path.join(current_server_dir, "myid")
- dbg_print("my_id_file = %s" % my_id_file)
- open(my_id_file,"w").write("%s\n" % server_id)
-
-def zookeeper_opers_start():
- zookeeper_server_ports_split = zookeeper_server_ports.split(",")
- zookeeper_opers_start_create_dirs(zookeeper_server_ports_split)
- conf_files = zookeeper_opers_start_create_conf(zookeeper_server_ports_split)
- cnt = 0
- for conf_file in conf_files:
- # no log file for now
- #cmd = run_cmd_add_log_file(cmd)
- search_str=len(conf_files)>1 and "My election bind port" or "binding to port"
- cmd = "%s %s" % (zookeeper_cmd % (int(options.zookeeper_jmx_start_port) + cnt), conf_file)
- cmd = run_cmd_add_log_file(cmd)
- ret = cmd_call(cmd, 60, re.compile(search_str))
- cnt +=1
-
-def zookeeper_opers_stop():
- # may be better to use pid, but somehow it is not in the datadir
- sys_call(kill_cmd_template % "QuorumPeerMain")
-
-def zookeeper_opers_wait_for_exist():
- pass
-def zookeeper_opers_wait_for_nonexist():
- pass
-def zookeeper_opers_wait_for_value():
- pass
-def zookeeper_opers_cmd():
- if not options.zookeeper_cmds:
- print "No zookeeper_cmds given"
- return
- splitted_cmds = ";".join(["echo %s" % x for x in options.zookeeper_cmds.split(";")])
- sys_call("(%s) | %s -server %s" % (splitted_cmds, zookeeper_cmd, zookeeper_server_ports))
-
-def main(argv):
- # default
- global options
- parser.add_option("-n", "--testname", action="store", dest="testname", default=None, help="A test name identifier")
- parser.add_option("-c", "--component", action="store", dest="component", default=None, choices=cmd_dict.keys(),
- help="%s" % cmd_dict.keys())
- parser.add_option("-o", "--operation", action="store", dest="operation", default=None, choices=allowed_opers,
- help="%s" % allowed_opers)
- parser.add_option("--wait_pattern", action="store", dest="wait_pattern", default=None,
- help="the pattern to wait for the operation to finish")
- parser.add_option("", "--output", action="store", dest="output", default=None,
- help="Output file name. Default to stdout")
- parser.add_option("", "--logfile", action="store", dest="logfile", default=None,
- help="log file for both stdout and stderror. Default auto generated")
- parser.add_option("","--timeout", action="store", type="long", dest="timeout", default=600,
- help="Time out in secs before waiting for the success pattern. [default: %default]")
- parser.add_option("", "--save_process_id", action="store_true", dest="save_process_id", default = False,
- help="Store the process id if set. [default: %default]")
- parser.add_option("", "--restart", action="store_true", dest="restart", default = False,
- help="Restart the process using previos config if set. [default: %default]")
-
- jvm_group = OptionGroup(parser, "jvm options", "")
- jvm_group.add_option("", "--jvm_direct_memory_size", action="store", dest="jvm_direct_memory_size", default = None,
- help="Set the jvm direct memory size. e.g., 2048m. Default using the one driver_cmd_dict.")
- jvm_group.add_option("", "--jvm_max_heap_size", action="store", dest="jvm_max_heap_size", default = None,
- help="Set the jvm max heap size. e.g., 1024m. Default using the one in driver_cmd_dict.")
- jvm_group.add_option("", "--jvm_min_heap_size", action="store", dest="jvm_min_heap_size", default = None,
- help="Set the jvm min heap size. e.g., 1024m. Default using the one in driver_cmd_dict.")
- jvm_group.add_option("", "--jvm_args", action="store", dest="jvm_args", default = None,
- help="Other jvm args. e.g., '-Xms24m -Xmx50m'")
- jvm_group.add_option("", "--jvm_gc_log", action="store", dest="jvm_gc_log", default = None,
- help="Enable gc and give jvm gc log file")
-
- test_case_group = OptionGroup(parser, "Testcase options", "")
- test_case_group.add_option("", "--testcase", action="store", dest="testcase", default = None,
- help="Run a test. Report error. Default no test")
-
- stats_group = OptionGroup(parser, "Stats options", "")
- stats_group.add_option("","--jmx_bean", action="store", dest="jmx_bean", default="list",
- help="jmx bean to get. [default: %default]")
- stats_group.add_option("","--jmx_att", action="store", dest="jmx_attr", default="all",
- help="jmx attr to get. [default: %default]")
-
- remote_group = OptionGroup(parser, "Remote options", "")
- remote_group.add_option("", "--remote_run", action="store_true", dest="remote_run", default = False,
- help="Run remotely based on config file. Default False")
- remote_group.add_option("", "--remote_deploy", action="store_true", dest="remote_deploy", default = False,
- help="Deploy the source tree to the remote machine based on config file. Default False")
- remote_group.add_option("", "--remote_config_file", action="store", dest="remote_config_file", default = None,
- help="Remote config file")
-
- zookeeper_group = OptionGroup(parser, "Zookeeper options", "")
- zookeeper_group.add_option("", "--zookeeper_server_ports", action="store", dest="zookeeper_server_ports", default = None,
- help="comma separated zookeeper ports, used to start/stop/connect to zookeeper")
- zookeeper_group.add_option("", "--zookeeper_path", action="store", dest="zookeeper_path", default = None,
- help="the zookeeper path to wait for")
- zookeeper_group.add_option("", "--zookeeper_value", action="store", dest="zookeeper_value", default = None,
- help="zookeeper path value")
- zookeeper_group.add_option("", "--zookeeper_cmds", action="store", dest="zookeeper_cmds", default = None,
- help="cmds to send to zookeeper client. Comma separated ")
- zookeeper_group.add_option("", "--zookeeper_server_ids", action="store", dest="zookeeper_server_ids", default = None,
- help="Comma separated list of server to start. If not given, start the number of servers in zookeeper_server_ports. This is used to start server on multiple machines ")
- zookeeper_group.add_option("", "--zookeeper_jmx_start_port", action="store", dest="zookeeper_jmx_start_port", default = 27960,
- help="Starting port for jmx")
- zookeeper_group.add_option("", "--zookeeper_reset", action="store_true", dest="zookeeper_reset", default = False,
- help="If true recreate server dir, otherwise start from existing server dir")
-
-
- debug_group = OptionGroup(parser, "Debug options", "")
- debug_group.add_option("-d", "--debug", action="store_true", dest="debug", default = False,
- help="debug mode")
- debug_group.add_option("--ant_debug", action="store_true", dest="ant_debug", default = False,
- help="ant debug mode")
- debug_group.add_option("--capture_java_call", action="store", dest="capture_java_call", default = None,
- help="capture the java call. give the class name or auto")
- debug_group.add_option("--enable_direct_java_call", action="store_true", dest="enable_direct_java_call", default = True,
- #debug_group.add_option("--enable_direct_java_call", action="store_true", dest="enable_direct_java_call", default = False,
- help="enable direct java call. ")
- debug_group.add_option("--check_class_path", action="store_true", dest="check_class_path", default = True,
- help="check if class path exists. ")
- debug_group.add_option("", "--sys_call_debug", action="store_true", dest="enable_sys_call_debug", default = False,
- help="debug sys call")
-
- # load local options
- #execfile(os.path.join(get_this_file_dirname(),"driver_local_options.py"))
- #pdb.set_trace()
-
- parser.add_option_group(jvm_group)
- parser.add_option_group(config_group)
- parser.add_option_group(other_option_group)
- parser.add_option_group(test_case_group)
- parser.add_option_group(stats_group)
- parser.add_option_group(remote_group)
- parser.add_option_group(zookeeper_group)
- parser.add_option_group(debug_group)
-
- (options, args) = parser.parse_args()
- set_debug(options.debug)
- set_sys_call_debug(options.enable_sys_call_debug)
- dbg_print("options = %s args = %s" % (options, args))
-
- arg_error=False
- if not options.component and not options.testcase and not options.remote_deploy:
- print("\n!!!Please give component!!!\n")
- arg_error=True
- if arg_error:
- parser.print_help()
- parser.exit()
-
- if afterParsingHook: afterParsingHook(options) # the hook to call after parsing, change options
-
- setup_env()
- if (not options.testname):
- options.testname = "TEST_NAME" in os.environ and os.environ["TEST_NAME"] or "default"
- os.environ["TEST_NAME"]= options.testname;
-
- if (not "WORK_SUB_DIR" in os.environ):
- os.environ["WORK_SUB_DIR"] = "log"
- if (not "LOG_SUB_DIR" in os.environ):
- os.environ["LOG_SUB_DIR"] = "log"
- setup_work_dir()
-
- if options.testcase:
- ret = run_testcase(options.testcase)
- if ret!=0: ret=1 # workaround a issue that ret of 256 will become 0 after sys.exit
- my_exit(ret)
- if options.remote_deploy or options.remote_run:
- if options.remote_config_file:
- parse_config(options.remote_config_file)
- if options.remote_deploy:
- sys_call_debug_begin()
- ret = do_remote_deploy()
- sys_call_debug_end()
- my_exit(ret)
- sys_call_debug_begin()
- ret = run_cmd()
- sys_call_debug_end()
-
- my_exit(ret)
-
-if __name__ == "__main__":
- main(sys.argv[1:])
-
-
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/4ebc0fad/helix-agent/src/main/scripts/integration-test/script/driver_cmd_dict.py
----------------------------------------------------------------------
diff --git a/helix-agent/src/main/scripts/integration-test/script/driver_cmd_dict.py b/helix-agent/src/main/scripts/integration-test/script/driver_cmd_dict.py
deleted file mode 100644
index a09fcf2..0000000
--- a/helix-agent/src/main/scripts/integration-test/script/driver_cmd_dict.py
+++ /dev/null
@@ -1,312 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-#
-# this is the file to be included by dds_driver.py for product (e.g., espresso) specific setting
-#
-possible_ivy_dir=[os.path.join(os.environ["HOME"],".m2/repository"),os.path.join(os.environ["HOME"],".gradle/cache"),os.path.join(os.environ["HOME"],".ivy2/lin-cache/ivy-cache"),os.path.join(os.environ["HOME"],".ivy2/lin-cache"),"/ivy/.ivy2/ivy-cache","/ivy/.ivy2", os.path.join(os.environ["VIEW_ROOT"],"build/ivy2/cache")]
-zookeeper_classpath="IVY_DIR/org/apache/zookeeper/zookeeper/3.3.3/zookeeper-3.3.3.jar:IVY_DIR/log4j/log4j/1.2.15/log4j-1.2.15.jar"
-
-# espresso use -2
-kill_cmd_template="jps | grep %s | cut -f1 -d\\ | xargs kill -2"
-kill_container_template="ps -ef | grep tail | grep %s | awk '{print $2}' | xargs kill -9"
-
-# This is not used in helix
-afterParsingHook=None
-
-# some global variables
-router_http_port=12917
-router_mgmt_port=12920
-storage_node_http_port=12918
-storage_node_mgmt_port=12919
-curl_kill_cmd_template="curl -s http://localhost:%d/pid | xargs kill -2"
-
-# used to run cmd, can combine multiple command
-cmd_dict={
- "storage-node":{"start":"%s; %s" % (curl_kill_cmd_template % storage_node_mgmt_port,"ant -f espresso-storage-node/run/build.xml run-storage-node"),"stop":curl_kill_cmd_template % storage_node_mgmt_port,"stats":[get_stats,"EspressoSingleNode"]}
- ,"router":{"start":"ant -f espresso-router/run/build.xml run-router","stop":curl_kill_cmd_template % router_mgmt_port,"stats":[get_stats,"EspressoRouter"]}
- ,"zookeeper":{"start":[zookeeper_opers,"start"],"stop":[zookeeper_opers,"stop"],"wait_for_exist":[zookeeper_opers,"wait_for_exist"],"wait_for_nonexist":[zookeeper_opers,"wait_for_nonexist"],"wait_for_value":[zookeeper_opers,"wait_for_value"],"cmd":[zookeeper_opers,"cmd"]}
- ,"cluster-manager":{"start":"ant -f cluster-manager/run/build.xml run-cluster-manager", "stop":kill_cmd_template % "HelixControllerMain"}
- ,"mock-storage":{"start":"ant -f cluster-manager/run/build.xml run-mock-storage", "stop":kill_cmd_template % "MockStorageProcess"}
- ,"cluster-state-verifier":{"start":"ant -d -f cluster-manager/run/build.xml run-cluster-state-verifier", "stop":kill_cmd_template % "ClusterStateVerifier"}
- ,"dummy-process":{"start":"ant -f cluster-manager/run/build.xml run-dummy-process", "stop":kill_cmd_template % "DummyProcess"}
- ,"mock-health-report-process":{"start":"ant -f cluster-manager/run/build.xml run-mock-health-report-process", "stop":kill_cmd_template % "MockHealthReportParticipant"}
- ,"clm_console":{"default":"ant -f cluster-manager/run/build.xml run-cm-console","stop":kill_cmd_template % "ClusterSetup"}
- ,"foo_test":{"start":"integration-test/testcases/foo_test.py"}
-}
-
-cmd_ret_pattern={ # the pattern when the call is considered return successfully
- "storage-node_start":re.compile("Espresso service started")
- ,"router_start":re.compile("Espresso service started")
- ,"cluster-manager_start":re.compile("No Messages to process")
- ,"mock-storage_start":re.compile("Mock storage started")
- ,"dummy-process_start":re.compile("Dummy process started")
- ,"mock-health-report-process_start":re.compile("MockHealthReportParticipant process started")
- ,"foo_test_start":re.compile("start")
-}
-
-# the mapping of option to the java options, if not give, then use directly
-direct_java_call_option_mapping={
- "dump_file":"-f "
- ,"value_file":"--value.dump.file="
- #,"log4j_file":"-Dlog4j.configuration=file://" # this is only for cluster manager
- #,"log4j_file":"--log_props="
- ,"config":"--container_props="
- ,"consumer_event_pattern":"event_pattern"
- ,"cmdline_props":"--cmdline_props="
- ,"cmdline_args":" " # just put the cmdline_args directly
- ,"relay_host":"--relay_host="
- ,"relay_port":"--relay_port="
- #,"jmx_service_port":"--jmx_service_port="
- ,"bootstrap_host":"--bootstrap_host="
- ,"bootstrap_port":"--bootstrap_port="
- ,"http_port":"--http_port="
- ,"checkpoint_dir":"--checkpoint_dir="
- ,"dbname":"--dbname="
- ,"tablename":"--tablename="
- ,"dburi":"--dburi="
- ,"dbuser":"--dbuser="
- ,"dbpasswd":"--dbpassword="
- ,"schemareg":"--schemareg="
- ,"schemareg":"--schemareg="
- ,"db_relay_config":"--db_relay_config="
-}
-
-# has default value, append to the beginning
-direct_java_call_jvm_args={
- "jvm_direct_memory_size":["-XX:MaxDirectMemorySize=","100m"]
- ,"jvm_max_heap_size":["-Xmx","512m"]
- ,"jvm_min_heap_size":["-Xms","100m"]
- ,"jvm_gc_log":["-Xloggc:",""]
- ,"jvm_args":["",""]
- ,"log4j_file":["-Dlog4j.configuration=file://",""] # this is only for cluster manager
-}
-direct_java_call_jvm_args_ordered=[
- "jvm_direct_memory_size"
- ,"jvm_max_heap_size"
- ,"jvm_min_heap_size"
- ,"jvm_gc_log"
- ,"jvm_args"
- ,"log4j_file"
-]
-# mapping from option to ant
-ant_call_option_mapping={
- "dump_file":"dump.file"
- ,"value_file":"value.dump.file"
- ,"log4j_file":"log4j.file"
- ,"config":"config.file"
- ,"jvm_direct_memory_size":"jvm.direct.memory.size"
- ,"jvm_max_heap_size":"jvm.max.heap.size"
- ,"jvm_gc_log":"jvm.gc.log"
- ,"jvm_args":"jvm.args"
- ,"cmdline_props":"cmdline.props"
- ,"cmdline_args":"config.cmdline"
- ,"relay_host":"relay.host"
- ,"relay_port":"relay.port"
- #,"jmx_service_port":"jmx.service.port"
- ,"bootstrap_host":"bootstrap.host"
- ,"bootstrap_port":"bootstrap.port"
- ,"consumer_event_pattern":"consumer.event.pattern"
- ,"http_port":"http.port"
- ,"checkpoint_dir":"checkpoint.dir"
-# ,"db_relay_config":"db.relay.config"
-}
-
-# class path
-import glob
-#print "view_root=" + get_view_root()
-cm_jar_files=glob.glob(os.path.join(get_view_root(),"../../../target/helix-core-*.jar"))
-#cm_jar_file=os.path.basename(cm_jar_file)
-#print cm_jar_file
-cmd_direct_call={
- "clm_console":
- {
- "class_path":[
- "IVY_DIR/com/github/sgroschupf/zkclient/0.1/zkclient-0.1.jar"
- ,"IVY_DIR/com/thoughtworks/xstream/xstream/1.3.1/xstream-1.3.1.jar"
- ,"IVY_DIR/commons-cli/commons-cli/1.2/commons-cli-1.2.jar"
- ,"IVY_DIR/commons-io/commons-io/1.4/commons-io-1.4.jar"
- ,"IVY_DIR/commons-lang/commons-lang/2.4/commons-lang-2.4.jar"
- ,"IVY_DIR/jdom/jdom/1.0/jdom-1.0.jar"
- ,"IVY_DIR/log4j/log4j/1.2.15/log4j-1.2.15.jar"
- ,"IVY_DIR/org/apache/zookeeper/zookeeper/3.3.3/zookeeper-3.3.3.jar"
- ,"IVY_DIR/org/codehaus/jackson/jackson-core-asl/1.8.5/jackson-core-asl-1.8.5.jar"
- ,"IVY_DIR/org/codehaus/jackson/jackson-mapper-asl/1.8.5/jackson-mapper-asl-1.8.5.jar"
- ,"IVY_DIR/xpp3/xpp3_min/1.1.4c/xpp3_min-1.1.4c.jar"
- ,"IVY_DIR/org/restlet/org.restlet/1.1.10/org.restlet-1.1.10.jar"
- ,"IVY_DIR/com/noelios/restlet/com.noelios.restlet/1.1.10/com.noelios.restlet-1.1.10.jar"
-]+cm_jar_files
- ,"class_name":"org.apache.helix.tools.ClusterSetup"
- ,"before_cmd":"../../../mvn package -Dmaven.test.skip.exec=true" # build jar first
- }
-
- ,"dummy-process":
- {
- "class_path":[
- "IVY_DIR/com/thoughtworks/xstream/xstream/1.3.1/xstream-1.3.1.jar"
- ,"IVY_DIR/commons-cli/commons-cli/1.2/commons-cli-1.2.jar"
- ,"IVY_DIR/commons-io/commons-io/1.4/commons-io-1.4.jar"
- ,"IVY_DIR/commons-lang/commons-lang/2.4/commons-lang-2.4.jar"
- ,"IVY_DIR/jdom/jdom/1.0/jdom-1.0.jar"
- ,"IVY_DIR/log4j/log4j/1.2.15/log4j-1.2.15.jar"
- ,"IVY_DIR/org/apache/zookeeper/zookeeper/3.3.3/zookeeper-3.3.3.jar"
- ,"IVY_DIR/org/codehaus/jackson/jackson-core-asl/1.8.5/jackson-core-asl-1.8.5.jar"
- ,"IVY_DIR/org/codehaus/jackson/jackson-mapper-asl/1.8.5/jackson-mapper-asl-1.8.5.jar"
- ,"IVY_DIR/xpp3/xpp3_min/1.1.4c/xpp3_min-1.1.4c.jar"
- ,"IVY_DIR/com/github/sgroschupf/zkclient/0.1/zkclient-0.1.jar"
- ,"IVY_DIR/org/apache/commons/commons-math/2.1/commons-math-2.1.jar"
-]+cm_jar_files
-
- ,"class_name":"org.apache.helix.mock.participant.DummyProcess"
- ,"before_cmd":"../../../mvn package -Dmaven.test.skip.exec=true" # build jar first
- }
-
- ,"mock-health-report-process":
- {
- "class_path":[
- "IVY_DIR/com/thoughtworks/xstream/xstream/1.3.1/xstream-1.3.1.jar"
- ,"IVY_DIR/commons-cli/commons-cli/1.2/commons-cli-1.2.jar"
- ,"IVY_DIR/commons-io/commons-io/1.4/commons-io-1.4.jar"
- ,"IVY_DIR/commons-lang/commons-lang/2.4/commons-lang-2.4.jar"
- ,"IVY_DIR/jdom/jdom/1.0/jdom-1.0.jar"
- ,"IVY_DIR/log4j/log4j/1.2.15/log4j-1.2.15.jar"
- ,"IVY_DIR/org/apache/zookeeper/zookeeper/3.3.3/zookeeper-3.3.3.jar"
- ,"IVY_DIR/org/codehaus/jackson/jackson-core-asl/1.8.5/jackson-core-asl-1.8.5.jar"
- ,"IVY_DIR/org/codehaus/jackson/jackson-mapper-asl/1.8.5/jackson-mapper-asl-1.8.5.jar"
- ,"IVY_DIR/xpp3/xpp3_min/1.1.4c/xpp3_min-1.1.4c.jar"
- ,"IVY_DIR/com/github/sgroschupf/zkclient/0.1/zkclient-0.1.jar"
- ,"IVY_DIR/org/apache/commons/commons-math/2.1/commons-math-2.1.jar"
- ,"IVY_DIR/org/restlet/org.restlet/1.1.10/org.restlet-1.1.10.jar"
- ,"IVY_DIR/com/noelios/restlet/com.noelios.restlet/1.1.10/com.noelios.restlet-1.1.10.jar"
-]+cm_jar_files
-
- ,"class_name":"org.apache.helix.mock.participant.MockHealthReportParticipant"
- ,"before_cmd":"../../../mvn package -Dmaven.test.skip.exec=true" # build jar first
- }
-
- ,"cluster-manager":
- {
- "class_path":[
- "IVY_DIR/com/thoughtworks/xstream/xstream/1.3.1/xstream-1.3.1.jar"
- ,"IVY_DIR/commons-cli/commons-cli/1.2/commons-cli-1.2.jar"
- ,"IVY_DIR/commons-io/commons-io/1.4/commons-io-1.4.jar"
- ,"IVY_DIR/commons-lang/commons-lang/2.4/commons-lang-2.4.jar"
- ,"IVY_DIR/jdom/jdom/1.0/jdom-1.0.jar"
- ,"IVY_DIR/log4j/log4j/1.2.15/log4j-1.2.15.jar"
- ,"IVY_DIR/org/apache/zookeeper/zookeeper/3.3.3/zookeeper-3.3.3.jar"
- ,"IVY_DIR/org/codehaus/jackson/jackson-core-asl/1.8.5/jackson-core-asl-1.8.5.jar"
- ,"IVY_DIR/org/codehaus/jackson/jackson-mapper-asl/1.8.5/jackson-mapper-asl-1.8.5.jar"
- ,"IVY_DIR/xpp3/xpp3_min/1.1.4c/xpp3_min-1.1.4c.jar"
- ,"IVY_DIR/com/github/sgroschupf/zkclient/0.1/zkclient-0.1.jar"
- ,"IVY_DIR/org/apache/commons/commons-math/2.1/commons-math-2.1.jar"
- ,"IVY_DIR/org/restlet/org.restlet/1.1.10/org.restlet-1.1.10.jar"
- ,"IVY_DIR/com/noelios/restlet/com.noelios.restlet/1.1.10/com.noelios.restlet-1.1.10.jar"
-]+cm_jar_files
- ,"class_name":"org.apache.helix.controller.HelixControllerMain"
- ,"before_cmd":"../../../mvn package -Dmaven.test.skip.exec=true" # build jar first
- }
-
- ,"cluster-state-verifier":
- {
- "class_path":[
- "IVY_DIR/com/github/sgroschupf/zkclient/0.1/zkclient-0.1.jar"
- ,"IVY_DIR/com/thoughtworks/xstream/xstream/1.3.1/xstream-1.3.1.jar"
- ,"IVY_DIR/commons-cli/commons-cli/1.2/commons-cli-1.2.jar"
- ,"IVY_DIR/commons-io/commons-io/1.4/commons-io-1.4.jar"
- ,"IVY_DIR/commons-lang/commons-lang/2.4/commons-lang-2.4.jar"
- ,"IVY_DIR/jdom/jdom/1.0/jdom-1.0.jar"
- ,"IVY_DIR/log4j/log4j/1.2.15/log4j-1.2.15.jar"
- ,"IVY_DIR/org/apache/zookeeper/zookeeper/3.3.3/zookeeper-3.3.3.jar"
- ,"IVY_DIR/org/codehaus/jackson/jackson-core-asl/1.8.5/jackson-core-asl-1.8.5.jar"
- ,"IVY_DIR/org/codehaus/jackson/jackson-mapper-asl/1.8.5/jackson-mapper-asl-1.8.5.jar"
- ,"IVY_DIR/xpp3/xpp3_min/1.1.4c/xpp3_min-1.1.4c.jar"
- ,"IVY_DIR/org/restlet/org.restlet/1.1.10/org.restlet-1.1.10.jar"
- ,"IVY_DIR/com/noelios/restlet/com.noelios.restlet/1.1.10/com.noelios.restlet-1.1.10.jar"
-]+cm_jar_files
- ,"class_name":"org.apache.helix.tools.ClusterStateVerifier"
- ,"before_cmd":"../../../mvn package -Dmaven.test.skip.exec=true" # build jar first
- }
-
- ,"mock-storage":
- {
- "class_path":[
- "IVY_DIR/com/github/sgroschupf/zkclient/0.1/zkclient-0.1.jar"
- ,"IVY_DIR/log4j/log4j/1.2.15/log4j-1.2.15.jar"
- ,"IVY_DIR/commons-cli/commons-cli/1.2/commons-cli-1.2.jar"
- ,"IVY_DIR/commons-math/commons-math/2.1/commons-math-2.1.jar"
-]+cm_jar_files
- ,"class_name":"org.apache.helix.mock.participant.MockStorageProcess"
- }
-}
-
-# file the log4j file
-def log4j_file_store_value(option, opt_str, value, parser):
- setattr(parser.values, option.dest, file_exists(value))
-# configure
-config_group = OptionGroup(parser, "Config options", "")
-config_group.add_option("-p", "--config", action="store", dest="config", default=None,
- help="config file path")
-config_group.add_option("--dump_file", action="store", dest="dump_file", default=None,
- help="Event dump file")
-config_group.add_option("--value_file", action="store", dest="value_file", default=None,
- help="Event value dump file")
-config_group.add_option("-l", "--log4j_file", action="callback", callback=log4j_file_store_value, type="str", dest="log4j_file", default=None,
- help="Log4j config file")
-#config_group.add_option("-l", "--log4j_file", action="store", dest="log4j_file", default=None,
-# help="Log4j config file")
-config_group.add_option("--relay_host", action="store", dest="relay_host", default=None,
- help="Host of relay for a consumer")
-config_group.add_option("--relay_port", action="store", dest="relay_port", default=None,
- help="Port of relay for a consumer")
-config_group.add_option("--http_port", action="store", dest="http_port", default=None,
- help="Http Port of the current started component")
-config_group.add_option("--db_relay_config", action="store", dest="db_relay_config", default=None,
- help="DB relay config file")
-config_group.add_option("--cmdline_props", action="store", dest="cmdline_props", default=None,
- help="Command line config props. Comma separate config parameter, e.g., --cmdline_props=databus.relay.eventBuffer.maxSize=1024000;...")
-config_group.add_option("--bootstrap_host", action="store", dest="bootstrap_host", default=None,
- help="Host of bootstrap server")
-config_group.add_option("--bootstrap_port", action="store", dest="bootstrap_port", default=None,
- help="Port of bootstrap server")
-config_group.add_option("--checkpoint_dir", action="store", dest="checkpoint_dir", default=None,
- help="Client checkpoint dir")
-config_group.add_option("--checkpoint_keep", action="store_true", dest="checkpoint_keep", default=False,
- help="Do NOT clean client checkpoint dir")
-config_group.add_option("--consumer_event_pattern", action="store", dest="consumer_event_pattern", default=None,
- help="Check consumer event pattern if set")
-config_group.add_option("--dbname", action="store", dest="dbname", default=None, help="Espresso db name")
-config_group.add_option("--tablename", action="store", dest="tablename", default=None, help="Espresso table name")
-config_group.add_option("--dburi", action="store", dest="dburi", default=None, help="Espresso db uri")
-config_group.add_option("--dbuser", action="store", dest="dbuser", default=None, help="Espresso db user")
-config_group.add_option("--dbpasswd", action="store", dest="dbpasswd", default=None, help="Espresso db password")
-config_group.add_option("--schemareg", action="store", dest="schemareg", default=None, help="Espresso schemareg ")
-config_group.add_option("-x","--extservice_props", action="append", dest="extservice_props", default=None,
- help="Config props to override the extservices. Can give multiple times. One for each property. <entry name>;<prop name>;value. e.g., databus2.relay.local.bizfollow;db.bizfollow.db_url;jdbc.. ")
-config_group.add_option("--cmdline_args", action="store", dest="cmdline_args", default=None, help="Command line arguments")
-
-other_option_group = OptionGroup(parser, "Other options", "")
-other_option_group.add_option("", "--component_id", action="store", dest="component_id", default = None,
- help="The compnent id (1,2..) if there are mutliple instance of a component")
-parser.add_option("","--sleep_before_wait", action="store", type="long", dest="sleep_before_wait", default=0,
- help="Sleep secs before waiting consumer reaching maxEventWindowScn. [default: %default]")
-parser.add_option("","--sleep_after_wait", action="store", type="long", dest="sleep_after_wait", default=1,
- help="Sleep secs after consumer reaching maxEventWindowScn. [default: %default]")
-parser.add_option("","--producer_log_purge_limit", action="store", type="int", dest="producer_log_purge_limit", default=1000,
- help="The limit on number of logs to purge for producer [default: %default]")
-