You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by bu...@apache.org on 2016/05/24 13:33:05 UTC
svn commit: r1745359 - in /uima/sandbox/uima-ducc/trunk:
src/main/admin/ducc.py src/main/config/activemq-ducc-unsecure.xml
src/main/config/activemq-ducc.xml
uima-ducc-examples/src/main/scripts/start_sim
Author: burn
Date: Tue May 24 13:33:04 2016
New Revision: 1745359
URL: http://svn.apache.org/viewvc?rev=1745359&view=rev
Log:
UIMA-4856 Add an unsecure broker config file for simulation testing; increase delay between sim agent starts
Added:
uima/sandbox/uima-ducc/trunk/src/main/config/activemq-ducc-unsecure.xml (with props)
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
uima/sandbox/uima-ducc/trunk/src/main/config/activemq-ducc.xml
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1745359&r1=1745358&r2=1745359&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Tue May 24 13:33:04 2016
@@ -89,7 +89,7 @@ class Ducc(DuccUtil):
# fixing it.
print "NOTOK"
- def run_broker(self, component):
+ def run_broker(self, simtest):
broker_port = self.ducc_properties.get('ducc.broker.port')
broker_jmx_port = self.ducc_properties.get('ducc.broker.jmx.port')
broker_url_decoration = self.ducc_properties.get('ducc.broker.server.url.decoration')
@@ -97,6 +97,8 @@ class Ducc(DuccUtil):
broker_config = self.ducc_properties.get('ducc.broker.configuration')
broker_home = self.ducc_properties.get('ducc.broker.home')
broker_credentials = self.ducc_properties.get('ducc.broker.credentials.file')
+ if ( simtest ):
+ broker_config = 'conf/activemq-ducc-unsecure.xml'
if ( broker_config[0] != '/' ): # relative to broker_home if not absolute
broker_config = broker_home + '/' + broker_config
@@ -285,7 +287,7 @@ class Ducc(DuccUtil):
def usage(self, msg):
print msg
print 'Usage:'
- print ' ducc.py -c <process> [-n <numagents>] [-b] [arguments ...]'
+ print ' ducc.py -c <process> [-n <numagents>] [-b] [-d date] [-o rmmem] [arguments ...]'
print ' ducc.py -k'
print 'Where:'
print ' -c <component> is the name of the comp[onent to start, currently one of'
@@ -293,14 +295,14 @@ class Ducc(DuccUtil):
print ' -- or --'
print ' all - to start all but the agents'
print ' NOTE -- that agents should be started separately'
- print ' -d date is the data on the caller, for startup verification'
- print ' -b uses nohup and places the process into the background'
print ' -n <numagents> if > 1, multiple agents are started (testing mode)'
+ print ' -b uses nohup and places the process into the background'
+ print ' -d date is the data on the caller, for startup verification'
print ' -o <mem-in-GB> rm memory override for use on small machines'
print ' -k causes the entire DUCC system to shutdown'
print ' --nodup If specified, do not start a process if it appears to be already started.'
print ' --or_parms [cold|warm|hot]'
- print ' --ducc_head nodename the name of the "ducc head" where ducc is started from'
+ print ' --simtest If specified, use unblocked broker for sim tests.'
print ' arguments - any additional arguments to pass to the component.'
sys.exit(1)
@@ -315,9 +317,10 @@ class Ducc(DuccUtil):
or_parms = None
nodup = False # we allow duplicates unless asked not to
localdate = time.time()
+ simtest = False
try:
- opts, args = getopt.getopt(argv, 'bc:d:n:o:sk?v', ['or_parms=', 'nodup' ])
+ opts, args = getopt.getopt(argv, 'bc:d:n:o:sk?v', ['or_parms=', 'nodup', 'simtest' ])
except:
self.usage('Bad arguments ' + ' '.join(argv))
@@ -340,6 +343,8 @@ class Ducc(DuccUtil):
or_parms = a
elif ( o == '--nodup' ):
nodup = True
+ elif ( o == '--simtest' ):
+ simtest = True
elif ( o == '-v'):
self.version()
else:
@@ -361,7 +366,7 @@ class Ducc(DuccUtil):
return
if ( component == 'broker' ):
- self.run_broker(background)
+ self.run_broker(simtest)
return
# fall-through, runs one of the ducc components proper
Added: uima/sandbox/uima-ducc/trunk/src/main/config/activemq-ducc-unsecure.xml
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/config/activemq-ducc-unsecure.xml?rev=1745359&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/config/activemq-ducc-unsecure.xml (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/config/activemq-ducc-unsecure.xml Tue May 24 13:33:04 2016
@@ -0,0 +1,115 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<beans
+ xmlns="http://www.springframework.org/schema/beans"
+ xmlns:amq="http://activemq.apache.org/schema/core"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
+ http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd">
+
+ <!-- Allows us to use system properties as variables in this configuration file -->
+ <!-- bean class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer"/-->
+ <bean class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
+ <property name="location" value="file:${DUCC_BROKER_CREDENTIALS_FILE}"/>
+ </bean>
+
+
+
+ <broker xmlns="http://activemq.apache.org/schema/core" persistent="false">
+
+ <destinationPolicy>
+ <policyMap>
+ <policyEntries>
+ <policyEntry topic=">" producerFlowControl="false" optimizedDispatch="true">
+ <pendingSubscriberPolicy>
+ <vmCursor />
+ </pendingSubscriberPolicy>
+ </policyEntry>
+ <policyEntry queue=">" producerFlowControl="false" optimizedDispatch="true" maxPageSize="2000">
+ <pendingQueuePolicy>
+ <vmQueueCursor/>
+ </pendingQueuePolicy>
+ </policyEntry>
+ </policyEntries>
+ </policyMap>
+ </destinationPolicy>
+
+ <managementContext>
+ <managementContext createConnector="true" connectorPort="${DUCC_AMQ_JMX_PORT}"/>
+ </managementContext>
+
+ <systemUsage>
+ <systemUsage>
+ <memoryUsage>
+ <!-- Memory for incoming messages before going to broker's Store -->
+ <memoryUsage limit="500 mb"/>
+ </memoryUsage>
+ <storeUsage>
+ <!-- Broker's memory where messages are stored -->
+ <!-- before being dispatched to consumers -->
+ <storeUsage limit="1 gb" name="foo"/>
+ </storeUsage>
+ <tempUsage>
+ <!-- Disk space dedicated to persistent messages -->
+ <!-- Currently not used by Ducc -->
+ <tempUsage limit="500 mb"/>
+ </tempUsage>
+ </systemUsage>
+ </systemUsage>
+
+ <plugins>
+ <!-- Configure authentication; Username, passwords and groups -->
+ <!-- The anonymousAccessAllowed=true - clients connecting with no username or password -->
+ <!-- will be auto assigned to username 'annonymous' and group 'annonymous' -->
+ <simpleAuthenticationPlugin anonymousAccessAllowed="true">
+ <users>
+ <authenticationUser username="${ducc.broker.admin.username}" password="${ducc.broker.admin.password}"
+ groups="ducc-admin"/>
+ </users>
+ </simpleAuthenticationPlugin>
+
+
+ <!-- Configure authorization -->
+ <!-- Creating and writing to Topics and Queues is NOT restricted -->
+ <authorizationPlugin>
+ <map>
+ <authorizationMap>
+ <authorizationEntries>
+ <authorizationEntry topic=">" read="ducc-admin" write="ducc-admin" admin="ducc-admin" />
+ <authorizationEntry queue=">" read="ducc-admin" write="ducc-admin" admin="ducc-admin" />
+
+ <!-- The following should be used ONLY for cluster simulation testing. -->
+ <!-- The megas use this broker as an application broker for the simulated services. -->
+ <authorizationEntry queue=">" read="anonymous" write="anonymous" admin="anonymous" />
+ <authorizationEntry topic="ActiveMQ.Advisory.>" read="anonymous" write="anonymous" admin="anonymous"/>
+
+ </authorizationEntries>
+ </authorizationMap>
+ </map>
+ </authorizationPlugin>
+ </plugins>
+
+
+ <transportConnectors>
+ <transportConnector name="openwire" uri="tcp://0.0.0.0:${DUCC_AMQ_PORT}?${DUCC_AMQ_DECORATION}"/>
+ </transportConnectors>
+
+ </broker>
+</beans>
Propchange: uima/sandbox/uima-ducc/trunk/src/main/config/activemq-ducc-unsecure.xml
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: uima/sandbox/uima-ducc/trunk/src/main/config/activemq-ducc-unsecure.xml
------------------------------------------------------------------------------
svn:executable = *
Modified: uima/sandbox/uima-ducc/trunk/src/main/config/activemq-ducc.xml
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/config/activemq-ducc.xml?rev=1745359&r1=1745358&r2=1745359&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/config/activemq-ducc.xml (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/config/activemq-ducc.xml Tue May 24 13:33:04 2016
@@ -76,8 +76,8 @@
<plugins>
<!-- Configure authentication; Username, passwords and groups -->
- <!-- The anonymousAccessAllowed=true - clients connecting with no username or password -->
- <!-- will be auto assigned to username 'annonymous' and group 'annonymous' -->
+ <!-- The anonymousAccessAllowed=false prevents clients connecting with no username or password -->
+ <!-- from being assigned to username 'annonymous' and group 'annonymous' -->
<simpleAuthenticationPlugin anonymousAccessAllowed="false">
<users>
<authenticationUser username="${ducc.broker.admin.username}" password="${ducc.broker.admin.password}"
@@ -87,21 +87,13 @@
<!-- Configure authorization -->
- <!-- Creating and writing to Topics is restricted to users in ducc-admin group -->
- <!-- But access to the topic used by the JDs (user code) must be unrestricted -->
- <!-- Everyone can read from, write to, and create queues -->
+ <!-- Creating and writing to Topics and Queues is restricted to users in ducc-admin group -->
<authorizationPlugin>
<map>
<authorizationMap>
<authorizationEntries>
<authorizationEntry topic=">" read="ducc-admin" write="ducc-admin" admin="ducc-admin" />
<authorizationEntry queue=">" read="ducc-admin" write="ducc-admin" admin="ducc-admin" />
- <authorizationEntry topic="ActiveMQ.Advisory.>" read="ducc-admin" write="ducc-admin" admin="ducc-admin"/>
-
- <!-- >authorizationEntry topic=">" read="anonymous,ducc-admin" write="ducc-admin" admin="ducc-admin" />
- <authorizationEntry queue=">" read="anonymous,ducc-admin" write="anonymous,ducc-admin" admin="anonymous,ducc-admin" />
- <authorizationEntry topic="ActiveMQ.Advisory.>" read="anonymous,ducc-admin" write="anonymous,ducc-admin" admin="anonymous,ducc-admin"/>
- <authorizationEntry topic="ducc.jd.state" read="anonymous,ducc-admin" write="anonymous,ducc-admin" admin="anonymous,ducc-admin"/-->
</authorizationEntries>
</authorizationMap>
</map>
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim?rev=1745359&r1=1745358&r2=1745359&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim Tue May 24 13:33:04 2016
@@ -45,9 +45,10 @@ from ducc import Ducc
from ducc_util import ThreadPool
# multi-threaded start can overwhelm ssh if going to the same target host. We inject
-# a short sleep between starts to make it better. This is how long to sleep.
+# a short sleep between starts to make it better. This is how long to sleep in secs.
+# Note: when 0.1 the ducc.properties merge step run by each agent caused problems/hangs
global SLEEP_TIME
-SLEEP_TIME = .1
+SLEEP_TIME = 0.5
class StartSim(DuccUtil):
@@ -61,7 +62,7 @@ class StartSim(DuccUtil):
broker_host = self.ducc_properties.get('ducc.broker.hostname')
print 'broker host', broker_host
- lines = self.ssh(broker_host, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c', 'broker', "'")
+ lines = self.ssh(broker_host, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c', 'broker', '--simtest', "'")
for i in range(0, 9):
if ( self.is_amq_active() ):