You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by cw...@apache.org on 2019/04/25 19:20:14 UTC
svn commit: r1858158 [2/2] -
/uima/site/trunk/uima-website/docs/d/uima-ducc-current/duccbook.html
Modified: uima/site/trunk/uima-website/docs/d/uima-ducc-current/duccbook.html
URL: http://svn.apache.org/viewvc/uima/site/trunk/uima-website/docs/d/uima-ducc-current/duccbook.html?rev=1858158&r1=1858157&r2=1858158&view=diff
==============================================================================
--- uima/site/trunk/uima-website/docs/d/uima-ducc-current/duccbook.html (original)
+++ uima/site/trunk/uima-website/docs/d/uima-ducc-current/duccbook.html Thu Apr 25 19:20:14 2019
@@ -7,7 +7,7 @@
<meta name="originator" content="TeX4ht (http://www.cse.ohio-state.edu/~gurari/TeX4ht/)">
<!-- html -->
<meta name="src" content="duccbook.tex">
-<meta name="date" content="2018-03-14 14:17:00">
+<meta name="date" content="2019-04-02 14:27:00">
<link rel="stylesheet" type="text/css" href="duccbook.css">
</head><body
>
@@ -27,7 +27,7 @@ class="cmr-12">UIMA</span><sup class="te
class="cmr-9">TM</span></sup><span
class="cmr-12">Development Community</span><br /><br /><br />
<br /> <span
-class="cmr-12">Version 2.2.2</span></div>
+class="cmr-12">Version 3.0.0</span></div>
<br />
<div class="date" ></div>
@@ -56,7 +56,7 @@ class="cmbx-10">Trademarks</span></span>
All terms mentioned in the text that are known to be trademarks or service marks have been appropriately capitalized.
Use of such terms in this book should not be regarded as affecting the validity of the the trademark or service
mark.
-<!--l. 47--><p class="noindent" >Publication date: March 2018
+<!--l. 47--><p class="noindent" >Publication date: April 2019
<h2 class="likechapterHead"><a
@@ -378,200 +378,218 @@ href="#x1-24300013.6" id="QQ2-1-260">Res
<br />  <span class="subsectionToc" >13.6.1 <a
href="#x1-24400013.6.1" id="QQ2-1-261">Nodepools</a></span>
<br />  <span class="subsectionToc" >13.6.2 <a
-href="#x1-24800013.6.2" id="QQ2-1-269">Class Definitions</a></span>
+href="#x1-24800013.6.2" id="QQ2-1-271">Class Definitions</a></span>
<br />  <span class="subsectionToc" >13.6.3 <a
-href="#x1-24900013.6.3" id="QQ2-1-271">Validation</a></span>
+href="#x1-24900013.6.3" id="QQ2-1-273">Validation</a></span>
<br /> <span class="sectionToc" >13.7 <a
-href="#x1-25200013.7" id="QQ2-1-274">Ducc Node Definitions</a></span>
+href="#x1-25200013.7" id="QQ2-1-276">Ducc Node Definitions</a></span>
<br /> <span class="sectionToc" >13.8 <a
-href="#x1-25300013.8" id="QQ2-1-276">Ducc User Definitions</a></span>
+href="#x1-25300013.8" id="QQ2-1-278">Ducc User Definitions</a></span>
<br /> <span class="sectionToc" >13.9 <a
-href="#x1-25400013.9" id="QQ2-1-278">DUCC Database Integration</a></span>
+href="#x1-25400013.9" id="QQ2-1-280">DUCC Database Integration</a></span>
<br />  <span class="subsectionToc" >13.9.1 <a
-href="#x1-25500013.9.1" id="QQ2-1-279">Overview</a></span>
+href="#x1-25500013.9.1" id="QQ2-1-281">Overview</a></span>
<br />  <span class="subsectionToc" >13.9.2 <a
-href="#x1-26000013.9.2" id="QQ2-1-284">Database Scripting Utilities</a></span>
+href="#x1-26000013.9.2" id="QQ2-1-286">Database Scripting Utilities</a></span>
<br />  <span class="subsectionToc" >13.9.3 <a
-href="#x1-26100013.9.3" id="QQ2-1-285">Database Configuration</a></span>
+href="#x1-26100013.9.3" id="QQ2-1-287">Database Configuration</a></span>
<br /> <span class="sectionToc" >13.10 <a
-href="#x1-26200013.10" id="QQ2-1-286">Administrative Commands</a></span>
+href="#x1-26200013.10" id="QQ2-1-288">Administrative Commands</a></span>
<br />  <span class="subsectionToc" >13.10.1 <a
-href="#x1-26300013.10.1" id="QQ2-1-287">start_ducc</a></span>
+href="#x1-26300013.10.1" id="QQ2-1-289">autostart.py</a></span>
<br />  <span class="subsectionToc" >13.10.2 <a
-href="#x1-27000013.10.2" id="QQ2-1-294">stop_ducc</a></span>
+href="#x1-26600013.10.2" id="QQ2-1-292">db_autostart_delete.py</a></span>
<br />  <span class="subsectionToc" >13.10.3 <a
-href="#x1-27500013.10.3" id="QQ2-1-299">check_ducc</a></span>
+href="#x1-27100013.10.3" id="QQ2-1-297">db_autostart_query.py</a></span>
<br />  <span class="subsectionToc" >13.10.4 <a
-href="#x1-27900013.10.4" id="QQ2-1-303">build_duccling</a></span>
+href="#x1-27400013.10.4" id="QQ2-1-300">start_ducc</a></span>
<br />  <span class="subsectionToc" >13.10.5 <a
-href="#x1-28000013.10.5" id="QQ2-1-304">ducc_disk_info</a></span>
+href="#x1-28100013.10.5" id="QQ2-1-307">stop_ducc</a></span>
<br />  <span class="subsectionToc" >13.10.6 <a
-href="#x1-28400013.10.6" id="QQ2-1-308">ducc_gather_logs</a></span>
+href="#x1-28700013.10.6" id="QQ2-1-313">check_ducc</a></span>
<br />  <span class="subsectionToc" >13.10.7 <a
-href="#x1-28800013.10.7" id="QQ2-1-312">ducc_post_install</a></span>
+href="#x1-29200013.10.7" id="QQ2-1-318">build_duccling</a></span>
<br />  <span class="subsectionToc" >13.10.8 <a
-href="#x1-29100013.10.8" id="QQ2-1-315">ducc_update</a></span>
+href="#x1-29300013.10.8" id="QQ2-1-319">ducc_disk_info</a></span>
<br />  <span class="subsectionToc" >13.10.9 <a
-href="#x1-29600013.10.9" id="QQ2-1-320">move_ducc</a></span>
+href="#x1-29700013.10.9" id="QQ2-1-323">ducc_gather_logs</a></span>
<br />  <span class="subsectionToc" >13.10.10 <a
-href="#x1-30000013.10.10" id="QQ2-1-324">rm_reconfigure</a></span>
+href="#x1-30100013.10.10" id="QQ2-1-327">ducc_post_install</a></span>
<br />  <span class="subsectionToc" >13.10.11 <a
-href="#x1-30300013.10.11" id="QQ2-1-327">rm_qoccupancy</a></span>
+href="#x1-30400013.10.11" id="QQ2-1-330">ducc_update</a></span>
<br />  <span class="subsectionToc" >13.10.12 <a
-href="#x1-30600013.10.12" id="QQ2-1-330">vary_off</a></span>
+href="#x1-30900013.10.12" id="QQ2-1-335">rm_reconfigure</a></span>
<br />  <span class="subsectionToc" >13.10.13 <a
-href="#x1-30900013.10.13" id="QQ2-1-333">vary_on</a></span>
+href="#x1-31200013.10.13" id="QQ2-1-338">rm_qoccupancy</a></span>
<br />  <span class="subsectionToc" >13.10.14 <a
-href="#x1-31200013.10.14" id="QQ2-1-336">ducc_properties_manager</a></span>
+href="#x1-31500013.10.14" id="QQ2-1-341">vary_off</a></span>
<br />  <span class="subsectionToc" >13.10.15 <a
-href="#x1-31700013.10.15" id="QQ2-1-341">db_create</a></span>
+href="#x1-31800013.10.15" id="QQ2-1-344">vary_on</a></span>
<br />  <span class="subsectionToc" >13.10.16 <a
-href="#x1-31900013.10.16" id="QQ2-1-343">db_loader</a></span>
+href="#x1-32100013.10.16" id="QQ2-1-347">ducc_properties_manager</a></span>
<br />  <span class="subsectionToc" >13.10.17 <a
-href="#x1-32500013.10.17" id="QQ2-1-349">db_tool</a></span>
+href="#x1-32600013.10.17" id="QQ2-1-352">db_create</a></span>
<br />  <span class="subsectionToc" >13.10.18 <a
-href="#x1-33000013.10.18" id="QQ2-1-354">ducc_get_process_swap_usage</a></span>
+href="#x1-32800013.10.18" id="QQ2-1-354">db_loader</a></span>
+<br />  <span class="subsectionToc" >13.10.19 <a
+href="#x1-33400013.10.19" id="QQ2-1-360">db_tool</a></span>
+<br />  <span class="subsectionToc" >13.10.20 <a
+href="#x1-33900013.10.20" id="QQ2-1-365">ducc_get_process_swap_usage</a></span>
<br /> <span class="sectionToc" >13.11 <a
-href="#x1-33400013.11" id="QQ2-1-358">Administrative Tasks</a></span>
+href="#x1-34300013.11" id="QQ2-1-369">Administrative Tasks</a></span>
<br />  <span class="subsectionToc" >13.11.1 <a
-href="#x1-33500013.11.1" id="QQ2-1-359">Add Node</a></span>
+href="#x1-34400013.11.1" id="QQ2-1-370">Add Node</a></span>
<br />  <span class="subsectionToc" >13.11.2 <a
-href="#x1-33700013.11.2" id="QQ2-1-361">Remove Node</a></span>
+href="#x1-34600013.11.2" id="QQ2-1-372">Remove Node</a></span>
<br />  <span class="subsectionToc" >13.11.3 <a
-href="#x1-33900013.11.3" id="QQ2-1-363">Notes</a></span>
+href="#x1-34800013.11.3" id="QQ2-1-374">Notes</a></span>
<br /><span class="chapterToc" >14 <a
-href="#x1-34000014" id="QQ2-1-364">Resource Management</a></span>
+href="#x1-34900014" id="QQ2-1-375">Resource Management</a></span>
<br /> <span class="sectionToc" >14.1 <a
-href="#x1-34100014.1" id="QQ2-1-365">Overview</a></span>
+href="#x1-35000014.1" id="QQ2-1-376">Overview</a></span>
<br /> <span class="sectionToc" >14.2 <a
-href="#x1-34200014.2" id="QQ2-1-366">Preemption vs Eviction</a></span>
+href="#x1-35100014.2" id="QQ2-1-377">Preemption vs Eviction</a></span>
<br /> <span class="sectionToc" >14.3 <a
-href="#x1-34300014.3" id="QQ2-1-367">Scheduling Policies</a></span>
+href="#x1-35200014.3" id="QQ2-1-378">Scheduling Policies</a></span>
<br /> <span class="sectionToc" >14.4 <a
-href="#x1-34400014.4" id="QQ2-1-368">Allotment</a></span>
+href="#x1-35300014.4" id="QQ2-1-379">Allotment</a></span>
<br /> <span class="sectionToc" >14.5 <a
-href="#x1-34500014.5" id="QQ2-1-369">Priority vs Weight</a></span>
+href="#x1-35400014.5" id="QQ2-1-380">Priority vs Weight</a></span>
<br /> <span class="sectionToc" >14.6 <a
-href="#x1-34800014.6" id="QQ2-1-372">Node Pools</a></span>
+href="#x1-35700014.6" id="QQ2-1-383">Node Pools</a></span>
<br /> <span class="sectionToc" >14.7 <a
-href="#x1-34900014.7" id="QQ2-1-373">Scheduling Classes</a></span>
+href="#x1-35800014.7" id="QQ2-1-384">Scheduling Classes</a></span>
<br /><span class="chapterToc" >15 <a
-href="#x1-35000015" id="QQ2-1-374">Service Management</a></span>
+href="#x1-35900015" id="QQ2-1-385">Service Management</a></span>
<br /><span class="chapterToc" >16 <a
-href="#x1-35100016" id="QQ2-1-375">DUCC Web Server Customization</a></span>
+href="#x1-36000016" id="QQ2-1-386">DUCC Web Server Customization</a></span>
<br /> <span class="sectionToc" >16.1 <a
-href="#x1-35200016.1" id="QQ2-1-376">Server Side</a></span>
+href="#x1-36100016.1" id="QQ2-1-387">Server Side</a></span>
<br /> <span class="sectionToc" >16.2 <a
-href="#x1-35300016.2" id="QQ2-1-377">Client Side</a></span>
+href="#x1-36200016.2" id="QQ2-1-388">Client Side</a></span>
<br /> <span class="sectionToc" >16.3 <a
-href="#x1-35400016.3" id="QQ2-1-378">Build and Install</a></span>
+href="#x1-36300016.3" id="QQ2-1-389">Build and Install</a></span>
<br /><span class="chapterToc" >17 <a
-href="#x1-35500017" id="QQ2-1-379">Simulation and System Testing</a></span>
+href="#x1-36400017" id="QQ2-1-390">Reliable DUCC</a></span>
<br /> <span class="sectionToc" >17.1 <a
-href="#x1-35600017.1" id="QQ2-1-380">Cluster Simulation</a></span>
-<br />  <span class="subsectionToc" >17.1.1 <a
-href="#x1-35700017.1.1" id="QQ2-1-381">Overview</a></span>
-<br />  <span class="subsectionToc" >17.1.2 <a
-href="#x1-35800017.1.2" id="QQ2-1-382">Node Configuration</a></span>
-<br />  <span class="subsectionToc" >17.1.3 <a
-href="#x1-35900017.1.3" id="QQ2-1-383">Setting up Test Mode</a></span>
-<br />  <span class="subsectionToc" >17.1.4 <a
-href="#x1-36000017.1.4" id="QQ2-1-384">Starting a Simulated Cluster</a></span>
-<br />  <span class="subsectionToc" >17.1.5 <a
-href="#x1-36400017.1.5" id="QQ2-1-388">Stopping a Simulated Cluster</a></span>
+href="#x1-36500017.1" id="QQ2-1-391">Introduction</a></span>
<br /> <span class="sectionToc" >17.2 <a
-href="#x1-36800017.2" id="QQ2-1-392">Job Simulation</a></span>
-<br />  <span class="subsectionToc" >17.2.1 <a
-href="#x1-36900017.2.1" id="QQ2-1-393">Overview</a></span>
-<br />  <span class="subsectionToc" >17.2.2 <a
-href="#x1-37000017.2.2" id="QQ2-1-394">Job meta-descriptors</a></span>
-<br />  <span class="subsectionToc" >17.2.3 <a
-href="#x1-37100017.2.3" id="QQ2-1-395"><span
-class="cmti-10">Prepare </span>Descriptors</a></span>
-<br />  <span class="subsectionToc" >17.2.4 <a
-href="#x1-37200017.2.4" id="QQ2-1-396">Services</a></span>
-<br />  <span class="subsectionToc" >17.2.5 <a
-href="#x1-37400017.2.5" id="QQ2-1-398">Generating a Job Set</a></span>
-<br />  <span class="subsectionToc" >17.2.6 <a
-href="#x1-37500017.2.6" id="QQ2-1-399">Running the Test Driver</a></span>
+href="#x1-36600017.2" id="QQ2-1-392">Configuring Host Machines</a></span>
<br /> <span class="sectionToc" >17.3 <a
-href="#x1-37600017.3" id="QQ2-1-400">Pre-Packaged Tests</a></span>
+href="#x1-36700017.3" id="QQ2-1-393">Configuring DUCC</a></span>
+<br /> <span class="sectionToc" >17.4 <a
+href="#x1-36800017.4" id="QQ2-1-394">Webserver</a></span>
+<br /> <span class="sectionToc" >17.5 <a
+href="#x1-36900017.5" id="QQ2-1-395">Database</a></span>
<br /><span class="chapterToc" >18 <a
-href="#x1-37700018" id="QQ2-1-401">State Directory</a></span>
+href="#x1-37000018" id="QQ2-1-396">Simulation and System Testing</a></span>
<br /> <span class="sectionToc" >18.1 <a
-href="#x1-37800018.1" id="QQ2-1-402">Overview</a></span>
+href="#x1-37100018.1" id="QQ2-1-397">Cluster Simulation</a></span>
+<br />  <span class="subsectionToc" >18.1.1 <a
+href="#x1-37200018.1.1" id="QQ2-1-398">Overview</a></span>
+<br />  <span class="subsectionToc" >18.1.2 <a
+href="#x1-37300018.1.2" id="QQ2-1-399">Node Configuration</a></span>
+<br />  <span class="subsectionToc" >18.1.3 <a
+href="#x1-37400018.1.3" id="QQ2-1-400">Setting up Test Mode</a></span>
+<br />  <span class="subsectionToc" >18.1.4 <a
+href="#x1-37500018.1.4" id="QQ2-1-401">Starting a Simulated Cluster</a></span>
+<br />  <span class="subsectionToc" >18.1.5 <a
+href="#x1-37900018.1.5" id="QQ2-1-405">Stopping a Simulated Cluster</a></span>
+<br /> <span class="sectionToc" >18.2 <a
+href="#x1-38300018.2" id="QQ2-1-409">Job Simulation</a></span>
+<br />  <span class="subsectionToc" >18.2.1 <a
+href="#x1-38400018.2.1" id="QQ2-1-410">Overview</a></span>
-<br /> <span class="sectionToc" >18.2 <a
-href="#x1-37900018.2" id="QQ2-1-403">Backup</a></span>
+<br />  <span class="subsectionToc" >18.2.2 <a
+href="#x1-38500018.2.2" id="QQ2-1-411">Job meta-descriptors</a></span>
+<br />  <span class="subsectionToc" >18.2.3 <a
+href="#x1-38600018.2.3" id="QQ2-1-412"><span
+class="cmti-10">Prepare </span>Descriptors</a></span>
+<br />  <span class="subsectionToc" >18.2.4 <a
+href="#x1-38700018.2.4" id="QQ2-1-413">Services</a></span>
+<br />  <span class="subsectionToc" >18.2.5 <a
+href="#x1-38900018.2.5" id="QQ2-1-415">Generating a Job Set</a></span>
+<br />  <span class="subsectionToc" >18.2.6 <a
+href="#x1-39000018.2.6" id="QQ2-1-416">Running the Test Driver</a></span>
<br /> <span class="sectionToc" >18.3 <a
-href="#x1-38000018.3" id="QQ2-1-404">Sub-directories</a></span>
-<br />  <span class="subsectionToc" >18.3.1 <a
-href="#x1-38100018.3.1" id="QQ2-1-405">agents</a></span>
-<br />  <span class="subsectionToc" >18.3.2 <a
-href="#x1-38200018.3.2" id="QQ2-1-406">daemons</a></span>
-<br />  <span class="subsectionToc" >18.3.3 <a
-href="#x1-38300018.3.3" id="QQ2-1-407">database</a></span>
-<br /> <span class="sectionToc" >18.4 <a
-href="#x1-38400018.4" id="QQ2-1-408">Files</a></span>
-<br />  <span class="subsectionToc" >18.4.1 <a
-href="#x1-38500018.4.1" id="QQ2-1-409">cassandra.pid</a></span>
-<br />  <span class="subsectionToc" >18.4.2 <a
-href="#x1-38600018.4.2" id="QQ2-1-410">duccling.version</a></span>
-<br />  <span class="subsectionToc" >18.4.3 <a
-href="#x1-38700018.4.3" id="QQ2-1-411">orchestrator.properties</a></span>
-<br />  <span class="subsectionToc" >18.4.4 <a
-href="#x1-38800018.4.4" id="QQ2-1-412">orchestrator-state.json</a></span>
-<br />  <span class="subsectionToc" >18.4.5 <a
-href="#x1-38900018.4.5" id="QQ2-1-413">sm.properties</a></span>
+href="#x1-39100018.3" id="QQ2-1-417">Pre-Packaged Tests</a></span>
<br /><span class="chapterToc" >19 <a
-href="#x1-39000019" id="QQ2-1-414">Understanding the DUCC logs</a></span>
+href="#x1-39200019" id="QQ2-1-418">State Directory</a></span>
<br /> <span class="sectionToc" >19.1 <a
-href="#x1-39100019.1" id="QQ2-1-415">Overview</a></span>
+href="#x1-39300019.1" id="QQ2-1-419">Overview</a></span>
<br /> <span class="sectionToc" >19.2 <a
-href="#x1-39200019.2" id="QQ2-1-416">Resource Manager Log (rm.log)</a></span>
-<br />  <span class="subsectionToc" >19.2.1 <a
-href="#x1-39300019.2.1" id="QQ2-1-417">Bootstrap Configuration</a></span>
-<br />  <span class="subsectionToc" >19.2.2 <a
-href="#x1-39800019.2.2" id="QQ2-1-422">Node Arrival and Missed Heartbeats</a></span>
-<br />  <span class="subsectionToc" >19.2.3 <a
-href="#x1-40100019.2.3" id="QQ2-1-425">Node Occupancy</a></span>
-<br />  <span class="subsectionToc" >19.2.4 <a
-href="#x1-40200019.2.4" id="QQ2-1-426">Job Arrival and Status Updates</a></span>
-<br />  <span class="subsectionToc" >19.2.5 <a
-href="#x1-40500019.2.5" id="QQ2-1-429">Calculation Of Job Caps</a></span>
-<br />  <span class="subsectionToc" >19.2.6 <a
-href="#x1-40600019.2.6" id="QQ2-1-430">The “how much” calculations</a></span>
-<br />  <span class="subsectionToc" >19.2.7 <a
-href="#x1-40700019.2.7" id="QQ2-1-431">The “what of” calculations</a></span>
-<br />  <span class="subsectionToc" >19.2.8 <a
-href="#x1-40800019.2.8" id="QQ2-1-432">Defragmentation</a></span>
-<br />  <span class="subsectionToc" >19.2.9 <a
-href="#x1-40900019.2.9" id="QQ2-1-433">Published Schedule</a></span>
+href="#x1-39400019.2" id="QQ2-1-420">Backup</a></span>
<br /> <span class="sectionToc" >19.3 <a
-href="#x1-41200019.3" id="QQ2-1-436">Service Manager Log (sm.log)</a></span>
+href="#x1-39500019.3" id="QQ2-1-421">Sub-directories</a></span>
<br />  <span class="subsectionToc" >19.3.1 <a
-href="#x1-41300019.3.1" id="QQ2-1-437">Bootstrap configuration</a></span>
+href="#x1-39600019.3.1" id="QQ2-1-422">agents</a></span>
<br />  <span class="subsectionToc" >19.3.2 <a
-href="#x1-41800019.3.2" id="QQ2-1-442">Receipt and analysis of Orchestrator State</a></span>
+href="#x1-39700019.3.2" id="QQ2-1-423">daemons</a></span>
<br />  <span class="subsectionToc" >19.3.3 <a
-href="#x1-41900019.3.3" id="QQ2-1-443">CLI Requests</a></span>
-<br />  <span class="subsectionToc" >19.3.4 <a
-href="#x1-42000019.3.4" id="QQ2-1-444">Dispatching / Startup of Service Instances</a></span>
-<br />  <span class="subsectionToc" >19.3.5 <a
-href="#x1-42100019.3.5" id="QQ2-1-445">Progression of Service State</a></span>
-<br />  <span class="subsectionToc" >19.3.6 <a
-href="#x1-42200019.3.6" id="QQ2-1-446">Starting and Logging Pingers</a></span>
-<br />  <span class="subsectionToc" >19.3.7 <a
-href="#x1-42300019.3.7" id="QQ2-1-447">Publishing State</a></span>
+href="#x1-39800019.3.3" id="QQ2-1-424">database</a></span>
<br /> <span class="sectionToc" >19.4 <a
-href="#x1-42400019.4" id="QQ2-1-448">Orchestrator Log (or.log)</a></span>
-<br /> <span class="sectionToc" >19.5 <a
-href="#x1-42500019.5" id="QQ2-1-449">Process Manager Log (pm.log)</a></span>
-<br /> <span class="sectionToc" >19.6 <a
-href="#x1-42600019.6" id="QQ2-1-450">Agent log Log (hostname.agent.log)</a></span>
+href="#x1-39900019.4" id="QQ2-1-425">Files</a></span>
+<br />  <span class="subsectionToc" >19.4.1 <a
+href="#x1-40000019.4.1" id="QQ2-1-426">cassandra.pid</a></span>
+<br />  <span class="subsectionToc" >19.4.2 <a
+href="#x1-40100019.4.2" id="QQ2-1-427">duccling.version</a></span>
+<br />  <span class="subsectionToc" >19.4.3 <a
+href="#x1-40200019.4.3" id="QQ2-1-428">orchestrator.properties</a></span>
+<br />  <span class="subsectionToc" >19.4.4 <a
+href="#x1-40300019.4.4" id="QQ2-1-429">orchestrator-state.json</a></span>
+<br />  <span class="subsectionToc" >19.4.5 <a
+href="#x1-40400019.4.5" id="QQ2-1-430">sm.properties</a></span>
+<br /><span class="chapterToc" >20 <a
+href="#x1-40500020" id="QQ2-1-431">Understanding the DUCC logs</a></span>
+<br /> <span class="sectionToc" >20.1 <a
+href="#x1-40600020.1" id="QQ2-1-432">Overview</a></span>
+<br /> <span class="sectionToc" >20.2 <a
+href="#x1-40700020.2" id="QQ2-1-433">Resource Manager Log (rm.log)</a></span>
+<br />  <span class="subsectionToc" >20.2.1 <a
+href="#x1-40800020.2.1" id="QQ2-1-434">Bootstrap Configuration</a></span>
+<br />  <span class="subsectionToc" >20.2.2 <a
+href="#x1-41300020.2.2" id="QQ2-1-439">Node Arrival and Missed Heartbeats</a></span>
+<br />  <span class="subsectionToc" >20.2.3 <a
+href="#x1-41600020.2.3" id="QQ2-1-442">Node Occupancy</a></span>
+<br />  <span class="subsectionToc" >20.2.4 <a
+href="#x1-41700020.2.4" id="QQ2-1-443">Job Arrival and Status Updates</a></span>
+<br />  <span class="subsectionToc" >20.2.5 <a
+href="#x1-42000020.2.5" id="QQ2-1-446">Calculation Of Job Caps</a></span>
+<br />  <span class="subsectionToc" >20.2.6 <a
+href="#x1-42100020.2.6" id="QQ2-1-447">The “how much” calculations</a></span>
+<br />  <span class="subsectionToc" >20.2.7 <a
+href="#x1-42200020.2.7" id="QQ2-1-448">The “what of” calculations</a></span>
+<br />  <span class="subsectionToc" >20.2.8 <a
+href="#x1-42300020.2.8" id="QQ2-1-449">Defragmentation</a></span>
+<br />  <span class="subsectionToc" >20.2.9 <a
+href="#x1-42400020.2.9" id="QQ2-1-450">Published Schedule</a></span>
+<br /> <span class="sectionToc" >20.3 <a
+href="#x1-42700020.3" id="QQ2-1-453">Service Manager Log (sm.log)</a></span>
+<br />  <span class="subsectionToc" >20.3.1 <a
+href="#x1-42800020.3.1" id="QQ2-1-454">Bootstrap configuration</a></span>
+<br />  <span class="subsectionToc" >20.3.2 <a
+href="#x1-43300020.3.2" id="QQ2-1-459">Receipt and analysis of Orchestrator State</a></span>
+<br />  <span class="subsectionToc" >20.3.3 <a
+href="#x1-43400020.3.3" id="QQ2-1-460">CLI Requests</a></span>
+<br />  <span class="subsectionToc" >20.3.4 <a
+href="#x1-43500020.3.4" id="QQ2-1-461">Dispatching / Startup of Service Instances</a></span>
+<br />  <span class="subsectionToc" >20.3.5 <a
+href="#x1-43600020.3.5" id="QQ2-1-462">Progression of Service State</a></span>
+
+
+<br />  <span class="subsectionToc" >20.3.6 <a
+href="#x1-43700020.3.6" id="QQ2-1-463">Starting and Logging Pingers</a></span>
+<br />  <span class="subsectionToc" >20.3.7 <a
+href="#x1-43800020.3.7" id="QQ2-1-464">Publishing State</a></span>
+<br /> <span class="sectionToc" >20.4 <a
+href="#x1-43900020.4" id="QQ2-1-465">Orchestrator Log (or.log)</a></span>
+<br /> <span class="sectionToc" >20.5 <a
+href="#x1-44000020.5" id="QQ2-1-466">Process Manager Log (pm.log)</a></span>
+<br /> <span class="sectionToc" >20.6 <a
+href="#x1-44100020.6" id="QQ2-1-467">Agent log Log (hostname.agent.log)</a></span>
</div>
@@ -606,10 +624,13 @@ href="#x1-245007r2">Nodepools: Overlappi
href="#x1-245008r3">Nodepools: Multiple
top-level Nodepools</a></span><br /><span class="lofToc" >13.4 <a
href="#x1-247007r4">Sample Nodepool Configuration</a></span><br /><span class="lofToc" >13.5 <a
-href="#x1-248001r5">Sample Class
-Configuration</a></span><br /><span class="lofToc" >13.6 <a
-href="#x1-252001r6">Sample Node Configuration</a></span><br /><span class="lofToc" >13.7 <a
-href="#x1-253001r7">Sample User Registration</a></span><br />
+href="#x1-247008r5">Sample Nodepool
+regex specification</a></span><br /><span class="lofToc" >13.6 <a
+href="#x1-247009r6">Sample file contents for jobdriver.nodes.regex </a></span><br /><span class="lofToc" >13.7 <a
+href="#x1-248001r7">Sample
+Class Configuration</a></span><br /><span class="lofToc" >13.8 <a
+href="#x1-252001r8">Sample Node Configuration</a></span><br /><span class="lofToc" >13.9 <a
+href="#x1-253001r9">Sample User Registration</a></span><br />
</div>
@@ -636,25 +657,27 @@ pipelines over a cluster of computing re
DUCC defines a formal job model that closely maps to a standard UIMA pipeline. Around this job model
DUCC provides cluster management services to automate the scale-out of UIMA pipelines over computing
clusters.
-<!--l. 39--><p class="noindent" >
+<!--l. 39--><p class="noindent" >As of DUCC version 3.0.0 both UIMAv2 and UIMAv3 applications are supported. Although DUCC distributes a UIMA-AS
+runtime, it only uses UIMA classes specified by the user’s application classpath.
+<!--l. 44--><p class="noindent" >
<h3 class="sectionHead"><span class="titlemark">1.2 </span> <a
id="x1-80001.2"></a>DUCC Job Model</h3>
-<!--l. 41--><p class="noindent" >The Job Model defines the steps necessary to scale-up a UIMA pipeline using DUCC. The goal of DUCC is to
+<!--l. 46--><p class="noindent" >The Job Model defines the steps necessary to scale-up a UIMA pipeline using DUCC. The goal of DUCC is to
scale-up any UIMA pipeline, including pipelines that must be deployed across multiple machines using shared
services.
-<!--l. 45--><p class="noindent" >The DUCC Job model consists of standard UIMA components: a Collection Reader (CR), a CAS Multiplier (CM),
+<!--l. 50--><p class="noindent" >The DUCC Job model consists of standard UIMA components: a Collection Reader (CR), a CAS Multiplier (CM),
application logic as implemented one or more Analysis Engines (AE), and a CAS Consumer (CC).
-<!--l. 49--><p class="noindent" >The Collection Reader builds input CASs and forwards them to the UIMA pipelines. In the DUCC model, the CR is run in a
+<!--l. 54--><p class="noindent" >The Collection Reader builds input CASs and forwards them to the UIMA pipelines. In the DUCC model, the CR is run in a
process separate from the rest of the pipeline. In fact, in all but the smallest clusters it is run on a different physical machine
than the rest of the pipeline. To achieve scalability, the CR must create very small CASs that do not contain application
data, but which contain references to data; for instance, file names. Ideally, the CR should be runnable in a process
not much larger than the smallest Java virtual machine. Later sections demonstrate methods for achieving
this.
-<!--l. 57--><p class="noindent" >Each pipeline must contain at least one CAS Multiplier which receives the CASs from the CR. The CMs encapsulate the
+<!--l. 62--><p class="noindent" >Each pipeline must contain at least one CAS Multiplier which receives the CASs from the CR. The CMs encapsulate the
knowledge of how to receive the data references in the small CASs received from the CRs and deliver the referenced data to
the application pipeline. DUCC packages the CM, AE(s), and CC into a single process, multiple instances of which are then
deployed over the cluster.
-<!--l. 63--><p class="noindent" >A DUCC job therefore consists of a small specification containing the following items:
+<!--l. 68--><p class="noindent" >A DUCC job therefore consists of a small specification containing the following items:
<ul class="itemize1">
<li class="itemize">The name of a resource containing the CR descriptor.
</li>
@@ -663,14 +686,14 @@ deployed over the cluster.
<li class="itemize">The name of a resource containing the AE descriptor.
</li>
<li class="itemize">The name of a resource containing the CC descriptor.
+
+
</li>
<li class="itemize">Other information required to parameterize the above and identify the job such as log directory, working
directory, desired scale-out, classpath, etc. These are described in detail in subsequent sections.</li></ul>
-
-
-<!--l. 75--><p class="noindent" >On job submission, DUCC creates a single process executing the CR and one or more processes containing the analysis
+<!--l. 80--><p class="noindent" >On job submission, DUCC creates a single process executing the CR and one or more processes containing the analysis
pipeline.
-<!--l. 78--><p class="noindent" >DUCC provides other facilities in support of scale-out:
+<!--l. 83--><p class="noindent" >DUCC provides other facilities in support of scale-out:
<ul class="itemize1">
<li class="itemize">The ability to reserve all or part of a node in the cluster.
</li>
@@ -683,18 +706,18 @@ pipeline.
<li class="itemize">A web server to display and manage work and cluster status.
</li>
<li class="itemize">A CLI and a Java API to support the above.</li></ul>
-<!--l. 89--><p class="noindent" >
+<!--l. 94--><p class="noindent" >
<h3 class="sectionHead"><span class="titlemark">1.3 </span> <a
id="x1-90001.3"></a>DUCC From UIMA to Full Scale-out</h3>
-<!--l. 91--><p class="noindent" >In this section we demonstrate the progression of a simple UIMA pipeline to a fully scaled-out job running under
+<!--l. 96--><p class="noindent" >In this section we demonstrate the progression of a simple UIMA pipeline to a fully scaled-out job running under
DUCC.
-<!--l. 94--><p class="noindent" ><span class="paragraphHead"><a
+<!--l. 99--><p class="noindent" ><span class="paragraphHead"><a
id="x1-100001.3"></a><span
class="cmbx-10">UIMA Pipelines</span></span>
A normal UIMA pipeline contains a Collection Reader (CR), one or more Analysis Engines (AE) connected in a pipeline, and
a CAS Consumer (CC) as shown in <a
href="#x1-10001r1">Figure  1.1</a>.
-<!--l. 99--><p class="noindent" ><hr class="figure"><div class="figure"
+<!--l. 104--><p class="noindent" ><hr class="figure"><div class="figure"
>
@@ -703,7 +726,7 @@ href="#x1-10001r1">Figure  1.1</a
-<!--l. 101--><p class="noindent" ><img
+<!--l. 106--><p class="noindent" ><img
src="images/uima-pipeline.jpg" alt="PIC"
>
<br /> <div class="caption"
@@ -711,8 +734,8 @@ src="images/uima-pipeline.jpg" alt="PIC"
class="content">Standard UIMA Pipeline</span></div><!--tex4ht:label?: x1-10001r1 -->
-<!--l. 104--><p class="noindent" ></div><hr class="endfigure">
-<!--l. 106--><p class="noindent" ><span class="paragraphHead"><a
+<!--l. 109--><p class="noindent" ></div><hr class="endfigure">
+<!--l. 111--><p class="noindent" ><span class="paragraphHead"><a
id="x1-110001.3"></a><span
class="cmbx-10">UIMA-AS Scaled Pipeline</span></span>
With UIMA-AS the CR is separated into a discrete process and a CAS Multiplier (CM) is introduced into the pipeline as an
@@ -721,7 +744,7 @@ href="#x1-11001r2">Figure  1.2</a
scaled-out over a computing cluster. The difficulty with this model is that each user is individually responsible for finding and
scheduling computing nodes, installing communication software such as ActiveMQ, and generally managing the distributed
job and associated hardware.
-<!--l. 116--><p class="noindent" ><hr class="figure"><div class="figure"
+<!--l. 121--><p class="noindent" ><hr class="figure"><div class="figure"
>
@@ -730,7 +753,7 @@ job and associated hardware.
-<!--l. 118--><p class="noindent" ><img
+<!--l. 123--><p class="noindent" ><img
src="images/uima-as-pipeline.png" alt="PIC"
>
<br /> <div class="caption"
@@ -738,24 +761,24 @@ src="images/uima-as-pipeline.png" alt="P
class="content">UIMA Pipeline As Scaled by UIMA-AS</span></div><!--tex4ht:label?: x1-11001r2 -->
-<!--l. 121--><p class="noindent" ></div><hr class="endfigure">
-<!--l. 123--><p class="noindent" ><span class="paragraphHead"><a
+<!--l. 126--><p class="noindent" ></div><hr class="endfigure">
+<!--l. 128--><p class="noindent" ><span class="paragraphHead"><a
id="x1-120001.3"></a><span
class="cmbx-10">UIMA Pipeline Scaled By DUCC</span></span>
DUCC is a UIMA and UIMA-AS-aware cluster manager. To scale out work under DUCC the developer tells DUCC what
the parts of the application are, and DUCC does the work to build the scale-out via UIMA/AS, to find and
schedule resources, to deploy the parts of the application over the cluster, and to manage the jobs while it
executes.
-<!--l. 129--><p class="noindent" >On job submission, the CR is wrapped with a DUCC main class and launched as a Job Driver (or JD). The DUCC main
+<!--l. 134--><p class="noindent" >On job submission, the CR is wrapped with a DUCC main class and launched as a Job Driver (or JD). The DUCC main
class establishes communication with other DUCC components and instantiates the CR. If the CR initializes
successfully, and indicates that there are greater than 0 work items to process, the specified CM, AE and CC
components are assembled into an aggregate, wrapped with a DUCC main class, and launched as a Job Process (or
JP).
-<!--l. 135--><p class="noindent" >The JP will replicate the aggregate as many times as specified, each aggregate instance running in a single thread. When the
+<!--l. 140--><p class="noindent" >The JP will replicate the aggregate as many times as specified, each aggregate instance running in a single thread. When the
aggregate initializes, and whenever an aggregate thread needs work, the JP wrapper will fetch the next work item from the
JD, as shown in <a
href="#x1-12001r3">Figure  1.3</a> below.
-<!--l. 140--><p class="noindent" ><hr class="figure"><div class="figure"
+<!--l. 145--><p class="noindent" ><hr class="figure"><div class="figure"
>
@@ -764,7 +787,7 @@ href="#x1-12001r3">Figure  1.3</a
-<!--l. 142--><p class="noindent" ><img
+<!--l. 147--><p class="noindent" ><img
src="images/ducc-sequential.png" alt="PIC"
>
<br /> <div class="caption"
@@ -772,8 +795,8 @@ src="images/ducc-sequential.png" alt="PI
class="content">UIMA Pipeline As Automatically Scaled Out By DUCC</span></div><!--tex4ht:label?: x1-12001r3 -->
-<!--l. 145--><p class="noindent" ></div><hr class="endfigure">
-<!--l. 147--><p class="noindent" ><span class="paragraphHead"><a
+<!--l. 150--><p class="noindent" ></div><hr class="endfigure">
+<!--l. 152--><p class="noindent" ><span class="paragraphHead"><a
id="x1-130001.3"></a><span
class="cmbx-10">UIMA Pipeline with User-Supplied DD Scaled By DUCC</span></span>
Application programmers may supply their own Deployment Descriptors to control intra-process threading and scale-out. If a
@@ -781,7 +804,7 @@ DD is specified in the job parameters, D
as depicted in <a
href="#x1-13001r4">Figure  1.4</a> below. In this case the user can still specify how many work items to deliver to the service
concurrently.
-<!--l. 155--><p class="noindent" ><hr class="figure"><div class="figure"
+<!--l. 160--><p class="noindent" ><hr class="figure"><div class="figure"
>
@@ -790,7 +813,7 @@ concurrently.
-<!--l. 157--><p class="noindent" ><img
+<!--l. 162--><p class="noindent" ><img
src="images/ducc-parallel.png" alt="PIC"
>
<br /> <div class="caption"
@@ -798,10 +821,10 @@ src="images/ducc-parallel.png" alt="PIC"
class="content">UIMA Pipeline With User-Supplied DD as Automatically Scaled Out By DUCC</span></div><!--tex4ht:label?: x1-13001r4 -->
-<!--l. 160--><p class="noindent" ></div><hr class="endfigure">
+<!--l. 165--><p class="noindent" ></div><hr class="endfigure">
<h3 class="sectionHead"><span class="titlemark">1.4 </span> <a
id="x1-140001.4"></a>Error Management </h3>
-<!--l. 164--><p class="noindent" >DUCC provides a number of facilities to assist error management:
+<!--l. 169--><p class="noindent" >DUCC provides a number of facilities to assist error management:
<ul class="itemize1">
<li class="itemize">DUCC captures exceptions in the JPs and delivers them to the Job Drivers. The JD wrappers implement logic
to enforce error thresholds, to identify and log errors, and to reflect job problems in the DUCC Web Server.
@@ -818,10 +841,10 @@ class="content">UIMA Pipeline With User-
</li>
<li class="itemize">Various error conditions encountered while a job is running will prevent a problematic job from continuing
scale out, and can result in termination of the job.</li></ul>
-<!--l. 186--><p class="noindent" >
+<!--l. 191--><p class="noindent" >
<h3 class="sectionHead"><span class="titlemark">1.5 </span> <a
id="x1-150001.5"></a>Cluster and Job Management</h3>
-<!--l. 187--><p class="noindent" >DUCC supports management of multiple jobs and multiple users in a distributed cluster:
+<!--l. 192--><p class="noindent" >DUCC supports management of multiple jobs and multiple users in a distributed cluster:
<dl class="description"><dt class="description">
<span
class="cmbx-10">Multiple User Support</span> </dt><dd
@@ -902,10 +925,10 @@ class="description">DUCC provides system
</dd></dl>
-<!--l. 256--><p class="noindent" >
+<!--l. 261--><p class="noindent" >
<h3 class="sectionHead"><span class="titlemark">1.6 </span> <a
id="x1-160001.6"></a>Security Measures</h3>
-<!--l. 257--><p class="noindent" >The following DUCC security measures are provided:
+<!--l. 262--><p class="noindent" >The following DUCC security measures are provided:
<dl class="description"><dt class="description">
<span
class="cmbx-10">user credentials</span> </dt><dd
@@ -929,7 +952,7 @@ class="description">The webserver facili
</li>
<li class="itemize">Each administrator has the ability to control certain aspects of any user’s active submissions, as well as
modification of some DUCC operational characteristics.</li></ul>
- <!--l. 275--><p class="noindent" >A simple interface is provided so that an installation can plug-in a site specific authentication mechanism comprising
+ <!--l. 280--><p class="noindent" >A simple interface is provided so that an installation can plug-in a site specific authentication mechanism comprising
userid and password.
</dd><dt class="description">
<span
@@ -938,13 +961,13 @@ class="description">DUCC uses ActiveMQ f
processes from participating. But when testing DUCC on a simulated cluster the AMQ broker runs without any access
restrictions so that it can be used as an application broker for UIMA-AS services used in simulation tests. See
<a
-href="#x1-36000017.1.4"><span
+href="#x1-37500018.1.4"><span
class="cmti-10">start</span><span
class="cmti-10">_sim</span></a>.</dd></dl>
-<!--l. 285--><p class="noindent" >
+<!--l. 290--><p class="noindent" >
<h4 class="subsectionHead"><span class="titlemark">1.6.1 </span> <a
id="x1-170001.6.1"></a>ducc_ling</h4>
-<!--l. 287--><p class="noindent" >ducc_ling contains the following functions, which the security-conscious may verify by examining the source in
+<!--l. 292--><p class="noindent" >ducc_ling contains the following functions, which the security-conscious may verify by examining the source in
$DUCC_HOME/duccling. All sensitive operations are performed only AFTER switching userids, to prevent unauthorized
root access to the system.
<ul class="itemize1">
@@ -978,10 +1001,10 @@ class="cmti-10">ducc </span>may use the
user <span
class="cmti-10">root </span>from using it for privileged operations.
</li></ul>
-<!--l. 312--><p class="noindent" >
+<!--l. 317--><p class="noindent" >
<h3 class="sectionHead"><span class="titlemark">1.7 </span> <a
id="x1-180001.7"></a>Security Issues</h3>
-<!--l. 313--><p class="noindent" >The following DUCC security issues should be considered:
+<!--l. 318--><p class="noindent" >The following DUCC security issues should be considered:
<dl class="description"><dt class="description">
<span
class="cmbx-10">submit transmission ’sniffed’</span> </dt><dd
@@ -1066,6 +1089,12 @@ class="cmbx-10">Machine</span> </dt><dd
class="description">A physical computing resource managed by the DUCC Resource Manager.
</dd><dt class="description">
<span
+class="cmbx-10">Managed Reservation</span> </dt><dd
+class="description">A DUCC managed reservation comprises an arbitrary process that is deployed on the
+ computing cluster within a <span
+class="cmti-10">share </span>assigned by the DUCC scheduler.
+ </dd><dt class="description">
+<span
class="cmbx-10">Node</span> </dt><dd
class="description">See Machine.
</dd><dt class="description">
@@ -1083,13 +1112,13 @@ class="cmti-10">shares </span>by the DUC
class="cmbx-10">Process Manager (PM)</span> </dt><dd
class="description">The Process Manager coordinates distribution of work among the Agents.
</dd><dt class="description">
+
+
<span
class="cmbx-10">Registered Service</span> </dt><dd
class="description">A registered service is a service that is registered with DUCC. DUCC saves the service
specification and fully manages the service, insuring it is running when needed, and shutdown when not.
</dd><dt class="description">
-
-
<span
class="cmbx-10">Resource Manager (RM)</span> </dt><dd
class="description">The Resource Manager schedules physical resources for DUCC work.
@@ -1827,6 +1856,44 @@ class="description">This specifies the m
<span
class="cmsy-10">--</span><span
class="cmbx-10">process</span><span
+class="cmbx-10">_error</span><span
+class="cmbx-10">_window</span><span
+class="cmbx-10">_threshold [integer]</span> </dt><dd
+class="description">This specifies an upper bound for number of process errors a JP can
+ tolerate. When this threshold is reached, the JP will terminate. If not specified, the default is 1 which means terminate
+ on first process error. Example:
+
+
+ <div class="verbatim" id="verbatim-22">
+      --process_error_window_threshold 2
+  <br />             </div>
+ <!--l. 304--><p class="nopar" >
+ </dd><dt class="description">
+<span
+class="cmsy-10">--</span><span
+class="cmbx-10">process</span><span
+class="cmbx-10">_error</span><span
+class="cmbx-10">_window</span><span
+class="cmbx-10">_size [integer]</span> </dt><dd
+class="description">This specifies a window the JP error handler uses to determine if it should
+ terminate on process error. When a defined number of errors in a given window is reached, the JP will terminate. If not
+ specified, the default is 1 which means terminate on first process error. This property is used together with
+ <span
+class="cmsy-10">--</span><span
+class="cmti-10">process</span><span
+class="cmti-10">_error</span><span
+class="cmti-10">_window</span><span
+class="cmti-10">_threshold</span>. Example:
+
+
+ <div class="verbatim" id="verbatim-23">
+      --process_error_window_size 10
+  <br />             </div>
+ <!--l. 312--><p class="nopar" >
+ </dd><dt class="description">
+<span
+class="cmsy-10">--</span><span
+class="cmbx-10">process</span><span
class="cmbx-10">_pipeline</span><span
class="cmbx-10">_count [integer]</span> </dt><dd
class="description">This specifies the number of pipelines per process to be deployed, i.e.
@@ -1836,10 +1903,10 @@ class="description">This specifies the n
Example:
- <div class="verbatim" id="verbatim-22">
+ <div class="verbatim" id="verbatim-24">
     --process_pipeline_count 7
 <br />             </div>
- <!--l. 307--><p class="nopar" >Alias: <span
+ <!--l. 322--><p class="nopar" >Alias: <span
class="cmsy-10">--</span>process_thread_count
</dd><dt class="description">
<span
@@ -1852,10 +1919,10 @@ class="description">This specifies the n
href="#x1-24800013.6.2">here.</a> Example:
- <div class="verbatim" id="verbatim-23">
+ <div class="verbatim" id="verbatim-25">
     --scheduling_class normal
 <br />             </div>
- <!--l. 318--><p class="nopar" >
+ <!--l. 333--><p class="nopar" >
</dd><dt class="description">
<span
class="cmsy-10">--</span><span
@@ -1866,10 +1933,10 @@ class="description">This specifies a bla
href="#x1-950005.4">here</a>. Example:
- <div class="verbatim" id="verbatim-24">
+ <div class="verbatim" id="verbatim-26">
     --service_dependency UIMA-AS:Service1:tcp:host1:61616 UIMA-AS:Service2:tcp:host2:123
 <br />             </div>
- <!--l. 325--><p class="nopar" >
+ <!--l. 340--><p class="nopar" >
</dd><dt class="description">
<span
class="cmsy-10">--</span><span
@@ -1880,17 +1947,17 @@ class="description">All the parameters u
file may then be used to submit the job (rather than providing all the parameters directory to submit). The leading <span
class="cmsy-10">--</span>
is omitted from the keywords.
- <!--l. 332--><p class="noindent" >For example,
+ <!--l. 347--><p class="noindent" >For example,
- <div class="verbatim" id="verbatim-25">
+ <div class="verbatim" id="verbatim-27">
     ducc_submit --specification job.props
 <br />     ducc_submit -f job.props
 <br />             </div>
- <!--l. 336--><p class="nopar" >where job.props contains:
+ <!--l. 351--><p class="nopar" >where job.props contains:
- <div class="verbatim" id="verbatim-26">
+ <div class="verbatim" id="verbatim-28">
working_directory                   = /home/bob/projects/ducc/ducc_test/test/bin
 <br />process_failures_limit              = 20
 <br />driver_descriptor_CR                = org.apache.uima.ducc.test.randomsleep.FixedSleepCR
@@ -1907,7 +1974,7 @@ class="cmsy-10">--</span>
 <br />process_jvm_args                    = -Xmx100M -DdefaultBrokerURL=tcp://localhost:61616
 <br />scheduling_class                    = normal
 <br />process_memory_size                 = 15</div>
- <!--l. 355--><p class="nopar" >Note that properties in a specification file may be overridden by other command-line parameters, as discussed
+ <!--l. 370--><p class="nopar" >Note that properties in a specification file may be overridden by other command-line parameters, as discussed
<a
href="#x1-210003">here</a>.
</dd><dt class="description">
@@ -1945,7 +2012,7 @@ class="cmbx-10">working</span><span
class="cmbx-10">_directory</span> </dt><dd
class="description">This specifies the working directory to be set by the Job Driver and Job Process processes. If not
specified, the current directory is used.</dd></dl>
-<!--l. 377--><p class="noindent" ><span class="paragraphHead"><a
+<!--l. 392--><p class="noindent" ><span class="paragraphHead"><a
id="x1-310003.5"></a><span
class="cmbx-10">Notes:</span></span>
<a
@@ -2003,7 +2070,7 @@ class="description">Optional. This speci
the shell requires a quoted string. Example:
- <div class="verbatim" id="verbatim-27">
+ <div class="verbatim" id="verbatim-29">
ducc_cancel --id 12 --reason "This is a pretty good reason."</div>
<!--l. 48--><p class="nopar" >
</dd><dt class="description">
@@ -2275,7 +2342,7 @@ class="description">Blank-delimited list
prefix. If specified, this is used for all DUCC processes in the job. Example:
- <div class="verbatim" id="verbatim-28">
+ <div class="verbatim" id="verbatim-30">
--environment TERM=xterm DISPLAY=:1.0 LANG UIMA_*</div>
<!--l. 68--><p class="nopar" >
<!--l. 71--><p class="noindent" >Additional entries may be copied from the user’s environment based on the setting of ducc.submit.environment.propagated
@@ -2297,7 +2364,7 @@ class="description">
Example:
- <div class="verbatim" id="verbatim-29">
+ <div class="verbatim" id="verbatim-31">
--log_directory /home/bob</div>
<!--l. 86--><p class="nopar" >
<!--l. 88--><p class="noindent" >Within this directory DUCC creates a sub-directory for each process, using the numerical ID of the job. The format of
@@ -2354,14 +2421,14 @@ class="description">All the parameters u
<!--l. 119--><p class="noindent" >For example,
- <div class="verbatim" id="verbatim-30">
+ <div class="verbatim" id="verbatim-32">
ducc_process_submit --specification job.props
 <br />ducc_process_submit -f job.props</div>
<!--l. 123--><p class="nopar" >
<!--l. 125--><p class="noindent" >where job.props contains:
- <div class="verbatim" id="verbatim-31">
+ <div class="verbatim" id="verbatim-33">
working_directory   = /home/bob/projects
 <br />environment         = AE_INIT_TIME=10000 LD_LIBRARY_PATH=/a/bogus/path
 <br />log_directory       = /home/bob/ducc/logs/
@@ -2620,14 +2687,14 @@ class="description">Blank-delimited list
with the same prefix. Example:
- <div class="verbatim" id="verbatim-32">
+ <div class="verbatim" id="verbatim-34">
--environment TERM=xterm DISPLAY=:1.0 LANG UIMA_*
 <br />          </div>
<!--l. 138--><p class="nopar" >
<!--l. 140--><p class="noindent" >Additional entries may be copied from the user’s environment based on the setting of
- <div class="verbatim" id="verbatim-33">
+ <div class="verbatim" id="verbatim-35">
ducc.submit.environment.propagated</div>
<!--l. 143--><p class="nopar" >in the global DUCC configuration ducc.properties.
<!--l. 146--><p class="noindent" >Note: When used as a CLI option, the environment string must usually be quoted to protect it from the
@@ -2714,7 +2781,7 @@ href="#x1-930005.2">UIMA-AS services</a>
Example:
- <div class="verbatim" id="verbatim-34">
+ <div class="verbatim" id="verbatim-36">
--jvm /share/jdk1.6/bin/java</div>
<!--l. 197--><p class="nopar" >
</dd><dt class="description">
@@ -2728,7 +2795,7 @@ href="#x1-930005.2">UIMA-AS
services</a>. It is a blank-delimited list of strings. Example:
- <div class="verbatim" id="verbatim-35">
+ <div class="verbatim" id="verbatim-37">
--process_jvm_args -Xmx100M -Xms50M</div>
<!--l. 206--><p class="nopar" >
<!--l. 208--><p class="noindent" >Note: When used as a CLI option, the argument string must usually be quoted to protect it from the
@@ -2742,7 +2809,7 @@ class="description">This specifies the p
If not specified, the default is $HOME/ducc/logs. Example:
- <div class="verbatim" id="verbatim-36">
+ <div class="verbatim" id="verbatim-38">
--log_directory /home/bob</div>
<!--l. 215--><p class="nopar" >
<!--l. 217--><p class="noindent" >Within this directory DUCC creates a subdirectory for each job, using the numerical ID of the job. The format of the
@@ -2817,7 +2884,7 @@ class="description">This specifies a bla
href="#x1-950005.4">here</a>. Example:
- <div class="verbatim" id="verbatim-37">
+ <div class="verbatim" id="verbatim-39">
--service_dependency UIMA-AS:Service1:tcp:node682:61616 UIMA-AS:OtherSvc:tcp:node123:123</div>
<!--l. 259--><p class="nopar" >
<!--l. 261--><p class="noindent" >Note: When used as a CLI option, the list must usually be quoted to protect it from the shell.
@@ -2845,7 +2912,7 @@ class="cmti-10">null </span>is passed
blanks):
- <div class="verbatim" id="verbatim-38">
+ <div class="verbatim" id="verbatim-40">
     service_ping_arguments=broker-jmx-port=pppp,meta-timeout=tttt</div>
<!--l. 280--><p class="nopar" >
<!--l. 282--><p class="noindent" >The keywords in the string have the following meaning:
@@ -2859,7 +2926,7 @@ class="description">This is the JMX port
“none”.
- <div class="verbatim" id="verbatim-39">
+ <div class="verbatim" id="verbatim-41">
    service_ping_arguments=broker-jmx-port=none</div>
<!--l. 293--><p class="nopar" >
</dd><dt class="description">
@@ -2913,7 +2980,7 @@ class="cmti-10">_class </span>is specifi
running the pinger. The arguments are specified as a blank-delimited list of strings. Example:
- <div class="verbatim" id="verbatim-40">
+ <div class="verbatim" id="verbatim-42">
--service_ping_jvm_args -Xmx400M -Xms100M</div>
<!--l. 319--><p class="nopar" >
<!--l. 321--><p class="noindent" >Note: When used as a CLI option, the arguments must usually be quoted to protect them from the
@@ -2961,7 +3028,7 @@ class="description">This indicates that
assigned by DUCC when the service is registered, or the service endpoint string. Example:
- <div class="verbatim" id="verbatim-41">
+ <div class="verbatim" id="verbatim-43">
ducc_services --start 23
 <br />ducc_services --start UIMA-AS:Service23:tcp://bob.com:12345</div>
<!--l. 359--><p class="nopar" >
@@ -2976,13 +3043,13 @@ class="description">This is the number o
instances are running and
- <div class="verbatim" id="verbatim-42">
+ <div class="verbatim" id="verbatim-44">
         ducc_services --start 33 --instances 5</div>
<!--l. 369--><p class="nopar" >is issued, five more service instances are started for service 33 for a total of ten, regardless of the number specified in
the registration.
- <div class="verbatim" id="verbatim-43">
+ <div class="verbatim" id="verbatim-45">
ducc_services --start 23 --intances 5
 <br />ducc_services --start UIMA-AS:Service23:tcp://bob.com:12345 --instances 3</div>
<!--l. 375--><p class="nopar" >
@@ -3000,7 +3067,7 @@ class="description">This specifies the s
by DUCC when the service is registered, or the service endpoint string. Example:
- <div class="verbatim" id="verbatim-44">
+ <div class="verbatim" id="verbatim-46">
ducc_services --stop 23
 <br />ducc_services --stop UIMA-AS:Service23:tcp://bob.com:12345</div>
<!--l. 391--><p class="nopar" >
@@ -3013,7 +3080,7 @@ class="description">This is the number o
running for a service with numeric id 33 and
- <div class="verbatim" id="verbatim-45">
+ <div class="verbatim" id="verbatim-47">
ducc_services --stop 33 --instances 5</div>
<!--l. 399--><p class="nopar" >is issued, five (randomly selected) service instances are stopped for service 33, leaving five running. The
registered number of instances is never reduced to zero even if the number of running instances is reduced to
@@ -3021,7 +3088,7 @@ class="description">This is the number o
<!--l. 404--><p class="noindent" >Example:
- <div class="verbatim" id="verbatim-46">
+ <div class="verbatim" id="verbatim-48">
ducc_services --stop 23 --intances 5
 <br />ducc_services --stop UIMA-AS:Service23:tcp://bob.com:12345 --instances 3</div>
<!--l. 408--><p class="nopar" >
@@ -3041,7 +3108,7 @@ class="description">Removes the <span
class="cmti-10">disabled </span>status, if any. Example:
- <div class="verbatim" id="verbatim-47">
+ <div class="verbatim" id="verbatim-49">
ducc_services --enable 23
 <br />ducc_services --enable UIMA-AS:Service23:tcp://bob.com:12345</div>
<!--l. 423--><p class="nopar" >
@@ -3061,7 +3128,7 @@ class="description">sets the <span
class="cmti-10">disabled </span>status. Example:
- <div class="verbatim" id="verbatim-48">
+ <div class="verbatim" id="verbatim-50">
ducc_services --disable 23
 <br />ducc_services --disable UIMA-AS:Service23:tcp://bob.com:12345</div>
<!--l. 437--><p class="nopar" >
@@ -3085,7 +3152,7 @@ class="cmti-10">reference-started </span
Example:
- <div class="verbatim" id="verbatim-49">
+ <div class="verbatim" id="verbatim-51">
ducc_services --observe_references 23
 <br />ducc_services --observe_references UIMA-AS:Service23:tcp://bob.com:12345</div>
<!--l. 456--><p class="nopar" >
@@ -3109,7 +3176,7 @@ class="cmti-10">reference-started </span
Example:
- <div class="verbatim" id="verbatim-50">
+ <div class="verbatim" id="verbatim-52">
ducc_services --igmore_references 23
 <br />ducc_services --ignore_references UIMA-AS:Service23:tcp://bob.com:12345</div>
<!--l. 472--><p class="nopar" >
@@ -3137,7 +3204,7 @@ class="description">This identifies the
by DUCC when the service is registered, or the service endpoint string. Example:
- <div class="verbatim" id="verbatim-51">
+ <div class="verbatim" id="verbatim-53">
ducc_services --modify 23 --instances 3
 <br />ducc_services --modify UIMA-AS:Service23:tcp://bob.com:12345 --intances 2</div>
<!--l. 492--><p class="nopar" ></dd></dl>
@@ -3203,7 +3270,7 @@ class="cmtt-10">_5:tcp://bobmach:61617 <
whose DUCC Ids are 9 and 5. It is currently used (referenced) by DUCC jobs 1 and 5.
- <div class="verbatim" id="verbatim-52">
+ <div class="verbatim" id="verbatim-54">
Service: UIMA-AS:FixedSleepAE_5:tcp://bobmach291:61617
 <br />   Service Class : Registered as ID 2 Owner[bob] instances[2] linger[5000]
 <br />   Implementors : 9 8
@@ -3252,12 +3319,12 @@ resources.
class="cmbx-10">Usage:</span></span>
-<div class="verbatim" id="verbatim-53">
+<div class="verbatim" id="verbatim-55">
     viaducc [defines] [command and parameters]</div>
<!--l. 54--><p class="nopar" >or
-<div class="verbatim" id="verbatim-54">
+<div class="verbatim" id="verbatim-56">
     java_viaducc [defines] [java-class and parameters]</div>
<!--l. 58--><p class="nopar" >
<!--l. 60--><p class="noindent" >The “defines” are described below. The “command and parameters” are either any command (with full path) and it’s
@@ -3288,7 +3355,7 @@ class="description">This species additio
quoted string of blank-delimited K=V environment values. For example:
- <div class="verbatim" id="verbatim-55">
+ <div class="verbatim" id="verbatim-57">
      -DDUCC_ENVIRONMENT="DUCC_RLIMIT_NOFILE=1000 V1=V2 A=B"</div>
<!--l. 77--><p class="nopar" >
</dd><dt class="description">
@@ -3450,7 +3517,7 @@ class="cmti-10">crontab </span>to check
minutes, and to send e-mail when there is a status change.
-<div class="verbatim" id="verbatim-56">
+<div class="verbatim" id="verbatim-58">
    */5 * * * *  /home/ducc/ducc_runtime/bin/ducc_watcher --target http://uima-ducc-demo.apache.org:42133 --job-driver-allocation JobDriver --email degenaro@apache.org
 <br />
 <br />    </div>
@@ -3494,7 +3561,7 @@ output.
<!--l. 49--><p class="noindent" >Below is the “main()” method of DuccJobSubmit, demonstrating the use of the API:
-<div class="verbatim" id="verbatim-57">
+<div class="verbatim" id="verbatim-59">
       public static void main(String[] args) {
 <br />        try {
 <br />            DuccJobSubmit ds = new DuccJobSubmit(args, null);
@@ -3629,7 +3696,7 @@ may also be used in service registration
example:
-<div class="verbatim" id="verbatim-58">
+<div class="verbatim" id="verbatim-60">
        service_jvm_args   -DSERVICE_ID=${DUCC_SERVICE_INSTANCE}
 <br />        process_executable_args -i ${DUCC_SERVICE_INSTANCE}</div>
<!--l. 112--><p class="nopar" >
@@ -3642,7 +3709,7 @@ indicate dependencies on specific servic
<!--l. 120--><p class="noindent" >A service endpoint is of the form
-<div class="verbatim" id="verbatim-59">
+<div class="verbatim" id="verbatim-61">
      <service-type>:<unique id></div>
<!--l. 123--><p class="nopar" >
<!--l. 125--><p class="noindent" >The <span
@@ -3653,7 +3720,7 @@ the same as the service endpoint specifi
form:
-<div class="verbatim" id="verbatim-60">
+<div class="verbatim" id="verbatim-62">
      queue-name:broker-url</div>
<!--l. 133--><p class="nopar" >where <span
class="cmti-10">queue-name </span>is the name of the ActiveMQ queue used by the service, and <span
@@ -3661,7 +3728,7 @@ class="cmti-10">broker-url </span>is the
Sample DUCC Service endpoints:
-<div class="verbatim" id="verbatim-61">
+<div class="verbatim" id="verbatim-63">
      UIMA-AS:WikipediaSearchServices:tcp://broker1:61616
 <br />      UIMA-AS:GoogleSearchServices:http://broker2:61618</div>
<!--l. 139--><p class="nopar" >
@@ -3681,7 +3748,7 @@ over DUCC and cluster restarts.
distribution bundled with DUCC.
-<div class="verbatim" id="verbatim-62">
+<div class="verbatim" id="verbatim-64">
      export UIMA_HOME=$DUCC_HOME/apache-uima
 <br />      $UIMA_HOME/bin/startBroker.sh</div>
<!--l. 156--><p class="nopar" >This uses the default configuration of port 61616 for tcp access and 1099 for JMX access (used by the default UIMA-AS
@@ -4111,7 +4178,7 @@ class="cmti-10">getStatistics()</span>.
id="x1-104001r1"></a>
-<div class="verbatim" id="verbatim-63">
+<div class="verbatim" id="verbatim-65">
import java.io.DataInputStream;
 <br />import java.io.InputStream;
 <br />import java.net.Socket;
@@ -4174,7 +4241,7 @@ class="cmbx-10">1. Establish a compilati
One DUCC jar is required in the CLASSPATH to build your pinger:
-<div class="verbatim" id="verbatim-64">
+<div class="verbatim" id="verbatim-66">
      DUCC_HOME/lib/uima-ducc-cli.jar</div>
<!--l. 461--><p class="nopar" >This provides the definition for the <span
class="cmti-10">AServicePing </span>and <span
@@ -4185,14 +4252,14 @@ class="cmbx-10">2. Create a registration
Next, create a service registration for the pinger. While debugging, it is useful set the directive
-<div class="verbatim" id="verbatim-65">
+<div class="verbatim" id="verbatim-67">
     service_ping_dolog = true</div>
<!--l. 468--><p class="nopar" >This will log any output from <span
class="cmtt-10">System.out.println() </span>to the declared log directory for the service. If not specified in the
reqistration, this directory is:
-<div class="verbatim" id="verbatim-66">
+<div class="verbatim" id="verbatim-68">
     $HOME/ducc/logs/S-<serviceid>/services</div>
<!--l. 473--><p class="nopar" >where <span
class="cmmi-10"><</span><span
@@ -4201,7 +4268,7 @@ class="cmmi-10">> </span>is the D
<!--l. 476--><p class="noindent" >Once the pinger is debugged you may want to turn logging off.
-<div class="verbatim" id="verbatim-67">
+<div class="verbatim" id="verbatim-69">
     service_ping_dolog = false</div>
<!--l. 479--><p class="nopar" >
<!--l. 481--><p class="noindent" >If your pinger requires a different version of Java than is used by DUCC, include a setting for the JAVA_HOME variable in
@@ -4212,7 +4279,7 @@ automatically. (However you may need oth
function.)
-<div class="verbatim" id="verbatim-68">
+<div class="verbatim" id="verbatim-70">
     bash-3.2$ cat myping.svc
 <br />
 <br />     description              = Ping-only service
@@ -4241,7 +4308,7 @@ class="cmti-10">ducc</span><span
class="cmti-10">_services CLI:</span></a>
-<div class="verbatim" id="verbatim-69">
+<div class="verbatim" id="verbatim-71">
     ducc_services --modify <serviceid> --service_ping_dolog true
 <br />     ducc_services --modify <serviceid> --service_ping_class OtherCustsomPing
 <br />                                        --service_ping_classpath /myhome/OtherCustomPing.class
@@ -4260,7 +4327,7 @@ directory, as most problems with pingers
in
-<div class="verbatim" id="verbatim-70">
+<div class="verbatim" id="verbatim-72">
     $DUCC_HOME/logs/sm.log</div>
<!--l. 525--><p class="nopar" >
<!--l. 527--><p class="noindent" >
@@ -4298,7 +4365,7 @@ determine whether the pinger is to be ru
update a global pinger’s registration to “internal”, to insure such pingers are properly vetted and approved by the
installation.
<!--l. 554--><p class="noindent" >More Details of registering global pingers is found in the <a
-href="#x1-35000015"><span
+href="#x1-35900015"><span
class="cmti-10">Administration section</span></a> of this document.
<!--l. 557--><p class="noindent" >
<h3 class="sectionHead"><span class="titlemark">5.8 </span> <a
@@ -4306,7 +4373,7 @@ class="cmti-10">Administration section</
<!--l. 559--><p class="noindent" >A sample custom UIMA-AS pinger is provided in the Examples directory shipped with DUCC in
-<div class="verbatim" id="verbatim-71">
+<div class="verbatim" id="verbatim-73">
     DUCC_HOME/examples/src/org/apache/uima/ducc/ping</div>
<!--l. 563--><p class="nopar" >
<!--l. 565--><p class="noindent" >This pinger increases or decreases the number of service instances based on the queue statistics found by querying ActiveMQ.
@@ -4323,7 +4390,7 @@ class="cmti-10">_arguments</span>
are specific to this pinger.
-<div class="verbatim" id="verbatim-72">
+<div class="verbatim" id="verbatim-74">
    service_ping_class=org.apache.uima.ducc.ping.SamplePing
 <br />    service_ping_arguments=meta-timeout=15010,broker-jmx-port=1099,window=5,min=1,
 <br />                           max=20,max-growth=3,fast-shrink=true,goal=2.5
@@ -4340,7 +4407,7 @@ are specific to this pinger.
<!--l. 589--><p class="noindent" >The full source for the sample pinger is found in
-<div class="verbatim" id="verbatim-73">
+<div class="verbatim" id="verbatim-75">
    DUCC_HOME/examples/src/org/apache/uima/ducc/ping/SamplePing.java</div>
<!--l. 592--><p class="nopar" >
<!--l. 594--><p class="noindent" >The following arguments are accepted by this pinger and may be specified in a single single comma-delimited string
@@ -4633,7 +4700,7 @@ The average time a single instance takes
class="cmti-10">Ti </span>is given by the simple formula
-<div class="verbatim" id="verbatim-74">
+<div class="verbatim" id="verbatim-76">
    Ti = (eT / Q) * active</div>
<!--l. 838--><p class="nopar" >where
<dl class="description"><dt class="description">
@@ -4652,7 +4719,7 @@ class="description">is the current numbe
class="cmti-10">Ti </span>is given by
-<div class="verbatim" id="verbatim-75">
+<div class="verbatim" id="verbatim-77">
   Tt = Ti * nthreads</div>
<!--l. 850--><p class="nopar" >
<!--l. 852--><p class="noindent" ><span class="paragraphHead"><a
@@ -4662,7 +4729,7 @@ We want <span
class="cmti-10">Tt </span>to become close to the current
-<div class="verbatim" id="verbatim-76">
+<div class="verbatim" id="verbatim-78">
   Tt * goal</div>
<!--l. 856--><p class="nopar" >
<!--l. 858--><p class="noindent" >where <span
@@ -4670,7 +4737,7 @@ class="cmti-10">goal </span>is given by
by
-<div class="verbatim" id="verbatim-77">
+<div class="verbatim" id="verbatim-79">
   r = eT / g</div>
<!--l. 862--><p class="nopar" >
<!--l. 864--><p class="noindent" >Because we know that the DUCC job driver will never over-commit; that is, we know the current demand will remain
@@ -4767,7 +4834,7 @@ class="enumerate-enumitem">The UIMA pipe
(JP). The name of this file is of the form
- <div class="verbatim" id="verbatim-78">
+ <div class="verbatim" id="verbatim-80">
         JOBID-uima-ae-descriptor-PROCESS.xml</div>
<!--l. 43--><p class="nopar" >where
<dl class="description"><dt class="description">
@@ -4784,7 +4851,7 @@ class="enumerate-enumitem">The UIMA-AS s
file is of the form
- <div class="verbatim" id="verbatim-79">
+ <div class="verbatim" id="verbatim-81">
         JOBID-uima-as-dd-PROCESS.xml</div>
<!--l. 54--><p class="nopar" >
<!--l. 56--><p class="noindent" >where
@@ -4807,7 +4874,7 @@ The Job Process logs are written to the
for the job. The log names are of the following form:
-<div class="verbatim" id="verbatim-80">
+<div class="verbatim" id="verbatim-82">
         JOBID-TYPE-NODE-PROCESS.log</div>
<!--l. 71--><p class="nopar" >where
[... 6583 lines stripped ...]