You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2013/07/23 17:09:20 UTC
svn commit: r1506103 - in
/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5:
c01-Agent.tex ducc-pops.tex
Author: degenaro
Date: Tue Jul 23 15:09:20 2013
New Revision: 1506103
URL: http://svn.apache.org/r1506103
Log:
UIMA-2864 DUCC POPs
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/c01-Agent.tex
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops.tex
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/c01-Agent.tex
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/c01-Agent.tex?rev=1506103&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/c01-Agent.tex (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/c01-Agent.tex Tue Jul 23 15:09:20 2013
@@ -0,0 +1,4 @@
+\begin{description}
+ \item deploy, monitor and control one or more processes supporting analytic pipelines on one node;
+ \item publish inventory and metrics;
+\end{description}
\ No newline at end of file
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops.tex
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops.tex?rev=1506103&r1=1506102&r2=1506103&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops.tex (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops.tex Tue Jul 23 15:09:20 2013
@@ -2,6 +2,8 @@
% normal
\newcommand{\varDUCC}{\text{DUCC}}
+\newcommand{\varOS}{\text{OS}}
+\newcommand{\varJava}{\text{Java}}
\newcommand{\varLinux}{\text{Linux}}
\newcommand{\varUIMA}{\text{UIMA}}
\newcommand{\varUnstructuredInformationManagementArchitecture}{\text{Unstructured Information Management Architecture}}
@@ -83,6 +85,7 @@
\newcommand{\varSchedulingClass}{\texttt{scheduling\_class}}
\newcommand{\varDuccAdministrators}{\texttt{ducc.administrators}}
+\newcommand{\varDuccProperties}{\texttt{ducc.properties}}
\newcommand{\varPID}{\texttt{PID}}
@@ -92,6 +95,10 @@
\newcommand{\varIScheduler}{\texttt{IScheduler}}
+\newcommand{\varRogue}{\texttt{Rogue}}
+
+\newcommand{\varDuccling}{\texttt{ducc\_ling}}
+
% italics
\newcommand{\varNull}{\textit{null}}
@@ -309,6 +316,17 @@
\end{itemize}
+ \section{Communications}
+
+ \varDUCC~comprises a collection of singleton and distributed daemons that need
+ to coordinate activities. This coordination is accomplished via messaging.
+
+ The system is fault tolerant with respect to lost messages, since
+ publications occur at regular intervals and each message encapsulates
+ the current and/or desired state for the target audience.
+ As such, actions may be be delayed but will be carried out as soon as the
+ next message arrives.
+
\section{Daemons}
\varDUCC~is implemented through a collection of configurable singleton
@@ -828,21 +846,200 @@
There is one \varProcessManager~per \varDUCC~cluster.
- The duties of the \varProcessManager are:
+ The duties of the \varProcessManager~are:
\textit{
\input{part5/c01-PM.tex}
}
+
+ The \varProcessManager~provides essential functionality for operation of the
+ \varDUCC~system.
+
+ The \varProcessManager~both publishes and receive reports.
+ The \varProcessManager~receives \varOrchestrator~publications comprising
+ Jobs, Reservations, and Services.
+ The \varProcessManager~distributes two publications at regular intervals.
+ One is heartbeat information to notify the \varOrchestrator~and
+ \varWebServer~that the \varProcessManager~is alive.
+ The other is compacted \varAgent-destined information regarding
+ processes that need to be started, stopped or modified.
+
+ The main function of the \varProcessManager~is to efficiently manage
+ the distributed \varAgents~each of which manages processes running
+ locally on its own respective \varNodeMachineComputer.
+ The \varProcessManager~interprets the \varOrchestrator~publications and
+ redistributes only the essentials to the collection of distributed
+ \varAgents~who each independently act to bring the
+ state of locally deployed processes into compliance.
\subsection{\varAgent}
- There is one \varAgent per \varNodeMachineComputer per \varDUCC~cluster.
+ There is one \varAgent~per~\varNodeMachineComputer~per \varDUCC~cluster.
+ The \varAgents~collectively provide essential functionality for operation
+ of the \varDUCC~system.
- The duties of the \varAgent are:
+ The duties of the \varAgent~are:
+ \textit{
+ \input{part5/c01-Agent.tex}
+ }
- \begin{description}
- \item deploy, monitor and control one or more processes supporting analytic pipelines on one node;
- \item publish inventory and metrics;
- \end{description}
+ The Logging module facilitates the recording to persistent storage of noteworthy
+ events occurring during the \varAgent~lifetime. Noteworthiness is configurable
+ as various levels, such as \varINFO, \varDEBUG~and \varTRACE.
+
+ \subsubsection{Core}
+
+ \begin{itemize}
+ \item NodeAgent;
+
+ This is the main thread for the \varAgent. It is responsible for starting, configuring
+ and ending the \varDUCC~framework duties on a \varNodeMachineComputer.
+
+ \begin{itemize}
+ \item Determine node name, IP address and JMX URL. Record same to configured state directory,
+ utilized by \varWebServer for ease of use display to users/administrators.
+ \item Determine \varOS~page size.
+ \item Determine \varOS~clock rate.
+ \item Determine number of \varDUCC~shares.
+ \item Manage use of privileged operations via appealing to \varDuccling.
+ \item Take down processes for which \varJob~has been de-allocated or ended.
+ \item Start up processes for which \varJob~has been allocated.
+ \item Handle \varDUCC-reservations.
+ \item Handle low swap space on node events.
+ \item Employ Rogue Process Detector.
+ \end{itemize}
+
+ \item NodeAgentInfo;
+
+ Representation of information compiled by the \varAgent:
+
+ \begin{itemize}
+ \item Agent State: INITIALIZING, READY, STOPPED, UNREACHABLE, FAILED
+ \item Agent Jmx port.
+ \item Node IP address.
+ \item Node Hostname.
+ \item Node CPUs.
+ \item \varOS~Name.
+ \item \varOS~Version.
+ \item \varOS~Architecture.
+ \item \varJava~Vendor.
+ \item \varJava~Version.
+ \end{itemize}
+
+ \item NodeMetricsGenerator;
+ \item ProcessLifecycleController;
+ \item ProcessReaperTask;
+ \item RogueProcessReaper;
+
+ This module is responsible for examining \varDuccProperties~to
+ extract various rogue processes handling configuration
+ information and applying accordingly.
+
+ Relevant configuration information comprises:
+ \begin{itemize}
+ \item ducc.agent.rogue.process.reaper.script;
+ \item ducc.agent.rogue.process.purge.delay;
+ \item ducc.agent.rogue.process.kill;
+ \end{itemize}
+
+ \end{itemize}
+
+ \subsubsection{Deploy}
+
+ \begin{itemize}
+ \item AbstractManagedService;
+ \item ManagedService;
+ \item ServiceAdapter;
+ \item ServiceStateNotificationAdapter;
+ \item ManagedUimaService;
+ \item ProcessEventListener;
+ \item UimaAsServiceConfiguration;
+ \end{itemize}
+
+ \subsubsection{Lifecycle}
+
+ The \varAgent~is responsible for receiving \varProcessManager~publications
+ and appropriately launching and terminating \varDUCC-managed processes.
+ And if enabled and available, to employ \varLinuxControlGroups~to assure
+ that user code does not exceed various limits in accordance with fair
+ sharing of the \varNodeMachineComputer.
+
+ \begin{itemize}
+ \item CGroups Manager;
+ \item Command Executor;
+ \item \varDUCC~Command Executor;
+ \item JVM Args Parser;
+ \item Launcher;
+ \item Managed Process;
+ \item Managed Service;
+ \end{itemize}
+
+ \subsubsection{Exceptions}
+
+ \begin{itemize}
+ \item NodeException;
+ \item NodeInitializationException;
+ \end{itemize}
+
+ \subsubsection{Rogue Process Detector}
+
+ Optionally by configuration, the \varAgents~will detect and terminate
+ \varRogue~processes.
+ \varRogue~processes are unsanctioned processes that exist on the \varAgent-managed
+ \varNodeMachineComputer.
+
+ Normally only known \varOS~processes, \varDUCC~framework and \varDUCC-managed
+ processes are sanctioned.
+ Exceptions to the rule can be configured.
+
+ \subsubsection{Node Inventory}
+
+ Gather and publish Node Inventory at regular intervals, comprising:
+
+ \begin{itemize}
+ \item \varDUCC~assigned process identity;
+ \item node identity;
+ \item process identity;
+ \item process state;
+ \item resource state;
+ \item allocation/de-allocation state;
+ \item process initialization time;
+ \item process run time;
+ \item process memory utilization info;
+ \item process swap utilization info;
+ \item process CPU utilization info;
+ \item process java utilization info;
+ \item process CGroup info;
+ \item process end reason;
+ \item process end code;
+
+ \end{itemize}
+
+ \subsubsection{Node Metrics}
+
+ Gather and publish Node Metrics at regular intervals, comprising:
+
+ \begin{itemize}
+ \item users;
+ \item node identity;
+ \item CPU info;
+ \item memory info;
+ \end{itemize}
+
+ \subsubsection{Collectors}
+
+ Various collectors compile Process and Node metrics:
+
+ \begin{itemize}
+ \item node memory;
+ \item node users;
+ \item node load average;
+ \item node CPU;
+ \item node load average;
+ \item process garbage collection;
+ \item process memory and fault;
+ \item process swap;
+ \item process CPU
+ \end{itemize}
\subsection{\varJobDriver (\varJD)}