You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by ri...@apache.org on 2015/07/31 14:24:01 UTC
svn commit: r1693583 [3/9] - in /incubator/brooklyn/site/v:
0.7.0-incubating/concepts/ 0.7.0-incubating/dev/tips/images/
0.7.0-incubating/ops/ 0.7.0-incubating/ops/catalog/
0.7.0-incubating/ops/locations/ 0.7.0-incubating/ops/persistence/
0.7.0-incubat...
Added: incubator/brooklyn/site/v/0.7.0-incubating/ops/troubleshooting/going-deep-in-java-and-logs.html
URL: http://svn.apache.org/viewvc/incubator/brooklyn/site/v/0.7.0-incubating/ops/troubleshooting/going-deep-in-java-and-logs.html?rev=1693583&view=auto
==============================================================================
--- incubator/brooklyn/site/v/0.7.0-incubating/ops/troubleshooting/going-deep-in-java-and-logs.html (added)
+++ incubator/brooklyn/site/v/0.7.0-incubating/ops/troubleshooting/going-deep-in-java-and-logs.html Fri Jul 31 12:23:59 2015
@@ -0,0 +1,1151 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+
+
+<head>
+
+
+<title>Troubleshooting: Going Deep in Java and Logs - Apache Brooklyn (incubating)</title>
+
+<meta http-equiv="content-type" content="text/html; charset=utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+
+<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
+<link href="/v/0.7.0-incubating/style/deps/octicons/octicons.css" rel="stylesheet">
+<link href="/v/0.7.0-incubating/style/deps/bootstrap-theme.css" rel="stylesheet">
+
+<link rel="stylesheet" href="/v/0.7.0-incubating/style/css/code.css" type="text/css" media="screen" />
+
+<link href="/v/0.7.0-incubating/style/css/website.css" rel="stylesheet">
+
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
+<script src="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/js/bootstrap.min.js"></script>
+<script type="text/javascript" src="/v/0.7.0-incubating/style/deps/jquery.cookie.js"></script>
+
+
+</head>
+
+
+<body>
+
+<nav class="navbar navbar-default navbar-fixed-top" id="header" role="navigation">
+ <div class="container-and-sidebars">
+
+ <div class="container-sidebar-left feather">
+ <a href="http://www.apache.org/">
+ <img src="/v/0.7.0-incubating/style/img/feather.png" alt="[Apache]" width="80" class="flip navbar-feather">
+ </a>
+ </div>
+
+ <div class="container container-between-sidebars top-menu">
+ <div class="container-fluid">
+ <!-- Brand and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a class="navbar-brand" href="/"><img src="/v/0.7.0-incubating/style/img/apache-brooklyn-logo-244px-wide.png" alt="brooklyn"></a>
+ </div>
+
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+ <ul class="nav navbar-nav navbar-right">
+
+
+ <li class="dropdown">
+ <a href="/learnmore/index.html">learn more</a>
+ <ul class="dropdown-menu" role="menu">
+ <li>
+ <a href="/learnmore/index.html">Learn More</a>
+ </li>
+ <li class="divider"></li>
+
+ <li>
+
+ <a href="/learnmore/blueprint-tour.html">Blueprint Tour
+ </a>
+
+ </li>
+
+ <li>
+
+ <a href="/learnmore/features/index.html">Features
+ </a>
+
+ </li>
+
+ <li>
+
+ <a href="/learnmore/theory.html">Theory
+ </a>
+
+ </li>
+
+ <li>
+
+ <a href="/learnmore/catalog/index.html">Browse Catalog
+ </a>
+
+ </li>
+
+ </ul>
+ </li>
+
+
+
+ <li class="">
+ <a href="/download/index.html">download</a>
+ </li>
+
+
+
+ <li class="">
+ <a href="/v/0.7.0-incubating/start/running.html">get started</a>
+ </li>
+
+
+
+ <li class="dropdown active">
+ <a href="/documentation/index.html">documentation</a>
+ <ul class="dropdown-menu" role="menu">
+ <li>
+ <a href="/documentation/index.html">Documentation</a>
+ </li>
+ <li class="divider"></li>
+
+ <li>
+
+ <a href="/v/0.7.0-incubating/index.html">User Guide
+ </a>
+
+ <div class="dropdown_section_header"><hr></div>
+
+ </li>
+
+ <li>
+
+ <a href="/v/0.7.0-incubating/yaml/creating-yaml.html">YAML Blueprints
+ </a>
+
+ </li>
+
+ <li>
+
+ <a href="/v/0.7.0-incubating/java/index.html">Java Blueprints
+ </a>
+
+ </li>
+
+ <li>
+
+ <a href="/v/0.7.0-incubating/ops/index.html">Operations
+ </a>
+
+ <div class="dropdown_section_header"><hr></div>
+
+ </li>
+
+ <li>
+
+ <a href="/v/0.7.0-incubating/dev/index.html">Developer Guide
+ </a>
+
+ </li>
+
+ <li>
+
+ <div class="dropdown_new_section"><hr></div>
+
+ <a href="/documentation/other-docs.html">Other Resources
+ </a>
+
+ </li>
+
+ </ul>
+ </li>
+
+
+
+ <li class="dropdown">
+ <a href="/community/index.html">community</a>
+ <ul class="dropdown-menu" role="menu">
+ <li>
+ <a href="/community/index.html">Community</a>
+ </li>
+ <li class="divider"></li>
+
+ <li>
+
+ <a href="/community/mailing-lists.html">Mailing Lists
+ </a>
+
+ </li>
+
+ <li>
+
+ <a href="/community/irc.html">IRC
+ </a>
+
+ </li>
+
+ <li>
+
+ <a href="https://issues.apache.org/jira/browse/BROOKLYN">Bug Tracker (JIRA)
+ <span class="octicon octicon-link-external"></span></a>
+
+ </li>
+
+ <li>
+
+ <a href="/community/how-to-contribute-docs.html">How to Contribute Documentation
+ </a>
+
+ </li>
+
+ </ul>
+ </li>
+
+
+
+ <li class="dropdown">
+ <a href="/developers/index.html">developers</a>
+ <ul class="dropdown-menu" role="menu">
+ <li>
+ <a href="/developers/index.html">Developers</a>
+ </li>
+ <li class="divider"></li>
+
+ <li>
+
+ <a href="/developers/how-to-contribute.html">How to Contribute
+ </a>
+
+ </li>
+
+ <li>
+
+ <a href="/v/0.7.0-incubating/dev/index.html">Developer Guide
+ </a>
+
+ </li>
+
+ <li>
+
+ <a href="/developers/committers/index.html">Committer Guide
+ </a>
+
+ </li>
+
+ <li>
+
+ <a href="/developers/code-standards.html">Code Standards
+ </a>
+
+ </li>
+
+ <li>
+
+ <a href="http://github.com/apache/incubator-brooklyn">GitHub
+ <span class="octicon octicon-link-external"></span></a>
+
+ </li>
+
+ <li>
+
+ <a href="https://issues.apache.org/jira/browse/BROOKLYN">Bug Tracker (JIRA)
+ <span class="octicon octicon-link-external"></span></a>
+
+ </li>
+
+ </ul>
+ </li>
+
+
+ </ul>
+ </div><!-- /.navbar-collapse -->
+ </div><!-- /.container-fluid -->
+ </div><!-- /.container -->
+
+ <div class="container-sidebar-right">
+ <div class="navbar-sidebar-right-icons">
+ <a href="https://github.com/apache/incubator-brooklyn" class="navbar-icon navbar-icon-shift icon-github"
+ data-toggle="tooltip" data-placement="bottom" title="GitHub: apache/incubator-brooklyn"/>
+ <a href="https://twitter.com/#!/search?q=brooklyncentral" class="navbar-icon navbar-icon-shift icon-twitter"
+ data-toggle="tooltip" data-placement="bottom" title="Twitter: @brooklyncentral"/>
+ <a href="http://webchat.freenode.net/?channels=brooklyncentral" class="navbar-icon icon-irc"
+ data-toggle="tooltip" data-placement="bottom" title="IRC: freenode #brooklyncentral"/>
+ <!-- extra a element seems needed as landing page seems to copy the last element here (!?)
+ -->
+ <a href="/" style="width: 0px; height: 0px;"></a>
+ </div>
+ </div>
+
+ </div>
+</nav>
+
+
+<div class="container" id="main_container">
+ <div class="row">
+ <div class="col-md-9" id="content_container">
+ <div id="page_notes"></div>
+ <h1>Troubleshooting: Going Deep in Java and Logs</h1>
+ <p>This guide takes a deep look at the Java and log messages for some failure scenarios,
+giving common steps used to identify the issues.</p>
+
+<h2 id="script-failure">Script Failure</h2>
+
+<p>Many blueprints run bash scripts as part of the installation. This section highlights how to identify a problem with
+a bash script.</p>
+
+<p>First letâs take a look at the <code>customize()</code> method of the Tomcat server blueprint:</p>
+
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="nd">@Override</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="nf">customize</span><span class="o">()</span> <span class="o">{</span>
+ <span class="n">newScript</span><span class="o">(</span><span class="n">CUSTOMIZING</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">body</span><span class="o">.</span><span class="na">append</span><span class="o">(</span><span class="s">"mkdir -p conf logs webapps temp"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">failOnNonZeroResultCode</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">execute</span><span class="o">();</span>
+
+ <span class="n">copyTemplate</span><span class="o">(</span><span class="n">entity</span><span class="o">.</span><span class="na">getConfig</span><span class="o">(</span><span class="n">TomcatServer</span><span class="o">.</span><span class="na">SERVER_XML_RESOURCE</span><span class="o">),</span> <span class="n">Os</span><span class="o">.</span><span class="na">mergePaths</span><span class="o">(</span><span class="n">getRunDir</span><span class="o">(),</span> <span class="s">"conf"</span><span class="o">,</span> <span class="s">"server.xml"</span><span class="o">));</span>
+ <span class="n">copyTemplate</span><span class="o">(</span><span class="n">entity</span><span class="o">.</span><span class="na">getConfig</span><span class="o">(</span><span class="n">TomcatServer</span><span class="o">.</span><span class="na">WEB_XML_RESOURCE</span><span class="o">),</span> <span class="n">Os</span><span class="o">.</span><span class="na">mergePaths</span><span class="o">(</span><span class="n">getRunDir</span><span class="o">(),</span> <span class="s">"conf"</span><span class="o">,</span> <span class="s">"web.xml"</span><span class="o">));</span>
+
+ <span class="k">if</span> <span class="o">(</span><span class="n">isProtocolEnabled</span><span class="o">(</span><span class="s">"HTTPS"</span><span class="o">))</span> <span class="o">{</span>
+ <span class="n">String</span> <span class="n">keystoreUrl</span> <span class="o">=</span> <span class="n">Preconditions</span><span class="o">.</span><span class="na">checkNotNull</span><span class="o">(</span><span class="n">getSslKeystoreUrl</span><span class="o">(),</span> <span class="s">"keystore URL must be specified if using HTTPS for "</span> <span class="o">+</span> <span class="n">entity</span><span class="o">);</span>
+ <span class="n">String</span> <span class="n">destinationSslKeystoreFile</span> <span class="o">=</span> <span class="n">getHttpsSslKeystoreFile</span><span class="o">();</span>
+ <span class="n">InputStream</span> <span class="n">keystoreStream</span> <span class="o">=</span> <span class="n">resource</span><span class="o">.</span><span class="na">getResourceFromUrl</span><span class="o">(</span><span class="n">keystoreUrl</span><span class="o">);</span>
+ <span class="n">getMachine</span><span class="o">().</span><span class="na">copyTo</span><span class="o">(</span><span class="n">keystoreStream</span><span class="o">,</span> <span class="n">destinationSslKeystoreFile</span><span class="o">);</span>
+ <span class="o">}</span>
+
+ <span class="n">getEntity</span><span class="o">().</span><span class="na">deployInitialWars</span><span class="o">();</span>
+ <span class="o">}</span></code></pre></div>
+
+<p>Here we can see that itâs running a script to create four directories before continuing with the customization. Letâs
+introduce an error by changing <code>mkdir</code> to <code>mkrid</code>:</p>
+
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">newScript</span><span class="o">(</span><span class="n">CUSTOMIZING</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">body</span><span class="o">.</span><span class="na">append</span><span class="o">(</span><span class="s">"mkrid -p conf logs webapps temp"</span><span class="o">)</span> <span class="c1">// `mkdir` changed to `mkrid`</span>
+ <span class="o">.</span><span class="na">failOnNonZeroResultCode</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">execute</span><span class="o">();</span></code></pre></div>
+
+<p>Now letâs try deploying this using the following YAML:</p>
+
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">Tomcat failure test</span>
+<span class="l-Scalar-Plain">location</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">localhost</span>
+<span class="l-Scalar-Plain">services</span><span class="p-Indicator">:</span>
+<span class="p-Indicator">-</span> <span class="l-Scalar-Plain">type</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">brooklyn.entity.webapp.tomcat.TomcatServer</span></code></pre></div>
+
+<p>Shortly after deployment, the entity fails with the following error:</p>
+
+<p><code>Failure running task ssh: customizing TomcatServerImpl{id=e1HP2s8x} (HmyPAozV):
+Execution failed, invalid result 127 for customizing TomcatServerImpl{id=e1HP2s8x}</code></p>
+
+<p><a href="images/script-failure-large.png"><img src="images/script-failure.png" alt="Script failure error in the Brooklyn debug console." /></a></p>
+
+<p>By selecting the <code>Activities</code> tab, we can drill into the task that failed. The list of tasks shown (where the
+effectors are shown as top-level tasks) are clickable links. Selecting that row will show the details of
+that particular task, including its sub-tasks. We can eventually get to the specific sub-task that failed:</p>
+
+<p><a href="images/failed-task-large.png"><img src="images/failed-task.png" alt="Task failure error in the Brooklyn debug console." /></a></p>
+
+<p>By clicking on the <code>stderr</code> link, we can see the script failed with the following error:</p>
+
+<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">/tmp/brooklyn-20150721-132251052-l4b9-customizing_TomcatServerImpl_i.sh: line 10: mkrid: command not found</span></code></pre></div>
+
+<p>This tells us <em>what</em> went wrong, but doesnât tell us <em>where</em>. In order to find that, weâll need to look at the
+stack trace that was logged when the exception was thrown.</p>
+
+<p>Itâs always worth looking at the Detailed Status section as sometimes this will give you the information you need.
+In this case, the stack trace is limited to the thread that was used to execute the task that ran the script:</p>
+
+<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">Failed after 40ms</span>
+
+<span class="go">STDERR</span>
+<span class="go">/tmp/brooklyn-20150721-132251052-l4b9-customizing_TomcatServerImpl_i.sh: line 10: mkrid: command not found</span>
+
+
+<span class="go">STDOUT</span>
+<span class="go">Executed /tmp/brooklyn-20150721-132251052-l4b9-customizing_TomcatServerImpl_i.sh, result 127: Execution failed, invalid result 127 for customizing TomcatServerImpl{id=e1HP2s8x}</span>
+
+<span class="go">java.lang.IllegalStateException: Execution failed, invalid result 127 for customizing TomcatServerImpl{id=e1HP2s8x}</span>
+<span class="go"> at brooklyn.entity.basic.lifecycle.ScriptHelper.logWithDetailsAndThrow(ScriptHelper.java:390)</span>
+<span class="go"> at brooklyn.entity.basic.lifecycle.ScriptHelper.executeInternal(ScriptHelper.java:379)</span>
+<span class="go"> at brooklyn.entity.basic.lifecycle.ScriptHelper$8.call(ScriptHelper.java:289)</span>
+<span class="go"> at brooklyn.entity.basic.lifecycle.ScriptHelper$8.call(ScriptHelper.java:287)</span>
+<span class="go"> at brooklyn.util.task.DynamicSequentialTask$DstJob.call(DynamicSequentialTask.java:343)</span>
+<span class="go"> at brooklyn.util.task.BasicExecutionManager$SubmissionCallable.call(BasicExecutionManager.java:469)</span>
+<span class="go"> at java.util.concurrent.FutureTask.run(FutureTask.java:262)</span>
+<span class="go"> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)</span>
+<span class="go"> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)</span>
+<span class="go">at java.lang.Thread.run(Thread.java:745)</span></code></pre></div>
+
+<p>In order to find the exception, weâll need to look in Brooklynâs debug log file. By default, the debug log file
+is named <code>brooklyn.debug.log</code>. Usually the easiest way to navigate the log file is to use <code>less</code>, e.g.
+<code>less brooklyn.debug.log</code>. We can quickly find find the stack trace by first navigating to the end of the log file
+with <code>Shift-G</code>, then performing a reverse-lookup by typing <code>?Tomcat</code> and pressing <code>Enter</code>. If searching for the
+blueprint type (in this case Tomcat) simply matches tasks unrelated to the exception, you can also search for
+the text of the error message, in this case <code>? invalid result 127</code>. You can make the search case-insensitivity by
+typing <code>-i</code> before performing the search. To skip the current match and move to the next on (i.e. âupâ as weâre
+performing a reverse-lookup), simply press <code>n</code></p>
+
+<p>In this case, the <code>?Tomcat</code> search takes us directly to the full stack trace (Only the last part of the trace
+is shown here):</p>
+
+<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">at com.google.common.util.concurrent.ForwardingFuture.get(ForwardingFuture.java:63) ~[guava-17.0.jar:na]</span>
+<span class="go">at brooklyn.util.task.BasicTask.get(BasicTask.java:343) ~[classes/:na]</span>
+<span class="go">at brooklyn.util.task.BasicTask.getUnchecked(BasicTask.java:352) ~[classes/:na]</span>
+<span class="go">... 9 common frames omitted</span>
+<span class="go">Caused by: brooklyn.util.exceptions.PropagatedRuntimeException: </span>
+<span class="go">at brooklyn.util.exceptions.Exceptions.propagate(Exceptions.java:97) ~[classes/:na]</span>
+<span class="go">at brooklyn.util.task.BasicTask.getUnchecked(BasicTask.java:354) ~[classes/:na]</span>
+<span class="go">at brooklyn.entity.basic.lifecycle.ScriptHelper.execute(ScriptHelper.java:339) ~[classes/:na]</span>
+<span class="go">at brooklyn.entity.webapp.tomcat.TomcatSshDriver.customize(TomcatSshDriver.java:72) ~[classes/:na]</span>
+<span class="go">at brooklyn.entity.basic.AbstractSoftwareProcessDriver$8.run(AbstractSoftwareProcessDriver.java:150) ~[classes/:na]</span>
+<span class="go">at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) ~[na:1.7.0_71]</span>
+<span class="go">at brooklyn.util.task.DynamicSequentialTask$DstJob.call(DynamicSequentialTask.java:343) ~[classes/:na]</span>
+<span class="go">... 5 common frames omitted</span>
+<span class="go">Caused by: java.util.concurrent.ExecutionException: java.lang.IllegalStateException: Execution failed, invalid result 127 for customizing TomcatServerImpl{id=e1HP2s8x}</span>
+<span class="go">at java.util.concurrent.FutureTask.report(FutureTask.java:122) [na:1.7.0_71]</span>
+<span class="go">at java.util.concurrent.FutureTask.get(FutureTask.java:188) [na:1.7.0_71]</span>
+<span class="go">at com.google.common.util.concurrent.ForwardingFuture.get(ForwardingFuture.java:63) ~[guava-17.0.jar:na]</span>
+<span class="go">at brooklyn.util.task.BasicTask.get(BasicTask.java:343) ~[classes/:na]</span>
+<span class="go">at brooklyn.util.task.BasicTask.getUnchecked(BasicTask.java:352) ~[classes/:na]</span>
+<span class="go">... 10 common frames omitted</span>
+<span class="go">Caused by: java.lang.IllegalStateException: Execution failed, invalid result 127 for customizing TomcatServerImpl{id=e1HP2s8x}</span>
+<span class="go">at brooklyn.entity.basic.lifecycle.ScriptHelper.logWithDetailsAndThrow(ScriptHelper.java:390) ~[classes/:na]</span>
+<span class="go">at brooklyn.entity.basic.lifecycle.ScriptHelper.executeInternal(ScriptHelper.java:379) ~[classes/:na]</span>
+<span class="go">at brooklyn.entity.basic.lifecycle.ScriptHelper$8.call(ScriptHelper.java:289) ~[classes/:na]</span>
+<span class="go">at brooklyn.entity.basic.lifecycle.ScriptHelper$8.call(ScriptHelper.java:287) ~[classes/:na]</span>
+<span class="go">... 6 common frames omitted</span></code></pre></div>
+
+<p>Brooklynâs use of tasks and helper classes can make the stack trace a little harder than usual to follow, but a good
+place to start is to look through the stack trace for the nodeâs implementation or ssh driver classes (usually
+named <code>FooNodeImpl</code> or <code>FooSshDriver</code>). In this case we can see the following:</p>
+
+<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">at brooklyn.entity.webapp.tomcat.TomcatSshDriver.customize(TomcatSshDriver.java:72) ~[classes/:na]</span></code></pre></div>
+
+<p>Combining this with the error message of <code>mkrid: command not found</code> we can see that indeed <code>mkdir</code> has been
+misspelled <code>mkrid</code> on line 72 of <code>TomcatSshDriver.java</code>.</p>
+
+<h2 id="non-script-failure">Non-Script Failure</h2>
+
+<p>The section above gives an example of a failure that occurs when a script is run. In this section we will look at
+a failure in a non-script related part of the code. Weâll use the <code>customize()</code> method of the Tomcat server again,
+but this time, weâll correct the spelling of âmkdirâ and add a line that attempts to copy a nonexistent resource
+to the remote server:</p>
+
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">newScript</span><span class="o">(</span><span class="n">CUSTOMIZING</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">body</span><span class="o">.</span><span class="na">append</span><span class="o">(</span><span class="s">"mkdir -p conf logs webapps temp"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">failOnNonZeroResultCode</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">execute</span><span class="o">();</span>
+
+<span class="n">copyTemplate</span><span class="o">(</span><span class="n">entity</span><span class="o">.</span><span class="na">getConfig</span><span class="o">(</span><span class="n">TomcatServer</span><span class="o">.</span><span class="na">SERVER_XML_RESOURCE</span><span class="o">),</span> <span class="n">Os</span><span class="o">.</span><span class="na">mergePaths</span><span class="o">(</span><span class="n">getRunDir</span><span class="o">(),</span> <span class="s">"conf"</span><span class="o">,</span> <span class="s">"server.xml"</span><span class="o">));</span>
+<span class="n">copyTemplate</span><span class="o">(</span><span class="n">entity</span><span class="o">.</span><span class="na">getConfig</span><span class="o">(</span><span class="n">TomcatServer</span><span class="o">.</span><span class="na">WEB_XML_RESOURCE</span><span class="o">),</span> <span class="n">Os</span><span class="o">.</span><span class="na">mergePaths</span><span class="o">(</span><span class="n">getRunDir</span><span class="o">(),</span> <span class="s">"conf"</span><span class="o">,</span> <span class="s">"web.xml"</span><span class="o">));</span>
+<span class="n">copyTemplate</span><span class="o">(</span><span class="s">"classpath://nonexistent.xml"</span><span class="o">,</span> <span class="n">Os</span><span class="o">.</span><span class="na">mergePaths</span><span class="o">(</span><span class="n">getRunDir</span><span class="o">(),</span> <span class="s">"conf"</span><span class="o">,</span> <span class="s">"nonexistent.xml"</span><span class="o">));</span> <span class="c1">// Resource does not exist!</span></code></pre></div>
+
+<p>Letâs deploy this using the same YAML from above. Hereâs the resulting error in the Brooklyn debug console:</p>
+
+<p><a href="images/resource-exception-large.png"><img src="images/resource-exception.png" alt="Resource exception in the Brooklyn debug console." /></a></p>
+
+<p>Again, this tells us <em>what</em> the error is, but we need to find <em>where</em> the code is that attempts to copy this file. In
+this case itâs shown in the Detailed Status section, and we donât need to go to the log file:</p>
+
+<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">Failed after 221ms: Error getting resource 'classpath://nonexistent.xml' for TomcatServerImpl{id=PVZxDKU1}: java.io.IOException: Error accessing classpath://nonexistent.xml: java.io.IOException: nonexistent.xml not found on classpath</span>
+
+<span class="go">java.lang.RuntimeException: Error getting resource 'classpath://nonexistent.xml' for TomcatServerImpl{id=PVZxDKU1}: java.io.IOException: Error accessing classpath://nonexistent.xml: java.io.IOException: nonexistent.xml not found on classpath</span>
+<span class="go"> at brooklyn.util.ResourceUtils.getResourceFromUrl(ResourceUtils.java:297)</span>
+<span class="go"> at brooklyn.util.ResourceUtils.getResourceAsString(ResourceUtils.java:475)</span>
+<span class="go"> at brooklyn.entity.basic.AbstractSoftwareProcessDriver.getResourceAsString(AbstractSoftwareProcessDriver.java:447)</span>
+<span class="go"> at brooklyn.entity.basic.AbstractSoftwareProcessDriver.processTemplate(AbstractSoftwareProcessDriver.java:469)</span>
+<span class="go"> at brooklyn.entity.basic.AbstractSoftwareProcessDriver.copyTemplate(AbstractSoftwareProcessDriver.java:390)</span>
+<span class="go"> at brooklyn.entity.basic.AbstractSoftwareProcessDriver.copyTemplate(AbstractSoftwareProcessDriver.java:379)</span>
+<span class="go"> at brooklyn.entity.webapp.tomcat.TomcatSshDriver.customize(TomcatSshDriver.java:79)</span>
+<span class="go"> at brooklyn.entity.basic.AbstractSoftwareProcessDriver$8.run(AbstractSoftwareProcessDriver.java:150)</span>
+<span class="go"> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)</span>
+<span class="go"> at brooklyn.util.task.DynamicSequentialTask$DstJob.call(DynamicSequentialTask.java:343)</span>
+<span class="go"> at brooklyn.util.task.BasicExecutionManager$SubmissionCallable.call(BasicExecutionManager.java:469)</span>
+<span class="go"> at java.util.concurrent.FutureTask.run(FutureTask.java:262)</span>
+<span class="go"> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)</span>
+<span class="go"> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)</span>
+<span class="go">at java.lang.Thread.run(Thread.java:745)</span>
+<span class="go"> Caused by: java.io.IOException: Error accessing classpath://nonexistent.xml: java.io.IOException: nonexistent.xml not found on classpath</span>
+<span class="go">at brooklyn.util.ResourceUtils.getResourceFromUrl(ResourceUtils.java:233)</span>
+<span class="go"> ... 14 more</span>
+<span class="go"> Caused by: java.io.IOException: nonexistent.xml not found on classpath</span>
+<span class="go"> at brooklyn.util.ResourceUtils.getResourceViaClasspath(ResourceUtils.java:372)</span>
+<span class="go">at brooklyn.util.ResourceUtils.getResourceFromUrl(ResourceUtils.java:230)</span>
+<span class="go"> ... 14 more</span></code></pre></div>
+
+<p>Looking for <code>Tomcat</code> in the stack trace, we can see in this case the problem lies at line 79 of <code>TomcatSshDriver.java</code></p>
+
+<h2 id="external-failure">External Failure</h2>
+
+<p>Sometimes an entity will fail outside the direct commands issues by Brooklyn. When installing and launching an entity,
+Brooklyn will check the return code of scripts that were run to ensure that they completed successfully (i.e. the
+return code of the script is zero). It is possible, for example, that a launch script completes successfully, but
+the entity fails to start.</p>
+
+<p>We can simulate this type of failure by launching Tomcat with an invalid configuration file. As seen in the previous
+examples, Brooklyn copies two xml configuration files to the server: <code>server.xml</code> and <code>web.xml</code></p>
+
+<p>The first few non-comment lines of <code>server.xml</code> are as follows (you can see the full file [here]
+(https://github.com/apache/incubator-brooklyn/blob/master/software/webapp/src/main/resources/brooklyn/entity/webapp/tomcat/server.xml)):</p>
+
+<div class="highlight"><pre><code class="language-xml" data-lang="xml"><span class="nt"><Server</span> <span class="na">port=</span><span class="s">"${driver.shutdownPort?c}"</span> <span class="na">shutdown=</span><span class="s">"SHUTDOWN"</span><span class="nt">></span>
+ <span class="nt"><Listener</span> <span class="na">className=</span><span class="s">"org.apache.catalina.core.AprLifecycleListener"</span> <span class="na">SSLEngine=</span><span class="s">"on"</span> <span class="nt">/></span>
+ <span class="nt"><Listener</span> <span class="na">className=</span><span class="s">"org.apache.catalina.core.JasperListener"</span> <span class="nt">/></span></code></pre></div>
+
+<p>Letâs add an unmatched XML element, which will make this XML file invalid:</p>
+
+<div class="highlight"><pre><code class="language-xml" data-lang="xml"><span class="nt"><Server</span> <span class="na">port=</span><span class="s">"${driver.shutdownPort?c}"</span> <span class="na">shutdown=</span><span class="s">"SHUTDOWN"</span><span class="nt">></span>
+ <span class="nt"><unmatched-element></span> <span class="c"><!-- This is invalid XML as we won't add </unmatched-element> --></span>
+ <span class="nt"><Listener</span> <span class="na">className=</span><span class="s">"org.apache.catalina.core.AprLifecycleListener"</span> <span class="na">SSLEngine=</span><span class="s">"on"</span> <span class="nt">/></span>
+ <span class="nt"><Listener</span> <span class="na">className=</span><span class="s">"org.apache.catalina.core.JasperListener"</span> <span class="nt">/></span></code></pre></div>
+
+<p>As Brooklyn doesnât know how these types of resources are used, theyâre not validated as theyâre copied to the remote machine.
+As far as Brooklyn is concerned, the file will have copied successfully.</p>
+
+<p>Letâs deploy Tomcat again, using the same YAML as before. This time, the deployment runs for a few minutes before failing
+with <code>Timeout waiting for SERVICE_UP</code>:</p>
+
+<p><a href="images/external-error-large.png"><img src="images/external-error.png" alt="External error in the Brooklyn debug console." /></a></p>
+
+<p>If we drill down into the tasks in the <code>Activities</code> tab, we can see that all of the installation and launch tasks
+completed successfully, and stdout of the <code>launch</code> script is as follows:</p>
+
+<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">Executed /tmp/brooklyn-20150721-153049139-fK2U-launching_TomcatServerImpl_id_.sh, result 0</span></code></pre></div>
+
+<p>The task that failed was the <code>post-start</code> task, and the stack trace from the Detailed Status section is as follows:</p>
+
+<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">Failed after 5m 1s: Timeout waiting for SERVICE_UP from TomcatServerImpl{id=BUHgQeOs}</span>
+
+<span class="go">java.lang.IllegalStateException: Timeout waiting for SERVICE_UP from TomcatServerImpl{id=BUHgQeOs}</span>
+<span class="go"> at brooklyn.entity.basic.Entities.waitForServiceUp(Entities.java:1073)</span>
+<span class="go"> at brooklyn.entity.basic.SoftwareProcessImpl.waitForServiceUp(SoftwareProcessImpl.java:388)</span>
+<span class="go"> at brooklyn.entity.basic.SoftwareProcessImpl.waitForServiceUp(SoftwareProcessImpl.java:385)</span>
+<span class="go"> at brooklyn.entity.basic.SoftwareProcessDriverLifecycleEffectorTasks.postStartCustom(SoftwareProcessDriverLifecycleEffectorTasks.java:164)</span>
+<span class="go"> at brooklyn.entity.software.MachineLifecycleEffectorTasks$7.run(MachineLifecycleEffectorTasks.java:433)</span>
+<span class="go"> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)</span>
+<span class="go"> at brooklyn.util.task.DynamicSequentialTask$DstJob.call(DynamicSequentialTask.java:343)</span>
+<span class="go"> at brooklyn.util.task.BasicExecutionManager$SubmissionCallable.call(BasicExecutionManager.java:469)</span>
+<span class="go"> at java.util.concurrent.FutureTask.run(FutureTask.java:262)</span>
+<span class="go"> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)</span>
+<span class="go"> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)</span>
+<span class="go">at java.lang.Thread.run(Thread.java:745)</span></code></pre></div>
+
+<p>This doesnât really tell us what we need to know, and looking in the <code>brooklyn.debug.log</code> file yields no further
+clues. The key here is the error message <code>Timeout waiting for SERVICE_UP</code>. After running the installation and
+launch scripts, assuming all scripts completed successfully, Brooklyn will periodically check the health of the node
+and will set the node on fire if the health check does not pass within a pre-prescribed period (the default is
+two minutes, and can be configured using the <code>start.timeout</code> config key). The periodic health check also continues
+after the successful launch in order to check continued operation of the node, but in this case it fails to pass
+at all.</p>
+
+<p>The first thing we need to do is to find out how Brooklyn determines the health of the node. The health-check is
+often implemented in the <code>isRunning()</code> method in the entityâs ssh driver. Tomcatâs implementation of <code>isRunning()</code>
+is as follows:</p>
+
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="nd">@Override</span>
+<span class="kd">public</span> <span class="kt">boolean</span> <span class="nf">isRunning</span><span class="o">()</span> <span class="o">{</span>
+ <span class="k">return</span> <span class="nf">newScript</span><span class="o">(</span><span class="n">MutableMap</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="n">USE_PID_FILE</span><span class="o">,</span> <span class="s">"pid.txt"</span><span class="o">),</span> <span class="n">CHECK_RUNNING</span><span class="o">).</span><span class="na">execute</span><span class="o">()</span> <span class="o">==</span> <span class="mi">0</span><span class="o">;</span>
+<span class="o">}</span></code></pre></div>
+
+<p>The <code>newScript</code> method has conveniences for default scripts to check if a process is running based on its PID. In this
+case, it will look for Tomcatâs PID in the <code>pid.txt</code> file and check if the PID is the PID of a running process</p>
+
+<p>Itâs worth a quick sanity check at this point to check if the PID file exists, and if the process is running.
+By default, the pid file is located in the run directory of the entity. You can find the location of the entityâs run
+directory by looking at the <code>run.dir</code> sensor. In this case it is <code>/tmp/brooklyn-martin/apps/jIzIHXtP/entities/TomcatServer_BUHgQeOs</code>.
+To find the pid, you simply cat the pid.txt file in this directory:</p>
+
+<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="gp">$</span> cat /tmp/brooklyn-martin/apps/jIzIHXtP/entities/TomcatServer_BUHgQeOs/pid.txt
+<span class="go">73714</span></code></pre></div>
+
+<p>In this case, the PID in the file is 73714. You can then check if the process is running using <code>ps</code>. You can also
+pipe the output to <code>fold</code> so the full launch command is visible:</p>
+
+<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="gp">$</span> ps -p <span class="m">73714</span> <span class="p">|</span> fold -w 120
+<span class="go">PID TTY TIME CMD</span>
+<span class="go">73714 ?? 0:08.03 /Library/Java/JavaVirtualMachines/jdk1.8.0_51.jdk/Contents/Home/bin/java -Dnop -Djava.util.logg</span>
+<span class="go">ing.manager=org.apache.juli.ClassLoaderLogManager -javaagent:/tmp/brooklyn-martin/apps/jIzIHXtP/entities/TomcatServer_BU</span>
+<span class="go">HgQeOs/brooklyn-jmxmp-agent-shaded-0.8.0-SNAPSHOT.jar -Xms200m -Xmx800m -XX:MaxPermSize=400m -Dcom.sun.management.jmxrem</span>
+<span class="go">ote -Dbrooklyn.jmxmp.rmi-port=1099 -Dbrooklyn.jmxmp.port=31001 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.manage</span>
+<span class="go">ment.jmxremote.authenticate=false -Djava.endorsed.dirs=/tmp/brooklyn-martin/installs/TomcatServer_7.0.56/apache-tomcat-7</span>
+<span class="go">.0.56/endorsed -classpath /tmp/brooklyn-martin/installs/TomcatServer_7.0.56/apache-tomcat-7.0.56/bin/bootstrap.jar:/tmp/</span>
+<span class="go">brooklyn-martin/installs/TomcatServer_7.0.56/apache-tomcat-7.0.56/bin/tomcat-juli.jar -Dcatalina.base=/tmp/brooklyn-mart</span>
+<span class="go">in/apps/jIzIHXtP/entities/TomcatServer_BUHgQeOs -Dcatalina.home=/tmp/brooklyn-martin/installs/TomcatServer_7.0.56/apache</span>
+<span class="go">-tomcat-7.0.56 -Djava.io.tmpdir=/tmp/brooklyn-martin/apps/jIzIHXtP/entities/TomcatServer_BUHgQeOs/temp org.apache.catali</span>
+<span class="go">na.startup.Bootstrap start</span></code></pre></div>
+
+<p>This confirms that the process is running. The next thing we can look at is the <code>service.notUp.indicators</code> sensor. This
+reads as follows:</p>
+
+<div class="highlight"><pre><code class="language-json" data-lang="json"><span class="p">{</span><span class="nt">"service.process.isRunning"</span><span class="p">:</span><span class="s2">"The software process for this entity does not appear to be running"</span><span class="p">}</span></code></pre></div>
+
+<p>This confirms that the problem is indeed due to the <code>service.process.isRunning</code> sensor. We assumed earlier that this was
+set by the <code>isRunning()</code> method in <code>TomcatSshDriver.java</code>, but this isnât always the case. The <code>service.process.isRunning</code>
+sensor is wired up by the <code>connectSensors()</code> method in the nodeâs implementation class, in this case
+<code>TomcatServerImpl.java</code>. Tomcatâs implementation of <code>connectSensors()</code> is as follows:</p>
+
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="nd">@Override</span>
+<span class="kd">public</span> <span class="kt">void</span> <span class="nf">connectSensors</span><span class="o">()</span> <span class="o">{</span>
+ <span class="kd">super</span><span class="o">.</span><span class="na">connectSensors</span><span class="o">();</span>
+
+ <span class="k">if</span> <span class="o">(</span><span class="n">getDriver</span><span class="o">().</span><span class="na">isJmxEnabled</span><span class="o">())</span> <span class="o">{</span>
+ <span class="n">String</span> <span class="n">requestProcessorMbeanName</span> <span class="o">=</span> <span class="s">"Catalina:type=GlobalRequestProcessor,name=\"http-*\""</span><span class="o">;</span>
+
+ <span class="n">Integer</span> <span class="n">port</span> <span class="o">=</span> <span class="n">isHttpsEnabled</span><span class="o">()</span> <span class="o">?</span> <span class="n">getAttribute</span><span class="o">(</span><span class="n">HTTPS_PORT</span><span class="o">)</span> <span class="o">:</span> <span class="n">getAttribute</span><span class="o">(</span><span class="n">HTTP_PORT</span><span class="o">);</span>
+ <span class="n">String</span> <span class="n">connectorMbeanName</span> <span class="o">=</span> <span class="n">format</span><span class="o">(</span><span class="s">"Catalina:type=Connector,port=%s"</span><span class="o">,</span> <span class="n">port</span><span class="o">);</span>
+
+ <span class="n">jmxWebFeed</span> <span class="o">=</span> <span class="n">JmxFeed</span><span class="o">.</span><span class="na">builder</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">entity</span><span class="o">(</span><span class="k">this</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">period</span><span class="o">(</span><span class="mi">3000</span><span class="o">,</span> <span class="n">TimeUnit</span><span class="o">.</span><span class="na">MILLISECONDS</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">pollAttribute</span><span class="o">(</span><span class="k">new</span> <span class="n">JmxAttributePollConfig</span><span class="o"><</span><span class="n">Integer</span><span class="o">>(</span><span class="n">ERROR_COUNT</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">objectName</span><span class="o">(</span><span class="n">requestProcessorMbeanName</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">attributeName</span><span class="o">(</span><span class="s">"errorCount"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">pollAttribute</span><span class="o">(</span><span class="k">new</span> <span class="n">JmxAttributePollConfig</span><span class="o"><</span><span class="n">Integer</span><span class="o">>(</span><span class="n">REQUEST_COUNT</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">objectName</span><span class="o">(</span><span class="n">requestProcessorMbeanName</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">attributeName</span><span class="o">(</span><span class="s">"requestCount"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">pollAttribute</span><span class="o">(</span><span class="k">new</span> <span class="n">JmxAttributePollConfig</span><span class="o"><</span><span class="n">Integer</span><span class="o">>(</span><span class="n">TOTAL_PROCESSING_TIME</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">objectName</span><span class="o">(</span><span class="n">requestProcessorMbeanName</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">attributeName</span><span class="o">(</span><span class="s">"processingTime"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">pollAttribute</span><span class="o">(</span><span class="k">new</span> <span class="n">JmxAttributePollConfig</span><span class="o"><</span><span class="n">String</span><span class="o">>(</span><span class="n">CONNECTOR_STATUS</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">objectName</span><span class="o">(</span><span class="n">connectorMbeanName</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">attributeName</span><span class="o">(</span><span class="s">"stateName"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">pollAttribute</span><span class="o">(</span><span class="k">new</span> <span class="n">JmxAttributePollConfig</span><span class="o"><</span><span class="n">Boolean</span><span class="o">>(</span><span class="n">SERVICE_PROCESS_IS_RUNNING</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">objectName</span><span class="o">(</span><span class="n">connectorMbeanName</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">attributeName</span><span class="o">(</span><span class="s">"stateName"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">onSuccess</span><span class="o">(</span><span class="n">Functions</span><span class="o">.</span><span class="na">forPredicate</span><span class="o">(</span><span class="n">Predicates</span><span class="o">.<</span><span class="n">Object</span><span class="o">></span><span class="n">equalTo</span><span class="o">(</span><span class="s">"STARTED"</span><span class="o">)))</span>
+ <span class="o">.</span><span class="na">setOnFailureOrException</span><span class="o">(</span><span class="kc">false</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">build</span><span class="o">();</span>
+
+ <span class="n">jmxAppFeed</span> <span class="o">=</span> <span class="n">JavaAppUtils</span><span class="o">.</span><span class="na">connectMXBeanSensors</span><span class="o">(</span><span class="k">this</span><span class="o">);</span>
+ <span class="o">}</span> <span class="k">else</span> <span class="o">{</span>
+ <span class="c1">// if not using JMX</span>
+ <span class="n">LOG</span><span class="o">.</span><span class="na">warn</span><span class="o">(</span><span class="s">"Tomcat running without JMX monitoring; limited visibility of service available"</span><span class="o">);</span>
+ <span class="n">connectServiceUpIsRunning</span><span class="o">();</span>
+ <span class="o">}</span>
+<span class="o">}</span></code></pre></div>
+
+<p>We can see here that if jmx is not enabled, the method will call <code>connectServiceUpIsRunning()</code> which will use the
+default PID-based method of determining if a process is running. However, as JMX <em>is</em> running, the <code>service.process.isRunning</code>
+sensor (denoted here by the <code>SERVICE_PROCESS_IS_RUNNING</code> variable) is set to true if and only if the
+<code>stateName</code> JMX attribute equals <code>STARTED</code>. We can see from the previous call to <code>.pollAttribute</code> that this
+attribute is also published to the <code>CONNECTOR_STATUS</code> sensor. The <code>CONNECTOR_STATUS</code> sensor is defined as follows:</p>
+
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">AttributeSensor</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">CONNECTOR_STATUS</span> <span class="o">=</span>
+ <span class="k">new</span> <span class="n">BasicAttributeSensor</span><span class="o"><</span><span class="n">String</span><span class="o">>(</span><span class="n">String</span><span class="o">.</span><span class="na">class</span><span class="o">,</span> <span class="s">"webapp.tomcat.connectorStatus"</span><span class="o">,</span> <span class="s">"Catalina connector state name"</span><span class="o">);</span></code></pre></div>
+
+<p>Letâs go back to the Brooklyn debug console and look for the <code>webapp.tomcat.connectorStatus</code>:</p>
+
+<p><a href="images/jmx-sensors-large.png"><img src="images/jmx-sensors.png" alt="Sensors view in the Brooklyn debug console." /></a></p>
+
+<p>As the sensor is not shown, itâs likely that itâs simply null or not set. We can check this by clicking
+the <code>Show/hide empty records</code> icon (highlighted in yellow above):</p>
+
+<p><a href="images/jmx-sensors-all-large.png"><img src="images/jmx-sensors-all.png" alt="All sensors view in the Brooklyn debug console." /></a></p>
+
+<p>We know from previous steps that the installation and launch scripts completed, and we know the procecess is running,
+but we can see here that the server is not responding to JMX requests. A good thing to check here would be that the
+JMX port is not being blocked by iptables, firewalls or security groups
+(see the (troubleshooting connectivity guide)[troubleshooting-connectivity.html]).
+Letâs assume that weâve checked that and theyâre all open. There is still one more thing that Brooklyn can tell us.</p>
+
+<p>Still on the <code>Sensors</code> tab, letâs take a look at the <code>log.location</code> sensor:</p>
+
+<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">/tmp/brooklyn-martin/apps/c3bmrlC3/entities/TomcatServer_C1TAjYia/logs/catalina.out</span></code></pre></div>
+
+<p>This is the location of Tomcatâs own log file. The location of the log file will differ from process to process
+and when writing a custom entity you will need to check the softwareâs own documentation. If your blueprintâs
+ssh driver extends <code>JavaSoftwareProcessSshDriver</code>, the value returned by the <code>getLogFileLocation()</code> method will
+automatically be published to the <code>log.location</code> sensor. Otherwise, you can publish the value yourself by calling
+<code>entity.setAttribute(Attributes.LOG_FILE_LOCATION, getLogFileLocation());</code> in your ssh driver</p>
+
+<p><strong>Note:</strong> The log file will be on the server to which you have deployed Tomcat, and not on the Brooklyn server.
+Letâs take a look in the log file:</p>
+
+<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">less /tmp/brooklyn-martin/apps/c3bmrlC3/entities/TomcatServer_C1TAjYia/logs/catalina.out</span>
+
+<span class="go">Jul 21, 2015 4:12:12 PM org.apache.tomcat.util.digester.Digester fatalError</span>
+<span class="go">SEVERE: Parse Fatal Error at line 143 column 3: The element type "unmatched-element" must be terminated by the matching end-tag "</unmatched-element>".</span>
+<span class="go"> org.xml.sax.SAXParseException; systemId: file:/tmp/brooklyn-martin/apps/c3bmrlC3/entities/TomcatServer_C1TAjYia/conf/server.xml; lineNumber: 143; columnNumber: 3; The element type "unmatched-element" must be terminated by the matching end-tag "</unmatched-element>".</span>
+<span class="go"> at com.sun.org.apache.xerces.internal.util.ErrorHandlerWrapper.createSAXParseException(ErrorHandlerWrapper.java:203)</span>
+<span class="go"> at com.sun.org.apache.xerces.internal.util.ErrorHandlerWrapper.fatalError(ErrorHandlerWrapper.java:177)</span>
+<span class="go"> at com.sun.org.apache.xerces.internal.impl.XMLErrorReporter.reportError(XMLErrorReporter.java:441)</span>
+<span class="go"> at com.sun.org.apache.xerces.internal.impl.XMLErrorReporter.reportError(XMLErrorReporter.java:368)</span>
+<span class="go"> at com.sun.org.apache.xerces.internal.impl.XMLScanner.reportFatalError(XMLScanner.java:1437)</span>
+<span class="go"> at com.sun.org.apache.xerces.internal.impl.XMLDocumentFragmentScannerImpl.scanEndElement(XMLDocumentFragmentScannerImpl.java:1749)</span>
+<span class="go"> at com.sun.org.apache.xerces.internal.impl.XMLDocumentFragmentScannerImpl$FragmentContentDriver.next(XMLDocumentFragmentScannerImpl.java:2973)</span>
+<span class="go"> at com.sun.org.apache.xerces.internal.impl.XMLDocumentScannerImpl.next(XMLDocumentScannerImpl.java:606)</span>
+<span class="go"> at com.sun.org.apache.xerces.internal.impl.XMLDocumentFragmentScannerImpl.scanDocument(XMLDocumentFragmentScannerImpl.java:510)</span>
+<span class="go"> at com.sun.org.apache.xerces.internal.parsers.XML11Configuration.parse(XML11Configuration.java:848)</span>
+<span class="go"> at com.sun.org.apache.xerces.internal.parsers.XML11Configuration.parse(XML11Configuration.java:777)</span>
+<span class="go"> at com.sun.org.apache.xerces.internal.parsers.XMLParser.parse(XMLParser.java:141)</span>
+<span class="go"> at com.sun.org.apache.xerces.internal.parsers.AbstractSAXParser.parse(AbstractSAXParser.java:1213)</span>
+<span class="go"> at com.sun.org.apache.xerces.internal.jaxp.SAXParserImpl$JAXPSAXParser.parse(SAXParserImpl.java:649)</span>
+<span class="go"> at org.apache.tomcat.util.digester.Digester.parse(Digester.java:1561)</span>
+<span class="go"> at org.apache.catalina.startup.Catalina.load(Catalina.java:615)</span>
+<span class="go"> at org.apache.catalina.startup.Catalina.start(Catalina.java:677)</span>
+<span class="go"> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)</span>
+<span class="go"> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)</span>
+<span class="go"> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)</span>
+<span class="go"> at java.lang.reflect.Method.invoke(Method.java:497)</span>
+<span class="go"> at org.apache.catalina.startup.Bootstrap.start(Bootstrap.java:321)</span>
+<span class="go">at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:455)</span>
+
+<span class="go"> Jul 21, 2015 4:12:12 PM org.apache.catalina.startup.Catalina load</span>
+<span class="go"> WARNING: Catalina.start using conf/server.xml: The element type "unmatched-element" must be terminated by the matching end-tag "</unmatched-element>".</span>
+<span class="go"> Jul 21, 2015 4:12:12 PM org.apache.catalina.startup.Catalina start</span>
+<span class="go"> SEVERE: Cannot start server. Server instance is not configured.</span></code></pre></div>
+
+<p>As expected, we can see here that the <code>unmatched-element</code> element has not been terminated in the <code>server.xml</code> file</p>
+
+ </div>
+
+ <div class="col-md-3">
+ <div class="list-group side-menu" id="side-menu">
+
+
+
+
+
+
+ <h4 class=" with_following">
+ <a href="/v/0.7.0-incubating/index.html" class="list-group-item breadcrumb breadcrumb0">
+ User Guide
+ </a></h4>
+
+ <h4 class=" with_preceding with_following">
+ <a href="/v/0.7.0-incubating/ops/index.html" class="list-group-item breadcrumb breadcrumb1">
+ Operations
+ </a></h4>
+
+ <h4 class=" with_preceding">
+ <a href="/v/0.7.0-incubating/ops/troubleshooting/index.html" class="list-group-item breadcrumb breadcrumb2">
+ Troubleshooting
+ </a></h4>
+
+
+
+
+
+ <a href="/v/0.7.0-incubating/ops/troubleshooting/overview.html" class="list-group-item">Overview
+ </a>
+
+
+
+
+
+ <a href="/v/0.7.0-incubating/ops/troubleshooting/deployment.html" class="list-group-item">Deployment
+ </a>
+
+
+
+
+
+ <a href="/v/0.7.0-incubating/ops/troubleshooting/connectivity.html" class="list-group-item">Server Connectivity
+ </a>
+
+
+
+
+
+ <a href="/v/0.7.0-incubating/ops/troubleshooting/softwareprocess.html" class="list-group-item">SoftwareProcess Entities
+ </a>
+
+
+
+
+
+ <a href="/v/0.7.0-incubating/ops/troubleshooting/going-deep-in-java-and-logs.html" class="list-group-item active">Going Deep in Java and Logs
+ </a>
+
+
+
+
+
+
+
+</div>
+<div id="width_reference"></div>
+
+
+<script language="JavaScript" type="application/javascript">
+
+
+sidemenu_x_sizer=function(){ $('#side-menu').width($('#side-menu').parent().find('#width_reference').outerWidth()); };
+$(sidemenu_x_sizer);
+$(window).resize(sidemenu_x_sizer);
+
+
+sidemenu_y_nonfloater=function(){
+ if ($('#side-menu').outerHeight(true) + $('#header').outerHeight(true) + $('#footer').outerHeight(true) > window.innerHeight ||
+ $('#side-menu').width() >= $('#content_container').width()/2) {
+ $('#side-menu').css('position', 'inherit');
+ } else {
+ // restore if screen has grown
+ $('#side-menu').css('position', 'fixed');
+ }
+};
+$(sidemenu_y_nonfloater);
+$(window).resize(sidemenu_y_nonfloater);
+
+
+
+var sideMenu = $("#side-menu"),
+ sideItems = sideMenu.find("a"),
+ // Anchors corresponding to menu items
+ scrollItems = sideItems.map(function(){
+ var item = $(this).attr("section-target");
+ if (item && item.length) { return item; }
+ });
+
+var highlight_section_last_top = -1;
+var highlight_section_completed = false;
+
+var highlight_section = function() {
+ // Get container scroll position
+ var highlight_section_new_top = $(this).scrollTop();
+ if (highlight_section_new_top == highlight_section_last_top) return;
+ var highlight_section_new_bottom = highlight_section_new_top + $(window).height();
+ var scroll_advancing = (highlight_section_new_top > highlight_section_last_top);
+
+ var last_item = null, active_item = $("#side-menu a.section#active");
+
+ var found_top = false;
+ var displayable_items = scrollItems.map(function(itemI){
+ item = $(scrollItems[itemI]);
+ if (item && item.length) {
+ if (highlight_section_last_top == -1 || !highlight_section_completed) {
+ // just opening page - take item matching hash, or otherwise the first item visible
+ if (item.selector === window.location.hash || (item.offset().top > highlight_section_new_top - 20 && !found_top)) {
+ found_top = true;
+ if (item.selector === window.location.hash && item.offset().top < highlight_section_new_top + 60) {
+ // because of our top header, we need to scroll 64px down from any link
+ $('html, body').animate({scrollTop: item.offset().top - 64}, 0);
+ }
+ return item;
+ }
+ } else if (scroll_advancing) {
+ // if scrolling advance, pick up a section when title starts before 1/3 height
+ if (item.offset().top < highlight_section_new_top + $(window).height()/3)
+ return item;
+
+ // or if containing div is finished (usu the whole main content)
+ div_containing_item = item.closest("div");
+ if (div_containing_item.offset().top + div_containing_item.height() < highlight_section_new_bottom + 15)
+ return item;
+ // or when next title is visible
+ if (last_item && item.offset().top < highlight_section_new_bottom + 15)
+ return last_item;
+ } else {
+ // if scrolling back, pick up a section as soon as the title is visible,
+ if (item.offset().top < highlight_section_new_top)
+ return item;
+ // or if title is before the 2/3 point
+ // (not sure about this, probably want also to have
+ // "AND the id.top is <= displayable_itemsrent_active_it.top" so we don't jump FORWARD a section
+ // when scrolling BACK, with lots of tiny sections)
+ if ((item.offset().top < highlight_section_new_top + 2*$(window).height()/3)
+ && (!active_item || !active_item.offset() || active_item.offset().top >= item.offset().top))
+ return item;
+
+ }
+ last_item = item;
+ }
+ });
+ if (!highlight_section_completed && document.readyState === "complete") {
+ highlight_section_completed = true;
+ }
+ if (!displayable_items.length) {
+ $("#side-menu a.section").removeClass("active");
+ } else {
+ displayable_items = displayable_items[displayable_items.length-1];
+ var id = displayable_items && displayable_items.length ? displayable_items[0].id : "";
+ // Set/remove active class
+ new_active = $("#side-menu a.section").filter("[section-target='#"+id+"']");
+ if (new_active.hasClass("active")) {
+ // nothing needed
+ } else {
+ $("#side-menu a.section").removeClass("active");
+ $("#side-menu a.section").filter("[section-target='#"+id+"']").addClass("active");
+ }
+ }
+
+ highlight_section_last_top = highlight_section_new_top;
+};
+var highlight_new_section = function() {
+ highlight_section_completed = false;
+ highlight_section_last_top = -1;
+ highlight_section();
+}
+
+$(window).scroll(highlight_section);
+$(highlight_new_section);
+
+// detect link change - courtesy http://www.bennadel.com/blog/1520-binding-events-to-non-dom-objects-with-jquery.htm
+ (
+ function( $ ){
+ // Default to the current location.
+ var strLocation = window.location.href;
+ var strHash = window.location.hash;
+ var strPrevLocation = "";
+ var strPrevHash = "";
+
+ // This is how often we will be checkint for
+ // changes on the location.
+ var intIntervalTime = 100;
+
+ // This method removes the pound from the hash.
+ var fnCleanHash = function( strHash ){
+ return(
+ strHash.substring( 1, strHash.length )
+ );
+ }
+
+ // This will be the method that we use to check
+ // changes in the window location.
+ var fnCheckLocation = function(){
+ // Check to see if the location has changed.
+ if (strLocation != window.location.href){
+
+ // Store the new and previous locations.
+ strPrevLocation = strLocation;
+ strPrevHash = strHash;
+ strLocation = window.location.href;
+ strHash = window.location.hash;
+
+ // The location has changed. Trigger a
+ // change event on the location object,
+ // passing in the current and previous
+ // location values.
+ $( window.location ).trigger(
+ "change",
+ {
+ currentHref: strLocation,
+ currentHash: fnCleanHash( strHash ),
+ previousHref: strPrevLocation,
+ previousHash: fnCleanHash( strPrevHash )
+ }
+ );
+
+ }
+ }
+
+ // Set an interval to check the location changes.
+ setInterval( fnCheckLocation, intIntervalTime );
+ }
+ )( jQuery );
+// and trigger highlight section on link change
+$(window.location).bind("change", highlight_new_section);
+
+</script>
+
+ </div>
+ </div>
+</div>
+
+<div id="footer">
+ <div class="container">
+ <div class="row">
+ <div class="col-md-10 text-muted">
+ Apache Brooklyn is distributed under the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache
+ License v2.0</a>.
+ <br />
+ Apache Brooklyn is currently undergoing Incubation at The Apache Software
+ Foundation.
+ </div>
+ <div class="col-md-2">
+ <a class="btn btn-sm btn-default" href="https://github.com/apache/incubator-brooklyn/edit/master/docs/guide/ops/troubleshooting/going-deep-in-java-and-logs.md">Edit This Page</a>
+ <a href="http://0.0.0.0:4000/community/how-to-contribute-docs.html"
+ data-toggle="tooltip" data-placement="top" title="How to Edit Documentation" data-delay="400"/>
+ <span class="octicon octicon-question octicon-footer"></span>
+ </a>
+ </div>
+ </div>
+ </div>
+</div>
+
+
+
+
+
+<script src="//cdnjs.cloudflare.com/ajax/libs/zeroclipboard/1.3.1/ZeroClipboard.min.js"></script>
+
+<script language="JavaScript" type="application/javascript">
+ fix_padding_function = function () {
+ $('body').css('padding-top', parseInt($('#header').css("height"))+10);
+ $('body').css('padding-bottom', parseInt($('#footer').css("height"))+10);
+ };
+ $(window).resize(fix_padding_function);
+ $(window).load(fix_padding_function);
+
+ $(function () {
+ $('[data-toggle="tooltip"]').tooltip({ delay: { show: 600, hide: 100 }})
+ });
+
+/* generate anchors for headers, a la github and http://blog.parkermoore.de/2014/08/01/header-anchor-links-in-vanilla-javascript-for-github-pages-and-jekyll/ */
+var anchorForId = function (id, text) {
+ var anchor = document.createElement("a");
+ anchor.className = "header-link";
+ anchor.href = "#" + id;
+ anchor.innerHTML = "<i class=\"fa fa-link\"></i>";
+ return anchor;
+};
+
+var linkifyAnchors = function (level, containingElement) {
+ var headers = contentBlock.find("h" + level);
+ for (var h = 0; h < headers.length; h++) {
+ var header = headers[h];
+ if (typeof header.id !== "undefined" && header.id !== "") {
+ header.appendChild(anchorForId(header.id, $(header).text()));
+ }
+ }
+};
+
+$(function () {
+ contentBlock = $("#content_container");
+ if (!contentBlock) return;
+ for (var level = 1; level <= 6; level++) {
+ linkifyAnchors(level, contentBlock);
+ }
+});
+
+<!-- Clipboard support -->
+ ZeroClipboard.config({ moviePath: '//cdnjs.cloudflare.com/ajax/libs/zeroclipboard/1.3.1/ZeroClipboard.swf' });
+
+$(function() {
+ $('div.highlight').prepend(
+ $('<div class="clipboard_container" title="Copy to Clipboard">'+
+ '<div class="fa clipboard_button">'+
+ '<div class="on-active"><div>Copied to Clipboard</div></div>'+
+ '</div></div>'));
+ $('div.clipboard_container').each(function(index) {
+ var clipboard = new ZeroClipboard();
+ clipboard.clip( $(this).find(":first")[0], $(this)[0] );
+ var target = $(this).next();
+ var txt = target.text().trim();
+ if (target.find('code.bash')) {
+ // Strip out bash prompts from the start of each line (i.e. '$' or '%' characters
+ // at the very start, or immediately following any newline). Correctly handles continuation
+ // lines, where a leading '$' or '%' is *not* a prompt character.
+ txt = txt.replace(/(^|[^\\]\n)[$%] /g, "$1");
+ }
+ clipboard.on( 'dataRequested', function (client, args) {
+ client.setText( txt );
+ });
+ });
+});
+
+<!-- search -->
+ $(function() {
+ $('#simple_google')
+ .submit(function() {
+ $('input[name="q"]').val("site:" + document.location.hostname + " " + $('input[name="brooklyn-search"]').val());
+ return true;
+ });
+ $('input[name="brooklyn-search"]').focus(function() {
+ if ($(this).val() === $(this).attr('placeholder')) {
+ $(this).val('');
+ }
+ })
+ .blur(function() {
+ if ($(this).val() === '') {
+ $(this).val($(this).attr('placeholder'));
+ }
+ })
+ .blur();
+ });
+
+
+ <!-- analytics -->
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-30530918-1']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+
+
+<!-- page warning (for archive pages) -->
+
+
+ $(function() {
+ if (document.location.pathname.replace(/^\/([^\/]*).*$/, '$1') === "v"){
+ var thisversion = document.location.pathname.split("/")[2],
+ msg = "";
+ if (thisversion != 'latest' && (!$.cookie('brooklyn_versions') ||
+ (($.inArray('ALL', $.cookie('brooklyn_versions').split(",")) === -1) &&
+ ($.inArray(thisversion, $.cookie('brooklyn_versions').split(",")) === -1))) ){
+ msg += "<div class='warning_banner_image'><img src='/v/0.7.0-incubating/style/img/warning.png'/></div>";
+ msg += "<p>You are browsing the archive site for version <strong>"+thisversion+"</strong>.</p>";
+ if (thisversion.indexOf("SNAPSHOT") >= 0) {
+ msg += "<p>Note that SNAPSHOT versions have not been voted on and are not endorsed by the Apache Software Foundation.</p>";
+ msg += "<p>Do you understand and accept the risks?</p>";
+ } else {
+ msg += "<p>Is this deliberate?</p>";
+ }
+ msg += "<center><p class='warning_banner_buttons'>";
+ msg += "<a href = 'javascript:void(0);' onclick=\"set_user_version('"+thisversion+"');\">Yes, hide this warning</a>";
+ msg += "<a href = '/v/latest/'>No, take me to the latest version guide</a>";
+ msg += "<a href = '/meta/versions.html'>Show all versions</a>";
+ msg += "</p></center>"
+
+ $('#page_notes').html(msg);
+ $('#page_notes').fadeIn('slow');
+ }
+ }
+ });
+
+ function get_user_versions() {
+ return $.cookie("brooklyn_versions") ? $.cookie("brooklyn_versions").split(",") : [];
+ };
+ function set_user_version(version) {
+ var version_cookie = get_user_versions();
+ version_cookie.push(version);
+ $.cookie('brooklyn_versions', version_cookie, { expires: 365, path: '/' });
+ $('#page_notes').fadeOut();
+ event.preventDefault ? event.preventDefault() : event.returnValue = false;
+ };
+ function set_user_versions_all() {
+ var version_cookie = get_user_versions();
+ version_cookie.push("ALL");
+ $.cookie('brooklyn_versions', version_cookie, { expires: 365, path: '/' });
+ $('#page_notes').fadeOut();
+ event.preventDefault ? event.preventDefault() : event.returnValue = false;
+ };
+ function clear_user_versions() {
+ $.removeCookie('brooklyn_versions', { path: '/' });
+ $('#page_notes').fadeIn('slow');
+ event.preventDefault ? event.preventDefault() : event.returnValue = false;
+ };
+
+</script>
+
+</body>
+
+</html>
Added: incubator/brooklyn/site/v/0.7.0-incubating/ops/troubleshooting/images/failed-task-large.png
URL: http://svn.apache.org/viewvc/incubator/brooklyn/site/v/0.7.0-incubating/ops/troubleshooting/images/failed-task-large.png?rev=1693583&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/brooklyn/site/v/0.7.0-incubating/ops/troubleshooting/images/failed-task-large.png
------------------------------------------------------------------------------
svn:mime-type = image/png
Added: incubator/brooklyn/site/v/0.7.0-incubating/ops/troubleshooting/images/jmx-sensors-large.png
URL: http://svn.apache.org/viewvc/incubator/brooklyn/site/v/0.7.0-incubating/ops/troubleshooting/images/jmx-sensors-large.png?rev=1693583&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/brooklyn/site/v/0.7.0-incubating/ops/troubleshooting/images/jmx-sensors-large.png
------------------------------------------------------------------------------
svn:mime-type = image/png