You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@tomcat.apache.org by ma...@apache.org on 2011/04/20 13:28:53 UTC
svn commit: r1095367 - in /tomcat/trunk: java/org/apache/catalina/startup/
java/org/apache/jasper/compiler/ java/org/apache/tomcat/util/scan/
webapps/docs/
Author: markt
Date: Wed Apr 20 11:28:53 2011
New Revision: 1095367
URL: http://svn.apache.org/viewvc?rev=1095367&view=rev
Log:
Switch JAR scanning to use JarInputStream rather JarFile for significant startup performance improvements
Added:
tomcat/trunk/java/org/apache/tomcat/util/scan/NonClosingJarInputStream.java (with props)
Modified:
tomcat/trunk/java/org/apache/catalina/startup/ContextConfig.java
tomcat/trunk/java/org/apache/catalina/startup/TldConfig.java
tomcat/trunk/java/org/apache/jasper/compiler/TldLocationsCache.java
tomcat/trunk/webapps/docs/changelog.xml
Modified: tomcat/trunk/java/org/apache/catalina/startup/ContextConfig.java
URL: http://svn.apache.org/viewvc/tomcat/trunk/java/org/apache/catalina/startup/ContextConfig.java?rev=1095367&r1=1095366&r2=1095367&view=diff
==============================================================================
--- tomcat/trunk/java/org/apache/catalina/startup/ContextConfig.java (original)
+++ tomcat/trunk/java/org/apache/catalina/startup/ContextConfig.java Wed Apr 20 11:28:53 2011
@@ -44,8 +44,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
-import java.util.zip.ZipEntry;
+import java.util.jar.JarInputStream;
import javax.servlet.ServletContainerInitializer;
import javax.servlet.ServletContext;
@@ -1376,17 +1375,28 @@ public class ContextConfig
for (WebXml fragment : fragments) {
URL url = fragment.getURL();
- JarFile jarFile = null;
+ JarInputStream jarInputStream = null;
InputStream is = null;
ServletContainerInitializer sci = null;
try {
if ("jar".equals(url.getProtocol())) {
- JarURLConnection conn =
+ JarURLConnection jarConn =
(JarURLConnection) url.openConnection();
- jarFile = conn.getJarFile();
- ZipEntry entry = jarFile.getEntry(SCI_LOCATION);
+ URL resourceURL = jarConn.getJarFileURL();
+ URLConnection resourceConn = resourceURL.openConnection();
+ resourceConn.setUseCaches(false);
+
+ jarInputStream =
+ new JarInputStream(resourceConn.getInputStream());
+ JarEntry entry = jarInputStream.getNextJarEntry();
+ while (entry != null) {
+ if (SCI_LOCATION.equals(entry.getName())) {
+ break;
+ }
+ entry = jarInputStream.getNextJarEntry();
+ }
if (entry != null) {
- is = jarFile.getInputStream(entry);
+ is = jarInputStream;
}
} else if ("file".equals(url.getProtocol())) {
String path = url.getPath();
@@ -1412,9 +1422,9 @@ public class ContextConfig
// Ignore
}
}
- if (jarFile != null) {
+ if (jarInputStream != null) {
try {
- jarFile.close();
+ jarInputStream.close();
} catch (IOException e) {
// Ignore
}
@@ -1504,14 +1514,24 @@ public class ContextConfig
protected void processResourceJARs(Set<WebXml> fragments) {
for (WebXml fragment : fragments) {
URL url = fragment.getURL();
- JarFile jarFile = null;
+ JarInputStream jarInputStream = null;
try {
// Note: Ignore file URLs for now since only jar URLs will be accepted
if ("jar".equals(url.getProtocol())) {
- JarURLConnection conn =
+ JarURLConnection jarConn =
(JarURLConnection) url.openConnection();
- jarFile = conn.getJarFile();
- ZipEntry entry = jarFile.getEntry("META-INF/resources/");
+ URL resourceURL = jarConn.getJarFileURL();
+ URLConnection resourceConn = resourceURL.openConnection();
+ resourceConn.setUseCaches(false);
+ jarInputStream =
+ new JarInputStream(resourceConn.getInputStream());
+ JarEntry entry = jarInputStream.getNextJarEntry();
+ while (entry != null) {
+ if ("META-INF/resources/".equals(entry.getName())) {
+ break;
+ }
+ entry = jarInputStream.getNextJarEntry();
+ }
if (entry != null) {
context.addResourceJarUrl(url);
}
@@ -1520,9 +1540,9 @@ public class ContextConfig
log.error(sm.getString("contextConfig.resourceJarFail", url,
context.getName()));
} finally {
- if (jarFile != null) {
+ if (jarInputStream != null) {
try {
- jarFile.close();
+ jarInputStream.close();
} catch (IOException e) {
// Ignore
}
@@ -1780,50 +1800,42 @@ public class ContextConfig
protected void processAnnotationsJar(URL url, WebXml fragment) {
- JarFile jarFile = null;
+ JarInputStream jarInputStream = null;
try {
URLConnection urlConn = url.openConnection();
- JarURLConnection jarUrlConn;
+ JarURLConnection jarConn;
if (!(urlConn instanceof JarURLConnection)) {
// This should never happen
sm.getString("contextConfig.jarUrl", url);
return;
}
- jarUrlConn = (JarURLConnection) urlConn;
- jarUrlConn.setUseCaches(false);
- jarFile = jarUrlConn.getJarFile();
-
- Enumeration<JarEntry> jarEntries = jarFile.entries();
- while (jarEntries.hasMoreElements()) {
- JarEntry jarEntry = jarEntries.nextElement();
- String entryName = jarEntry.getName();
+ jarConn = (JarURLConnection) urlConn;
+ jarConn.setUseCaches(false);
+ URL resourceURL = jarConn.getJarFileURL();
+ URLConnection resourceConn = resourceURL.openConnection();
+
+ jarInputStream = new JarInputStream(resourceConn.getInputStream());
+
+ JarEntry entry = jarInputStream.getNextJarEntry();
+ while (entry != null) {
+ String entryName = entry.getName();
if (entryName.endsWith(".class")) {
- InputStream is = null;
try {
- is = jarFile.getInputStream(jarEntry);
- processAnnotationsStream(is, fragment);
+ processAnnotationsStream(jarInputStream, fragment);
} catch (IOException e) {
log.error(sm.getString("contextConfig.inputStreamJar",
entryName, url),e);
- } finally {
- if (is != null) {
- try {
- is.close();
- } catch (Throwable t) {
- ExceptionUtils.handleThrowable(t);
- }
- }
}
}
}
} catch (IOException e) {
log.error(sm.getString("contextConfig.jarFile", url), e);
} finally {
- if (jarFile != null) {
+ if (jarInputStream != null) {
try {
- jarFile.close();
+ jarInputStream.close();
} catch (Throwable t) {
ExceptionUtils.handleThrowable(t);
}
@@ -2302,45 +2314,48 @@ public class ContextConfig
private Map<String,WebXml> fragments = new HashMap<String,WebXml>();
@Override
- public void scan(JarURLConnection urlConn) throws IOException {
+ public void scan(JarURLConnection jarConn) throws IOException {
- JarFile jarFile = null;
- InputStream stream = null;
+ // JarURLConnection#getJarFile() creates temporary copies of the JAR
+ // if the underlying resource is not a file URL. That can be slow so
+ // the InputStream for the resource is used
+ URL resourceURL = jarConn.getJarFileURL();
+
+ JarInputStream jarInputStream = null;
WebXml fragment = new WebXml();
try {
- urlConn.setUseCaches(false);
- jarFile = urlConn.getJarFile();
- JarEntry fragmentEntry =
- jarFile.getJarEntry(FRAGMENT_LOCATION);
- if (fragmentEntry == null) {
+ URLConnection resourceConn = resourceURL.openConnection();
+ resourceConn.setUseCaches(false);
+ jarInputStream =
+ new JarInputStream(resourceConn.getInputStream());
+ JarEntry entry = jarInputStream.getNextJarEntry();
+ while (entry != null) {
+ if (FRAGMENT_LOCATION.equals(entry.getName())) {
+ break;
+ }
+ entry = jarInputStream.getNextJarEntry();
+ }
+
+ if (entry == null) {
// If there is no web.xml, normal JAR no impact on
// distributable
fragment.setDistributable(true);
} else {
- stream = jarFile.getInputStream(fragmentEntry);
InputSource source = new InputSource(
- urlConn.getJarFileURL().toString() +
- "!/" + FRAGMENT_LOCATION);
- source.setByteStream(stream);
+ resourceURL.toString() + "!/" + FRAGMENT_LOCATION);
+ source.setByteStream(jarInputStream);
parseWebXml(source, fragment, true);
}
} finally {
- if (jarFile != null) {
+ if (jarInputStream != null) {
try {
- jarFile.close();
- } catch (Throwable t) {
- ExceptionUtils.handleThrowable(t);
- }
- }
- if (stream != null) {
- try {
- stream.close();
+ jarInputStream.close();
} catch (Throwable t) {
ExceptionUtils.handleThrowable(t);
}
}
- fragment.setURL(urlConn.getURL());
+ fragment.setURL(jarConn.getURL());
if (fragment.getName() == null) {
fragment.setName(fragment.getURL().toString());
}
Modified: tomcat/trunk/java/org/apache/catalina/startup/TldConfig.java
URL: http://svn.apache.org/viewvc/tomcat/trunk/java/org/apache/catalina/startup/TldConfig.java?rev=1095367&r1=1095366&r2=1095367&view=diff
==============================================================================
--- tomcat/trunk/java/org/apache/catalina/startup/TldConfig.java (original)
+++ tomcat/trunk/java/org/apache/catalina/startup/TldConfig.java Wed Apr 20 11:28:53 2011
@@ -21,15 +21,15 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.JarURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Enumeration;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
import javax.servlet.ServletContext;
import javax.servlet.descriptor.TaglibDescriptor;
@@ -44,6 +44,7 @@ import org.apache.tomcat.JarScannerCallb
import org.apache.tomcat.util.ExceptionUtils;
import org.apache.tomcat.util.digester.Digester;
import org.apache.tomcat.util.res.StringManager;
+import org.apache.tomcat.util.scan.NonClosingJarInputStream;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
@@ -377,9 +378,10 @@ public final class TldConfig implements
log.trace(sm.getString("tldConfig.webxmlAdd", resourcePath,
descriptor.getTaglibURI()));
}
+ InputStream stream = null;
try {
- InputStream stream = context.getServletContext(
- ).getResourceAsStream(resourcePath);
+ stream = context.getServletContext().getResourceAsStream(
+ resourcePath);
XmlErrorHandler handler = tldScanStream(stream);
handler.logFindings(log, resourcePath);
taglibUris.add(descriptor.getTaglibURI());
@@ -387,6 +389,14 @@ public final class TldConfig implements
} catch (IOException ioe) {
log.warn(sm.getString("tldConfig.webxmlFail", resourcePath,
descriptor.getTaglibURI()), ioe);
+ } finally {
+ if (stream != null) {
+ try {
+ stream.close();
+ } catch (Throwable t) {
+ ExceptionUtils.handleThrowable(t);
+ }
+ }
}
}
}
@@ -494,34 +504,41 @@ public final class TldConfig implements
* Scans the given JarURLConnection for TLD files located in META-INF
* (or a sub-directory of it).
*
- * @param conn The JarURLConnection to the JAR file to scan
+ * @param jarConn The JarURLConnection to the JAR file to scan
*
* Keep in sync with o.a.j.comiler.TldLocationsCache
*/
- private void tldScanJar(JarURLConnection conn) {
+ private void tldScanJar(JarURLConnection jarConn) {
- JarFile jarFile = null;
+ // JarURLConnection#getJarFile() creates temporary copies of the JAR if
+ // the underlying resource is not a file URL. That can be slow so the
+ // InputStream for the resource is used
+ URL resourceURL = jarConn.getJarFileURL();
+ NonClosingJarInputStream jarInputStream = null;
String name = null;
+
try {
- conn.setUseCaches(false);
- jarFile = conn.getJarFile();
- Enumeration<JarEntry> entries = jarFile.entries();
- while (entries.hasMoreElements()) {
- JarEntry entry = entries.nextElement();
+ URLConnection resourceConn = resourceURL.openConnection();
+ resourceConn.setUseCaches(false);
+ jarInputStream =
+ new NonClosingJarInputStream(resourceConn.getInputStream());
+
+ JarEntry entry = jarInputStream.getNextJarEntry();
+ while (entry != null) {
name = entry.getName();
- if (!name.startsWith("META-INF/")) continue;
- if (!name.endsWith(".tld")) continue;
- InputStream stream = jarFile.getInputStream(entry);
- XmlErrorHandler handler = tldScanStream(stream);
- handler.logFindings(log, conn.getURL() + name);
+ if (name.startsWith("META-INF/") && name.endsWith(".tld")) {
+ XmlErrorHandler handler = tldScanStream(jarInputStream);
+ handler.logFindings(log, jarConn.getURL() + name);
+ }
+ entry = jarInputStream.getNextJarEntry();
}
} catch (IOException ioe) {
- log.warn(sm.getString("tldConfig.jarFail", conn.getURL() + name),
+ log.warn(sm.getString("tldConfig.jarFail", jarConn.getURL() + name),
ioe);
} finally {
- if (jarFile != null) {
+ if (jarInputStream != null) {
try {
- jarFile.close();
+ jarInputStream.reallyClose();
} catch (Throwable t) {
ExceptionUtils.handleThrowable(t);
}
@@ -556,13 +573,6 @@ public final class TldConfig implements
throw new IOException(s);
} finally {
tldDigester.reset();
- if (resourceStream != null) {
- try {
- resourceStream.close();
- } catch (Throwable t) {
- ExceptionUtils.handleThrowable(t);
- }
- }
}
return result;
}
Modified: tomcat/trunk/java/org/apache/jasper/compiler/TldLocationsCache.java
URL: http://svn.apache.org/viewvc/tomcat/trunk/java/org/apache/jasper/compiler/TldLocationsCache.java?rev=1095367&r1=1095366&r2=1095367&view=diff
==============================================================================
--- tomcat/trunk/java/org/apache/jasper/compiler/TldLocationsCache.java (original)
+++ tomcat/trunk/java/org/apache/jasper/compiler/TldLocationsCache.java Wed Apr 20 11:28:53 2011
@@ -21,14 +21,14 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.JarURLConnection;
-import java.util.Enumeration;
+import java.net.URL;
+import java.net.URLConnection;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
import javax.servlet.ServletContext;
@@ -40,6 +40,7 @@ import org.apache.juli.logging.Log;
import org.apache.juli.logging.LogFactory;
import org.apache.tomcat.JarScanner;
import org.apache.tomcat.JarScannerCallback;
+import org.apache.tomcat.util.scan.NonClosingJarInputStream;
/**
@@ -392,32 +393,39 @@ public class TldLocationsCache {
* (or a subdirectory of it), adding an implicit map entry to the taglib
* map for any TLD that has a <uri> element.
*
- * @param conn The JarURLConnection to the JAR file to scan
+ * @param jarConn The JarURLConnection to the JAR file to scan
*
* Keep in sync with o.a.c.startup.TldConfig
*/
- private void tldScanJar(JarURLConnection conn) throws IOException {
+ private void tldScanJar(JarURLConnection jarConn) throws IOException {
- JarFile jarFile = null;
- String resourcePath = conn.getJarFileURL().toString();
+ // JarURLConnection#getJarFile() creates temporary copies of the JAR if
+ // the underlying resource is not a file URL. That can be slow so the
+ // InputStream for the resource is used
+ URL resourceURL = jarConn.getJarFileURL();
+ String resourcePath = resourceURL.toString();
+
+ NonClosingJarInputStream jarInputStream = null;
+
boolean foundTld = false;
try {
- conn.setUseCaches(false);
- jarFile = conn.getJarFile();
- Enumeration<JarEntry> entries = jarFile.entries();
- while (entries.hasMoreElements()) {
- JarEntry entry = entries.nextElement();
+ URLConnection resourceConn = resourceURL.openConnection();
+ resourceConn.setUseCaches(false);
+ jarInputStream =
+ new NonClosingJarInputStream(resourceConn.getInputStream());
+ JarEntry entry = jarInputStream.getNextJarEntry();
+ while (entry != null) {
String name = entry.getName();
- if (!name.startsWith("META-INF/")) continue;
- if (!name.endsWith(".tld")) continue;
- foundTld = true;
- InputStream stream = jarFile.getInputStream(entry);
- tldScanStream(resourcePath, name, stream);
+ if (name.startsWith("META-INF/") && name.endsWith(".tld")) {
+ foundTld = true;
+ tldScanStream(resourcePath, name, jarInputStream);
+ }
+ entry = jarInputStream.getNextJarEntry();
}
} finally {
- if (jarFile != null) {
+ if (jarInputStream != null) {
try {
- jarFile.close();
+ jarInputStream.reallyClose();
} catch (Throwable t) {
ExceptionUtils.handleThrowable(t);
}
@@ -468,14 +476,6 @@ public class TldLocationsCache {
} catch (JasperException e) {
// Hack - makes exception handling simpler
throw new IOException(e);
- } finally {
- if (stream != null) {
- try {
- stream.close();
- } catch (Throwable t) {
- ExceptionUtils.handleThrowable(t);
- }
- }
}
}
Added: tomcat/trunk/java/org/apache/tomcat/util/scan/NonClosingJarInputStream.java
URL: http://svn.apache.org/viewvc/tomcat/trunk/java/org/apache/tomcat/util/scan/NonClosingJarInputStream.java?rev=1095367&view=auto
==============================================================================
--- tomcat/trunk/java/org/apache/tomcat/util/scan/NonClosingJarInputStream.java (added)
+++ tomcat/trunk/java/org/apache/tomcat/util/scan/NonClosingJarInputStream.java Wed Apr 20 11:28:53 2011
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tomcat.util.scan;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.jar.JarInputStream;
+
+/**
+ * When using a {@link JarInputStream} with an XML parser, the stream will be
+ * closed by the parser. This causes problems if multiple entries from the JAR
+ * need to be parsed. This implementation makes {{@link #close()} a NO-OP and
+ * adds {@link #reallyClose()} that will close the stream.
+ */
+public class NonClosingJarInputStream extends JarInputStream {
+
+ public NonClosingJarInputStream(InputStream in, boolean verify)
+ throws IOException {
+ super(in, verify);
+ }
+
+ public NonClosingJarInputStream(InputStream in) throws IOException {
+ super(in);
+ }
+
+ @Override
+ public void close() throws IOException {
+ // Make this a NO-OP so that further entries can be read from the stream
+ }
+
+ public void reallyClose() throws IOException {
+ super.close();
+ }
+}
Propchange: tomcat/trunk/java/org/apache/tomcat/util/scan/NonClosingJarInputStream.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: tomcat/trunk/webapps/docs/changelog.xml
URL: http://svn.apache.org/viewvc/tomcat/trunk/webapps/docs/changelog.xml?rev=1095367&r1=1095366&r2=1095367&view=diff
==============================================================================
--- tomcat/trunk/webapps/docs/changelog.xml (original)
+++ tomcat/trunk/webapps/docs/changelog.xml Wed Apr 20 11:28:53 2011
@@ -79,6 +79,13 @@
fragments to the list of JARs to skip when scanning for TLDs and web
fragments. (markt)
</add>
+ <fix>
+ While scanning JARs for TLDs and fragments, avoid using JarFile and use
+ JarInputStream as in most circumstances where JARs are scanned, JarFile
+ will create a temporary copy of the JAR rather than using the resource
+ directly. This change significantly improves startup performance for
+ applications with lots of JARs to be scanned. (markt)
+ </fix>
</changelog>
</subsection>
<subsection name="Coyote">
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@tomcat.apache.org
For additional commands, e-mail: dev-help@tomcat.apache.org
Re: svn commit: r1095367 - in /tomcat/trunk: java/org/apache/catalina/startup/
java/org/apache/jasper/compiler/ java/org/apache/tomcat/util/scan/ webapps/docs/
Posted by sebb <se...@gmail.com>.
On 20 April 2011 17:42, Mark Thomas <ma...@apache.org> wrote:
> On 20/04/2011 17:33, Filip Hanik - Dev Lists wrote:
>> On 4/20/2011 10:07 AM, Mark Thomas wrote:
>>> The new code is better than the old. You are welcome to try and improve
>>> it further although I don't think there is much more scope for
>>> improvement given the performance figures I am seeing.
>> My suggestion here would be to:
>> 1. When a JAR file has been scanned, then save the result of the scan
>> to the work directory containing
>> - name of jar
>> - timestamp of the JAR
>> - MD5 of jar
>> - size of jar in bytes
>> - result of scan
>>
>> This would serve a jar scanner cache. If I restart tomcat several times
>> without any changes to the libraries, there is no need to incur the 12
>> second startup time. While 30s -> 12s is a dramatic improvement, you can
>> bring down the penalty of a scan even more.
>
> As per my previous mail, that 12s is nearly all app init time, *not* JAR
> scanning time. From the current performance figures I'm far from
> convinced that a cache would add any value.
And the disk cache files still have to be opened and read ...
>
> Mark
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@tomcat.apache.org
> For additional commands, e-mail: dev-help@tomcat.apache.org
>
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@tomcat.apache.org
For additional commands, e-mail: dev-help@tomcat.apache.org
Re: svn commit: r1095367 - in /tomcat/trunk: java/org/apache/catalina/startup/
java/org/apache/jasper/compiler/ java/org/apache/tomcat/util/scan/ webapps/docs/
Posted by Mark Thomas <ma...@apache.org>.
On 20/04/2011 17:33, Filip Hanik - Dev Lists wrote:
> On 4/20/2011 10:07 AM, Mark Thomas wrote:
>> The new code is better than the old. You are welcome to try and improve
>> it further although I don't think there is much more scope for
>> improvement given the performance figures I am seeing.
> My suggestion here would be to:
> 1. When a JAR file has been scanned, then save the result of the scan
> to the work directory containing
> - name of jar
> - timestamp of the JAR
> - MD5 of jar
> - size of jar in bytes
> - result of scan
>
> This would serve a jar scanner cache. If I restart tomcat several times
> without any changes to the libraries, there is no need to incur the 12
> second startup time. While 30s -> 12s is a dramatic improvement, you can
> bring down the penalty of a scan even more.
As per my previous mail, that 12s is nearly all app init time, *not* JAR
scanning time. From the current performance figures I'm far from
convinced that a cache would add any value.
Mark
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@tomcat.apache.org
For additional commands, e-mail: dev-help@tomcat.apache.org
Re: svn commit: r1095367 - in /tomcat/trunk: java/org/apache/catalina/startup/
java/org/apache/jasper/compiler/ java/org/apache/tomcat/util/scan/ webapps/docs/
Posted by Filip Hanik - Dev Lists <de...@hanik.com>.
On 4/20/2011 10:07 AM, Mark Thomas wrote:
> The new code is better than the old. You are welcome to try and improve
> it further although I don't think there is much more scope for
> improvement given the performance figures I am seeing.
My suggestion here would be to:
1. When a JAR file has been scanned, then save the result of the scan to the work directory containing
- name of jar
- timestamp of the JAR
- MD5 of jar
- size of jar in bytes
- result of scan
This would serve a jar scanner cache. If I restart tomcat several times without any changes to the libraries, there is no need to incur the
12 second startup time. While 30s -> 12s is a dramatic improvement, you can bring down the penalty of a scan even more.
Ideally there is also an option to disable the cache, for the skeptic ones :)
Filip
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@tomcat.apache.org
For additional commands, e-mail: dev-help@tomcat.apache.org
Re: svn commit: r1095367 - in /tomcat/trunk: java/org/apache/catalina/startup/
java/org/apache/jasper/compiler/ java/org/apache/tomcat/util/scan/ webapps/docs/
Posted by Mark Thomas <ma...@apache.org>.
On 20/04/2011 17:02, Konstantin Kolinko wrote:
> 2011/4/20 Mark Thomas <ma...@apache.org>:
>>>> + // JarURLConnection#getJarFile() creates temporary copies of the JAR
>>>> + // if the underlying resource is not a file URL. That can be slow so
>>>
>>> What URLs are there? Why aren't they file ones?
>>
>> Resources obtained via the context are jndi URLs.
>>
>
> Isn't it possible access the actual files?
If they exist - which they may not.
The new code is better than the old. You are welcome to try and improve
it further although I don't think there is much more scope for
improvement given the performance figures I am seeing.
Mark
> AFAIK, even if we run with expandWARs=false the jar files are
> extracted and copied to app's workdir.
>
>>> A zip file has "central directory" i.e. master index at the end of the
>>> file. IIRC when you are using random-access methods of Zip file, that
>>> index is loaded into memory once. Scanning the file sequentially does
>>> not use the index and I think it will be slower.
>>
>> Old code:
>> - read entire file (sequentially)
>> - write entire file to temp location
>> - read required data from file
>>
>> New code:
>> - read file sequentially until we find the bit we want.
>>
>> On a reasonably complex app with quite a few JARs the start-up time went
>> from ~30s to ~12s with these changes with most of the 12s now being app
>> init code. Copying the file was killing performance.
>>
>
> Best regards,
> Konstantin Kolinko
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@tomcat.apache.org
> For additional commands, e-mail: dev-help@tomcat.apache.org
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@tomcat.apache.org
For additional commands, e-mail: dev-help@tomcat.apache.org
Re: svn commit: r1095367 - in /tomcat/trunk: java/org/apache/catalina/startup/
java/org/apache/jasper/compiler/ java/org/apache/tomcat/util/scan/ webapps/docs/
Posted by Konstantin Kolinko <kn...@gmail.com>.
2011/4/20 Mark Thomas <ma...@apache.org>:
>>> + // JarURLConnection#getJarFile() creates temporary copies of the JAR
>>> + // if the underlying resource is not a file URL. That can be slow so
>>
>> What URLs are there? Why aren't they file ones?
>
> Resources obtained via the context are jndi URLs.
>
Isn't it possible access the actual files?
AFAIK, even if we run with expandWARs=false the jar files are
extracted and copied to app's workdir.
>> A zip file has "central directory" i.e. master index at the end of the
>> file. IIRC when you are using random-access methods of Zip file, that
>> index is loaded into memory once. Scanning the file sequentially does
>> not use the index and I think it will be slower.
>
> Old code:
> - read entire file (sequentially)
> - write entire file to temp location
> - read required data from file
>
> New code:
> - read file sequentially until we find the bit we want.
>
> On a reasonably complex app with quite a few JARs the start-up time went
> from ~30s to ~12s with these changes with most of the 12s now being app
> init code. Copying the file was killing performance.
>
Best regards,
Konstantin Kolinko
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@tomcat.apache.org
For additional commands, e-mail: dev-help@tomcat.apache.org
Re: svn commit: r1095367 - in /tomcat/trunk: java/org/apache/catalina/startup/
java/org/apache/jasper/compiler/ java/org/apache/tomcat/util/scan/ webapps/docs/
Posted by Mark Thomas <ma...@apache.org>.
On 20/04/2011 16:51, Konstantin Kolinko wrote:
> 2011/4/20 <ma...@apache.org>:
>> Author: markt
>> Date: Wed Apr 20 11:28:53 2011
>> New Revision: 1095367
>>
>> URL: http://svn.apache.org/viewvc?rev=1095367&view=rev
>> Log:
>> Switch JAR scanning to use JarInputStream rather JarFile for significant startup performance improvements
>>
>
>
>> + <fix>
>> + While scanning JARs for TLDs and fragments, avoid using JarFile and use
>> + JarInputStream as in most circumstances where JARs are scanned, JarFile
>> + will create a temporary copy of the JAR rather than using the resource
>> + directly. This change significantly improves startup performance for
>> + applications with lots of JARs to be scanned. (markt)
>> + </fix>
>
>> + // JarURLConnection#getJarFile() creates temporary copies of the JAR
>> + // if the underlying resource is not a file URL. That can be slow so
>
> What URLs are there? Why aren't they file ones?
Resources obtained via the context are jndi URLs.
> A zip file has "central directory" i.e. master index at the end of the
> file. IIRC when you are using random-access methods of Zip file, that
> index is loaded into memory once. Scanning the file sequentially does
> not use the index and I think it will be slower.
Old code:
- read entire file (sequentially)
- write entire file to temp location
- read required data from file
New code:
- read file sequentially until we find the bit we want.
On a reasonably complex app with quite a few JARs the start-up time went
from ~30s to ~12s with these changes with most of the 12s now being app
init code. Copying the file was killing performance.
Mark
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@tomcat.apache.org
For additional commands, e-mail: dev-help@tomcat.apache.org
Re: svn commit: r1095367 - in /tomcat/trunk: java/org/apache/catalina/startup/
java/org/apache/jasper/compiler/ java/org/apache/tomcat/util/scan/ webapps/docs/
Posted by Konstantin Kolinko <kn...@gmail.com>.
2011/4/20 <ma...@apache.org>:
> Author: markt
> Date: Wed Apr 20 11:28:53 2011
> New Revision: 1095367
>
> URL: http://svn.apache.org/viewvc?rev=1095367&view=rev
> Log:
> Switch JAR scanning to use JarInputStream rather JarFile for significant startup performance improvements
>
> + <fix>
> + While scanning JARs for TLDs and fragments, avoid using JarFile and use
> + JarInputStream as in most circumstances where JARs are scanned, JarFile
> + will create a temporary copy of the JAR rather than using the resource
> + directly. This change significantly improves startup performance for
> + applications with lots of JARs to be scanned. (markt)
> + </fix>
> + // JarURLConnection#getJarFile() creates temporary copies of the JAR
> + // if the underlying resource is not a file URL. That can be slow so
What URLs are there? Why aren't they file ones?
A zip file has "central directory" i.e. master index at the end of the
file. IIRC when you are using random-access methods of Zip file, that
index is loaded into memory once. Scanning the file sequentially does
not use the index and I think it will be slower.
Best regards,
Konstantin Kolinko
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@tomcat.apache.org
For additional commands, e-mail: dev-help@tomcat.apache.org