You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jmeter.apache.org by pm...@apache.org on 2016/07/31 14:39:30 UTC
svn commit: r1754663 - in /jmeter/trunk:
src/core/org/apache/jmeter/testelement/ src/core/org/apache/jmeter/threads/
src/jorphan/org/apache/jorphan/util/
src/protocol/http/org/apache/jmeter/protocol/http/sampler/
src/protocol/http/org/apache/jmeter/pro...
Author: pmouawad
Date: Sun Jul 31 14:39:30 2016
New Revision: 1754663
URL: http://svn.apache.org/viewvc?rev=1754663&view=rev
Log:
Bug 59882 - Reduce memory allocations for better throughput
Based on PR 217 contributed by Benoit Wiart (b.wiart at ubik-ingenierie.com)
This closes #217 on github.
Bugzilla Id: 59882
Added:
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/util/DirectAccessByteArrayOutputStream.java (with props)
Modified:
jmeter/trunk/src/core/org/apache/jmeter/testelement/AbstractTestElement.java
jmeter/trunk/src/core/org/apache/jmeter/threads/JMeterThread.java
jmeter/trunk/src/jorphan/org/apache/jorphan/util/JOrphanUtils.java
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBase.java
jmeter/trunk/xdocs/changes.xml
Modified: jmeter/trunk/src/core/org/apache/jmeter/testelement/AbstractTestElement.java
URL: http://svn.apache.org/viewvc/jmeter/trunk/src/core/org/apache/jmeter/testelement/AbstractTestElement.java?rev=1754663&r1=1754662&r2=1754663&view=diff
==============================================================================
--- jmeter/trunk/src/core/org/apache/jmeter/testelement/AbstractTestElement.java (original)
+++ jmeter/trunk/src/core/org/apache/jmeter/testelement/AbstractTestElement.java Sun Jul 31 14:39:30 2016
@@ -187,6 +187,16 @@ public abstract class AbstractTestElemen
}
return prop;
}
+
+ /**
+ * Null property are wrapped in a {@link NullProperty}
+ * This method avoids this wrapping
+ * for internal use only
+ * @since 3.1
+ */
+ private JMeterProperty getRawProperty(String key) {
+ return propMap.get(key);
+ }
@Override
public void traverse(TestElementTraverser traverser) {
@@ -230,8 +240,8 @@ public abstract class AbstractTestElemen
@Override
public int getPropertyAsInt(String key, int defaultValue) {
- JMeterProperty jmp = getProperty(key);
- return jmp instanceof NullProperty ? defaultValue : jmp.getIntValue();
+ JMeterProperty jmp = getRawProperty(key);
+ return jmp == null || jmp instanceof NullProperty ? defaultValue : jmp.getIntValue();
}
@Override
@@ -241,8 +251,8 @@ public abstract class AbstractTestElemen
@Override
public boolean getPropertyAsBoolean(String key, boolean defaultVal) {
- JMeterProperty jmp = getProperty(key);
- return jmp instanceof NullProperty ? defaultVal : jmp.getBooleanValue();
+ JMeterProperty jmp = getRawProperty(key);
+ return jmp == null || jmp instanceof NullProperty ? defaultVal : jmp.getBooleanValue();
}
@Override
@@ -257,8 +267,8 @@ public abstract class AbstractTestElemen
@Override
public long getPropertyAsLong(String key, long defaultValue) {
- JMeterProperty jmp = getProperty(key);
- return jmp instanceof NullProperty ? defaultValue : jmp.getLongValue();
+ JMeterProperty jmp = getRawProperty(key);
+ return jmp == null || jmp instanceof NullProperty ? defaultValue : jmp.getLongValue();
}
@Override
@@ -273,8 +283,8 @@ public abstract class AbstractTestElemen
@Override
public String getPropertyAsString(String key, String defaultValue) {
- JMeterProperty jmp = getProperty(key);
- return jmp instanceof NullProperty ? defaultValue : jmp.getStringValue();
+ JMeterProperty jmp = getRawProperty(key);
+ return jmp == null || jmp instanceof NullProperty ? defaultValue : jmp.getStringValue();
}
/**
Modified: jmeter/trunk/src/core/org/apache/jmeter/threads/JMeterThread.java
URL: http://svn.apache.org/viewvc/jmeter/trunk/src/core/org/apache/jmeter/threads/JMeterThread.java?rev=1754663&r1=1754662&r2=1754663&view=diff
==============================================================================
--- jmeter/trunk/src/core/org/apache/jmeter/threads/JMeterThread.java (original)
+++ jmeter/trunk/src/core/org/apache/jmeter/threads/JMeterThread.java Sun Jul 31 14:39:30 2016
@@ -457,16 +457,20 @@ public class JMeterThread implements Run
// Perform the actual sample
currentSampler = sampler;
- for(SampleMonitor monitor : sampleMonitors) {
- monitor.sampleStarting(sampler);
+ if(!sampleMonitors.isEmpty()) {
+ for(SampleMonitor monitor : sampleMonitors) {
+ monitor.sampleStarting(sampler);
+ }
}
SampleResult result = null;
try {
result = sampler.sample(null); // TODO: remove this useless Entry parameter
} finally {
- for(SampleMonitor monitor : sampleMonitors) {
- monitor.sampleEnded(sampler);
- }
+ if(!sampleMonitors.isEmpty()) {
+ for(SampleMonitor monitor : sampleMonitors) {
+ monitor.sampleEnded(sampler);
+ }
+ }
}
currentSampler = null;
Modified: jmeter/trunk/src/jorphan/org/apache/jorphan/util/JOrphanUtils.java
URL: http://svn.apache.org/viewvc/jmeter/trunk/src/jorphan/org/apache/jorphan/util/JOrphanUtils.java?rev=1754663&r1=1754662&r2=1754663&view=diff
==============================================================================
--- jmeter/trunk/src/jorphan/org/apache/jorphan/util/JOrphanUtils.java (original)
+++ jmeter/trunk/src/jorphan/org/apache/jorphan/util/JOrphanUtils.java Sun Jul 31 14:39:30 2016
@@ -252,7 +252,7 @@ public final class JOrphanUtils {
/**
* Version of String.replaceAll() for JDK1.3
* See below for another version which replaces strings rather than chars
- *
+ * and provides a fast path which does not allocate memory
* @param source
* input string
* @param search
@@ -262,15 +262,22 @@ public final class JOrphanUtils {
* @return the output string
*/
public static String replaceAllChars(String source, char search, String replace) {
+ int indexOf = source.indexOf(search);
+ if(indexOf == -1) {
+ return source;
+ }
+
+ int offset = 0;
char[] chars = source.toCharArray();
StringBuilder sb = new StringBuilder(source.length()+20);
- for(char c : chars){
- if (c == search){
- sb.append(replace);
- } else {
- sb.append(c);
- }
+ while(indexOf != -1) {
+ sb.append(chars, offset, indexOf-offset);
+ sb.append(replace);
+ offset = indexOf +1;
+ indexOf = source.indexOf(search, offset);
}
+ sb.append(chars, offset, chars.length- offset);
+
return sb.toString();
}
Modified: jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java
URL: http://svn.apache.org/viewvc/jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java?rev=1754663&r1=1754662&r2=1754663&view=diff
==============================================================================
--- jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java (original)
+++ jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java Sun Jul 31 14:39:30 2016
@@ -97,6 +97,7 @@ import org.apache.http.impl.client.Defau
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.conn.SystemDefaultDnsResolver;
import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.message.BufferedHeader;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.params.CoreProtocolPNames;
@@ -107,6 +108,7 @@ import org.apache.http.protocol.BasicHtt
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.apache.http.protocol.HttpCoreContext;
+import org.apache.http.util.CharArrayBuffer;
import org.apache.jmeter.protocol.http.control.AuthManager;
import org.apache.jmeter.protocol.http.control.CacheManager;
import org.apache.jmeter.protocol.http.control.CookieManager;
@@ -750,7 +752,8 @@ public class HTTPHC4Impl extends HTTPHCA
HttpClientKey key = new HttpClientKey(url, useProxy, proxyHost, proxyPort, proxyUser, proxyPass);
HttpClient httpClient = null;
- if(this.testElement.isConcurrentDwn()) {
+ boolean concurrentDwn = this.testElement.isConcurrentDwn();
+ if(concurrentDwn) {
httpClient = (HttpClient) JMeterContextService.getContext().getSamplerContext().get(HTTPCLIENT_TOKEN);
}
@@ -785,7 +788,7 @@ public class HTTPHC4Impl extends HTTPHCA
// Modern browsers use more connections per host than the current httpclient default (2)
// when using parallel download the httpclient and connection manager are shared by the downloads threads
// to be realistic JMeter must set an higher value to DefaultMaxPerRoute
- if(this.testElement.isConcurrentDwn()) {
+ if(concurrentDwn) {
try {
int maxConcurrentDownloads = Integer.parseInt(this.testElement.getConcurrentPool());
connManager.setDefaultMaxPerRoute(Math.max(maxConcurrentDownloads, connManager.getDefaultMaxPerRoute()));
@@ -844,7 +847,7 @@ public class HTTPHC4Impl extends HTTPHCA
}
}
- if(this.testElement.isConcurrentDwn()) {
+ if(concurrentDwn) {
JMeterContextService.getContext().getSamplerContext().put(HTTPCLIENT_TOKEN, httpClient);
}
@@ -954,11 +957,12 @@ public class HTTPHC4Impl extends HTTPHCA
* @return string containing the headers, one per line
*/
private String getResponseHeaders(HttpResponse response, HttpContext localContext) {
- StringBuilder headerBuf = new StringBuilder();
+ Header[] rh = response.getAllHeaders();
+
+ StringBuilder headerBuf = new StringBuilder(40 * (rh.length+1));
headerBuf.append(response.getStatusLine());// header[0] is not the status line...
headerBuf.append("\n"); // $NON-NLS-1$
- Header[] rh = response.getAllHeaders();
for (Header responseHeader : rh) {
writeResponseHeader(headerBuf, responseHeader);
}
@@ -966,16 +970,21 @@ public class HTTPHC4Impl extends HTTPHCA
}
/**
- * Write responseHeader to headerBuffer
+ * Write responseHeader to headerBuffer in an optimized way
* @param headerBuffer {@link StringBuilder}
* @param responseHeader {@link Header}
*/
- private void writeResponseHeader(StringBuilder headerBuffer,
- Header responseHeader) {
- headerBuffer.append(responseHeader.getName())
+ private void writeResponseHeader(StringBuilder headerBuffer, Header responseHeader) {
+ if(responseHeader instanceof BufferedHeader) {
+ CharArrayBuffer buffer = ((BufferedHeader)responseHeader).getBuffer();
+ headerBuffer.append(buffer.buffer(), 0, buffer.length()).append("\n"); // $NON-NLS-1$;
+ }
+ else {
+ headerBuffer.append(responseHeader.getName())
.append(": ") // $NON-NLS-1$
.append(responseHeader.getValue())
.append("\n"); // $NON-NLS-1$
+ }
}
/**
@@ -1078,7 +1087,7 @@ public class HTTPHC4Impl extends HTTPHCA
private String getConnectionHeaders(HttpRequest method) {
if(method != null) {
// Get all the request headers
- StringBuilder hdrs = new StringBuilder(100);
+ StringBuilder hdrs = new StringBuilder(150);
Header[] requestHeaders = method.getAllHeaders();
for (Header requestHeader : requestHeaders) {
// Exclude the COOKIE header, since cookie is reported separately in the sample
@@ -1221,7 +1230,7 @@ public class HTTPHC4Impl extends HTTPHCA
}
bos.flush();
// We get the posted bytes using the encoding used to create it
- postedBody.append(new String(bos.toByteArray(),
+ postedBody.append(bos.toString(
contentEncoding == null ? "US-ASCII" // $NON-NLS-1$ this is the default used by HttpClient
: contentEncoding));
bos.close();
@@ -1344,11 +1353,8 @@ public class HTTPHC4Impl extends HTTPHCA
post.getEntity().writeTo(bos);
bos.flush();
// We get the posted bytes using the encoding used to create it
- if (contentEncoding != null) {
- postedBody.append(new String(bos.toByteArray(), contentEncoding));
- } else {
- postedBody.append(new String(bos.toByteArray(), SampleResult.DEFAULT_HTTP_ENCODING));
- }
+ postedBody.append(bos.toString(contentEncoding != null?contentEncoding:SampleResult.DEFAULT_HTTP_ENCODING));
+
bos.close();
} else {
postedBody.append("<RequestEntity was not repeatable, cannot view what was sent>");
Modified: jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBase.java
URL: http://svn.apache.org/viewvc/jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBase.java?rev=1754663&r1=1754662&r2=1754663&view=diff
==============================================================================
--- jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBase.java (original)
+++ jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBase.java Sun Jul 31 14:39:30 2016
@@ -19,6 +19,7 @@ package org.apache.jmeter.protocol.http.
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
@@ -56,6 +57,7 @@ import org.apache.jmeter.protocol.http.p
import org.apache.jmeter.protocol.http.parser.LinkExtractorParser;
import org.apache.jmeter.protocol.http.sampler.ResourcesDownloader.AsynSamplerResultHolder;
import org.apache.jmeter.protocol.http.util.ConversionUtils;
+import org.apache.jmeter.protocol.http.util.DirectAccessByteArrayOutputStream;
import org.apache.jmeter.protocol.http.util.EncoderCache;
import org.apache.jmeter.protocol.http.util.HTTPArgument;
import org.apache.jmeter.protocol.http.util.HTTPConstants;
@@ -682,9 +684,13 @@ public abstract class HTTPSamplerBase ex
* @return port number or UNSPECIFIED_PORT (== 0)
*/
public int getPortIfSpecified() {
- String port_s = getPropertyAsString(PORT, UNSPECIFIED_PORT_AS_STRING);
+ String portAsString = getPropertyAsString(PORT);
+ if(portAsString == null || portAsString.isEmpty()) {
+ return UNSPECIFIED_PORT;
+ }
+
try {
- return Integer.parseInt(port_s.trim());
+ return Integer.parseInt(portAsString.trim());
} catch (NumberFormatException e) {
return UNSPECIFIED_PORT;
}
@@ -992,13 +998,21 @@ public abstract class HTTPSamplerBase ex
* @return the QueryString value
*/
public String getQueryString(String contentEncoding) {
+
+ CollectionProperty arguments = getArguments().getArguments();
+ // Optimisation : avoid building useless objects if empty arguments
+ if(arguments.size() == 0) {
+ return "";
+ }
+
// Check if the sampler has a specified content encoding
if (JOrphanUtils.isBlank(contentEncoding)) {
// We use the encoding which should be used according to the HTTP spec, which is UTF-8
contentEncoding = EncoderCache.URL_ARGUMENT_ENCODING;
}
- StringBuilder buf = new StringBuilder();
- PropertyIterator iter = getArguments().iterator();
+
+ StringBuilder buf = new StringBuilder(arguments.size() * 15);
+ PropertyIterator iter = arguments.iterator();
boolean first = true;
while (iter.hasNext()) {
HTTPArgument item = null;
@@ -1598,7 +1612,9 @@ public abstract class HTTPSamplerBase ex
protected HTTPSampleResult resultProcessing(boolean areFollowingRedirect, int frameDepth, HTTPSampleResult res) {
boolean wasRedirected = false;
if (!areFollowingRedirect && res.isRedirect()) {
- log.debug("Location set to - " + res.getRedirectLocation());
+ if(log.isDebugEnabled()) {
+ log.debug("Location set to - " + res.getRedirectLocation());
+ }
if (getFollowRedirects()) {
res = followRedirects(res, frameDepth);
@@ -1606,7 +1622,8 @@ public abstract class HTTPSamplerBase ex
wasRedirected = true;
}
}
- if (isImageParser() && (SampleResult.TEXT).equals(res.getDataType()) && res.isSuccessful()) {
+
+ if (res.isSuccessful() && SampleResult.TEXT.equals(res.getDataType()) && isImageParser() ) {
if (frameDepth > MAX_FRAME_DEPTH) {
HTTPSampleResult errSubResult = new HTTPSampleResult(res);
errSubResult.removeSubResults();
@@ -1751,27 +1768,29 @@ public abstract class HTTPSamplerBase ex
* @throws IOException if reading the result fails
*/
public byte[] readResponse(SampleResult sampleResult, InputStream in, int length) throws IOException {
+
+ OutputStream w = null;
try {
byte[] readBuffer = new byte[8192]; // 8kB is the (max) size to have the latency ('the first packet')
int bufferSize = 32;// Enough for MD5
MessageDigest md = null;
- boolean asMD5 = useMD5();
- if (asMD5) {
+ boolean knownResponseLength = length > 0;// may also happen if long value > int.max
+ if (useMD5()) {
try {
md = MessageDigest.getInstance("MD5"); //$NON-NLS-1$
} catch (NoSuchAlgorithmException e) {
log.error("Should not happen - could not find MD5 digest", e);
- asMD5 = false;
}
} else {
- if (length <= 0) {// may also happen if long value > int.max
+ if (!knownResponseLength) {
bufferSize = 4 * 1024;
} else {
bufferSize = length;
}
}
- ByteArrayOutputStream w = new ByteArrayOutputStream(bufferSize);
+
+
int bytesRead = 0;
int totalBytes = 0;
boolean first = true;
@@ -1779,32 +1798,62 @@ public abstract class HTTPSamplerBase ex
if (first) {
sampleResult.latencyEnd();
first = false;
+ if(md == null) {
+ if(knownResponseLength) {
+ w = new DirectAccessByteArrayOutputStream(bufferSize);
+ }
+ else {
+ w = new org.apache.commons.io.output.ByteArrayOutputStream(bufferSize);
+ }
+ }
}
- if (asMD5 && md != null) {
+
+ if (md == null) {
+ w.write(readBuffer, 0, bytesRead);
+ } else {
md.update(readBuffer, 0, bytesRead);
totalBytes += bytesRead;
- } else {
- w.write(readBuffer, 0, bytesRead);
}
}
+
if (first) { // Bug 46838 - if there was no data, still need to set latency
sampleResult.latencyEnd();
+ return new byte[0];
}
- in.close();
- w.flush();
- if (asMD5 && md != null) {
+
+ if (md != null) {
byte[] md5Result = md.digest();
- w.write(JOrphanUtils.baToHexBytes(md5Result));
sampleResult.setBytes(totalBytes);
+ return JOrphanUtils.baToHexBytes(md5Result);
}
- w.close();
- return w.toByteArray();
+
+ return toByteArray(w);
} finally {
IOUtils.closeQuietly(in);
+ IOUtils.closeQuietly(w);
}
}
/**
+ * Optimized method to get byte array from {@link OutputStream}
+ * @param w {@link OutputStream}
+ * @return byte array
+ */
+ private byte[] toByteArray(OutputStream w) {
+ if(w instanceof DirectAccessByteArrayOutputStream) {
+ return ((DirectAccessByteArrayOutputStream) w).toByteArray();
+ }
+
+ if(w instanceof org.apache.commons.io.output.ByteArrayOutputStream) {
+ return ((org.apache.commons.io.output.ByteArrayOutputStream) w).toByteArray();
+ }
+
+ log.warn("Unknown stream type " + w.getClass());
+
+ return null;
+ }
+
+ /**
* JMeter 2.3.1 and earlier only had fields for one file on the GUI:
* <ul>
* <li>FILE_NAME</li>
Added: jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/util/DirectAccessByteArrayOutputStream.java
URL: http://svn.apache.org/viewvc/jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/util/DirectAccessByteArrayOutputStream.java?rev=1754663&view=auto
==============================================================================
--- jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/util/DirectAccessByteArrayOutputStream.java (added)
+++ jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/util/DirectAccessByteArrayOutputStream.java Sun Jul 31 14:39:30 2016
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jmeter.protocol.http.util;
+
+import java.io.ByteArrayOutputStream;
+import java.util.Arrays;
+
+/**
+ * this is a non thread-safe specialization of java {@link ByteArrayOutputStream}
+ * it returns the internal buffer if its size matches the byte count
+ *
+ * @since 3.1
+ */
+public class DirectAccessByteArrayOutputStream extends ByteArrayOutputStream {
+
+ public DirectAccessByteArrayOutputStream(int initialSize) {
+ super(initialSize);
+ }
+
+ @SuppressWarnings("sync-override")
+ @Override
+ public byte[] toByteArray() {
+ // no need to copy the buffer if it has the right size
+ // avoid an unneeded memory allocation
+ if(this.count == this.buf.length) {
+ return this.buf;
+ }
+
+ return Arrays.copyOf(buf, count);
+ }
+
+}
Propchange: jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/util/DirectAccessByteArrayOutputStream.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: jmeter/trunk/xdocs/changes.xml
URL: http://svn.apache.org/viewvc/jmeter/trunk/xdocs/changes.xml?rev=1754663&r1=1754662&r2=1754663&view=diff
==============================================================================
--- jmeter/trunk/xdocs/changes.xml [utf-8] (original)
+++ jmeter/trunk/xdocs/changes.xml [utf-8] Sun Jul 31 14:39:30 2016
@@ -79,7 +79,7 @@ Summary
<h3>HTTP Samplers and Test Script Recorder</h3>
<ul>
- <li><bug>XXXXX</bug>Sample Bugzilla title</li>
+ <li><bug>59882</bug>Reduce memory allocations for better throughput. Contributed by Benoit Wiart (b.wiart at ubik-ingenierie.com) through <pr>217</pr></li>
</ul>
<h3>Other samplers</h3>