You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jmeter.apache.org by fs...@apache.org on 2017/07/13 18:23:05 UTC
svn commit: r1801859 - in /jmeter/trunk:
src/protocol/http/org/apache/jmeter/protocol/http/control/
src/protocol/http/org/apache/jmeter/protocol/http/sampler/
test/src/org/apache/jmeter/protocol/http/control/ xdocs/ xdocs/usermanual/
Author: fschumacher
Date: Thu Jul 13 18:23:05 2017
New Revision: 1801859
URL: http://svn.apache.org/viewvc?rev=1801859&view=rev
Log:
Implement caching of requests with "vary" header.
Bugzilla Id: 61176 (closes #298 on github)
Modified:
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/control/CacheManager.java
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPJavaImpl.java
jmeter/trunk/test/src/org/apache/jmeter/protocol/http/control/TestCacheManagerBase.java
jmeter/trunk/xdocs/changes.xml
jmeter/trunk/xdocs/usermanual/component_reference.xml
Modified: jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/control/CacheManager.java
URL: http://svn.apache.org/viewvc/jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/control/CacheManager.java?rev=1801859&r1=1801858&r2=1801859&view=diff
==============================================================================
--- jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/control/CacheManager.java (original)
+++ jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/control/CacheManager.java Thu Jul 13 18:23:05 2017
@@ -22,16 +22,26 @@ import java.io.Serializable;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.collections.map.LRUMap;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.http.Header;
+import org.apache.http.HeaderElement;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.utils.DateUtils;
+import org.apache.http.message.BasicHeader;
import org.apache.jmeter.config.ConfigTestElement;
import org.apache.jmeter.engine.event.LoopIterationEvent;
import org.apache.jmeter.protocol.http.sampler.HTTPSampleResult;
@@ -100,11 +110,28 @@ public class CacheManager extends Config
private final String lastModified;
private final String etag;
private final Date expires;
+ private final String varyHeader;
+ /**
+ *
+ * @param lastModified
+ * @param expires
+ * @param etag
+ * @deprecated use {@link CacheEntry(String lastModified, Date expires, String etag, String varyHeader)} instead
+ */
+ @Deprecated
public CacheEntry(String lastModified, Date expires, String etag) {
this.lastModified = lastModified;
this.etag = etag;
this.expires = expires;
+ this.varyHeader = null;
+ }
+
+ public CacheEntry(String lastModified, Date expires, String etag, String varyHeader) {
+ this.lastModified = lastModified;
+ this.etag = etag;
+ this.expires = expires;
+ this.varyHeader = varyHeader;
}
public String getLastModified() {
@@ -117,12 +144,16 @@ public class CacheManager extends Config
@Override
public String toString() {
- return lastModified + " " + etag;
+ return lastModified + " " + etag + " " + varyHeader;
}
public Date getExpires() {
return expires;
}
+
+ public String getVaryHeader() {
+ return varyHeader;
+ }
}
/**
@@ -132,19 +163,34 @@ public class CacheManager extends Config
* @param res result
*/
public void saveDetails(URLConnection conn, HTTPSampleResult res){
- if (isCacheable(res) && !hasVaryHeader(conn)){
+ final String varyHeader = conn.getHeaderField(HTTPConstants.VARY);
+ if (isCacheable(res, varyHeader)){
String lastModified = conn.getHeaderField(HTTPConstants.LAST_MODIFIED);
String expires = conn.getHeaderField(HTTPConstants.EXPIRES);
String etag = conn.getHeaderField(HTTPConstants.ETAG);
String url = conn.getURL().toString();
String cacheControl = conn.getHeaderField(HTTPConstants.CACHE_CONTROL);
String date = conn.getHeaderField(HTTPConstants.DATE);
- setCache(lastModified, cacheControl, expires, etag, url, date);
+ setCache(lastModified, cacheControl, expires, etag, url, date, getVaryHeader(varyHeader, asHeaders(res.getRequestHeaders())));
}
}
- private boolean hasVaryHeader(URLConnection conn) {
- return conn.getHeaderField(HTTPConstants.VARY) != null;
+ private Pair<String, String> getVaryHeader(String headerName, Header[] reqHeaders) {
+ if (headerName == null) {
+ return null;
+ }
+ final Set<String> names = new HashSet<>(Arrays.asList(headerName.split(",\\s*")));
+ final Map<String, List<String>> values = new HashMap<>();
+ for (final String name: names) {
+ values.put(name, new ArrayList<String>());
+ }
+ for (Header header: reqHeaders) {
+ if (names.contains(header.getName())) {
+ log.debug("Found vary value {} for {} in response", header, headerName);
+ values.get(header.getName()).add(header.getValue());
+ }
+ }
+ return new ImmutablePair<>(headerName, values.toString());
}
/**
@@ -157,22 +203,19 @@ public class CacheManager extends Config
* result to decide if result is cacheable
*/
public void saveDetails(HttpResponse method, HTTPSampleResult res) {
- if (isCacheable(res) && !hasVaryHeader(method)){
+ final String varyHeader = getHeader(method, HTTPConstants.VARY);
+ if (isCacheable(res, varyHeader)){
String lastModified = getHeader(method ,HTTPConstants.LAST_MODIFIED);
String expires = getHeader(method ,HTTPConstants.EXPIRES);
String etag = getHeader(method ,HTTPConstants.ETAG);
String cacheControl = getHeader(method, HTTPConstants.CACHE_CONTROL);
String date = getHeader(method, HTTPConstants.DATE);
- setCache(lastModified, cacheControl, expires, etag, res.getUrlAsString(), date); // TODO correct URL?
+ setCache(lastModified, cacheControl, expires, etag, res.getUrlAsString(), date, getVaryHeader(varyHeader, asHeaders(res.getRequestHeaders()))); // TODO correct URL?
}
}
- private boolean hasVaryHeader(HttpResponse method) {
- return getHeader(method, HTTPConstants.VARY) != null;
- }
-
// helper method to save the cache entry
- private void setCache(String lastModified, String cacheControl, String expires, String etag, String url, String date) {
+ private void setCache(String lastModified, String cacheControl, String expires, String etag, String url, String date, Pair<String, String> varyHeader) {
if (log.isDebugEnabled()){
log.debug("setCache("
+ lastModified + ","
@@ -202,7 +245,17 @@ public class CacheManager extends Config
// else expiresDate computed in (expires!=null) condition is used
}
}
- getCache().put(url, new CacheEntry(lastModified, expiresDate, etag));
+ if (varyHeader != null) {
+ if (log.isDebugEnabled()) {
+ log.debug("Set entry into cache for url {} and vary {} ({})", url,
+ varyHeader,
+ varyUrl(url, varyHeader.getLeft(), varyHeader.getRight()));
+ }
+ getCache().put(url, new CacheEntry(lastModified, expiresDate, etag, varyHeader.getLeft()));
+ getCache().put(varyUrl(url, varyHeader.getLeft(), varyHeader.getRight()), new CacheEntry(lastModified, expiresDate, etag, null));
+ } else {
+ getCache().put(url, new CacheEntry(lastModified, expiresDate, etag, null));
+ }
}
private Date extractExpiresDateFromExpires(String expires) {
@@ -280,7 +333,10 @@ public class CacheManager extends Config
* Is the sample result OK to cache?
* i.e is it in the 2xx range or equal to 304, and is it a cacheable method?
*/
- private boolean isCacheable(HTTPSampleResult res){
+ private boolean isCacheable(HTTPSampleResult res, String varyHeader){
+ if ("*".equals(varyHeader)) {
+ return false;
+ }
final String responseCode = res.getResponseCode();
return isCacheableMethod(res)
&& (("200".compareTo(responseCode) <= 0 // $NON-NLS-1$
@@ -309,7 +365,7 @@ public class CacheManager extends Config
* @param request where to set the headers
*/
public void setHeaders(URL url, HttpRequestBase request) {
- CacheEntry entry = getCache().get(url.toString());
+ CacheEntry entry = getEntry(url.toString(), request.getAllHeaders());
if (log.isDebugEnabled()){
log.debug("{}(OAH) {} {}", request.getMethod(), url.toString(), entry);
}
@@ -335,7 +391,7 @@ public class CacheManager extends Config
* @param conn where to set the headers
*/
public void setHeaders(HttpURLConnection conn, URL url) {
- CacheEntry entry = getCache().get(url.toString());
+ CacheEntry entry = getEntry(url.toString(), asHeaders(conn.getHeaderFields()));
if (log.isDebugEnabled()){
log.debug("{}(Java) {} {}", conn.getRequestMethod(), url.toString(), entry);
}
@@ -352,14 +408,86 @@ public class CacheManager extends Config
}
/**
- * Check the cache, if the entry has an expires header and the entry has not expired, return true<br>
- * @param url {@link URL} to look up in cache
- * @return <code>true</code> if entry has an expires header and the entry has not expired, else <code>false</code>
+ * Check the cache, if the entry has an expires header and the entry has not
+ * expired, return <code>true</code><br>
+ *
+ * @param url
+ * {@link URL} to look up in cache
+ * @return <code>true</code> if entry has an expires header and the entry
+ * has not expired, else <code>false</code>
+ * @deprecated use a version of {@link CacheManager#inCache(URL, Header[])}
+ * or
+ * {@link CacheManager#inCache(URL, org.apache.jmeter.protocol.http.control.Header[])}
*/
+ @Deprecated
public boolean inCache(URL url) {
- CacheEntry entry = getCache().get(url.toString());
- log.debug("inCache {} {}", url, entry);
- if (entry != null){
+ return entryStillValid(getEntry(url.toString(), null));
+ }
+
+ public boolean inCache(URL url, Header[] allHeaders) {
+ return entryStillValid(getEntry(url.toString(), allHeaders));
+ }
+
+ public boolean inCache(URL url, org.apache.jmeter.protocol.http.control.Header[] allHeaders) {
+ return entryStillValid(getEntry(url.toString(), asHeaders(allHeaders)));
+ }
+
+ private Header[] asHeaders(
+ org.apache.jmeter.protocol.http.control.Header[] allHeaders) {
+ final List<Header> result = new ArrayList<>(allHeaders.length);
+ for (org.apache.jmeter.protocol.http.control.Header header: allHeaders) {
+ result.add(new HeaderAdapter(header));
+ }
+ return result.toArray(new Header[result.size()]);
+ }
+
+ private Header[] asHeaders(String allHeaders) {
+ List<Header> result = new ArrayList<>();
+ for (String line: allHeaders.split("\\n")) {
+ String[] splitted = line.split(": ", 2);
+ if (splitted.length == 2) {
+ result.add(new BasicHeader(splitted[0], splitted[1]));
+ }
+ }
+ return result.toArray(new Header[result.size()]);
+ }
+
+ private Header[] asHeaders(Map<String, List<String>> headers) {
+ List<Header> result = new ArrayList<>(headers.size());
+ for (Map.Entry<String, List<String>> header: headers.entrySet()) {
+ new BasicHeader(header.getKey(), String.join(", ", header.getValue()));
+ }
+ return result.toArray(new Header[result.size()]);
+ }
+
+ private static class HeaderAdapter implements Header {
+
+ private final org.apache.jmeter.protocol.http.control.Header delegate;
+
+ public HeaderAdapter(org.apache.jmeter.protocol.http.control.Header delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public HeaderElement[] getElements() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String getName() {
+ return delegate.getName();
+ }
+
+ @Override
+ public String getValue() {
+ return delegate.getValue();
+ }
+
+ }
+
+ private boolean entryStillValid(CacheEntry entry) {
+ log.debug("Check if entry {} is still valid", entry);
+ if (entry != null && entry.getVaryHeader() == null) {
final Date expiresDate = entry.getExpires();
if (expiresDate != null) {
if (expiresDate.after(new Date())) {
@@ -373,8 +501,36 @@ public class CacheManager extends Config
return false;
}
+ private CacheEntry getEntry(String url, Header[] headers) {
+ CacheEntry entry = getCache().get(url);
+ log.debug("Cache: {}", getCache());
+ log.debug("inCache {} {} {}", url, entry, headers);
+ if (entry == null) {
+ log.debug("No entry found for url {}", url);
+ return null;
+ }
+ if (entry.getVaryHeader() == null) {
+ log.debug("Entry {} with no vary found for url {}", entry, url);
+ return entry;
+ }
+ if (headers == null) {
+ log.debug("Entry {} found, but it should depend on vary {} for url {}", entry, entry.getVaryHeader(), url);
+ return null;
+ }
+ Pair<String, String> varyPair = getVaryHeader(entry.getVaryHeader(), headers);
+ if (varyPair != null) {
+ log.debug("Looking again for {} because of {} with vary: {} ({})", url, entry, entry.getVaryHeader(), varyPair);
+ return getEntry(varyUrl(url, entry.getVaryHeader(), varyPair.getRight()), null);
+ }
+ return null;
+ }
+
+ private String varyUrl(String url, String headerName, String headerValue) {
+ return "vary-" + headerName + "-" + headerValue + "-" + url;
+ }
+
private Map<String, CacheEntry> getCache() {
- return localCache != null?localCache:threadCache.get();
+ return localCache != null ? localCache : threadCache.get();
}
public boolean getClearEachIteration() {
Modified: jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java
URL: http://svn.apache.org/viewvc/jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java?rev=1801859&r1=1801858&r2=1801859&view=diff
==============================================================================
--- jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java (original)
+++ jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java Thu Jul 13 18:23:05 2017
@@ -440,7 +440,7 @@ public class HTTPHC4Impl extends HTTPHCA
res.sampleStart();
final CacheManager cacheManager = getCacheManager();
- if (cacheManager != null && HTTPConstants.GET.equalsIgnoreCase(method) && cacheManager.inCache(url)) {
+ if (cacheManager != null && HTTPConstants.GET.equalsIgnoreCase(method) && cacheManager.inCache(url, httpRequest.getAllHeaders())) {
return updateSampleResultForResourceInCache(res);
}
CloseableHttpResponse httpResponse = null;
Modified: jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPJavaImpl.java
URL: http://svn.apache.org/viewvc/jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPJavaImpl.java?rev=1801859&r1=1801858&r2=1801859&view=diff
==============================================================================
--- jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPJavaImpl.java (original)
+++ jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPJavaImpl.java Thu Jul 13 18:23:05 2017
@@ -26,6 +26,7 @@ import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.URL;
import java.net.URLConnection;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -484,7 +485,7 @@ public class HTTPJavaImpl extends HTTPAb
// Check cache for an entry with an Expires header in the future
final CacheManager cacheManager = getCacheManager();
if (cacheManager != null && HTTPConstants.GET.equalsIgnoreCase(method)) {
- if (cacheManager.inCache(url)) {
+ if (cacheManager.inCache(url, getHeaders(getHeaderManager()))) {
return updateSampleResultForResourceInCache(res);
}
}
@@ -626,6 +627,20 @@ public class HTTPJavaImpl extends HTTPAb
}
}
+ private Header[] getHeaders(HeaderManager headerManager) {
+ if (headerManager == null) {
+ return new Header[0];
+ }
+ final List<Header> allHeaders = new ArrayList<>();
+ final CollectionProperty headers = headerManager.getHeaders();
+ if (headers != null) {
+ for (final JMeterProperty jMeterProperty : headers) {
+ allHeaders.add((Header) jMeterProperty.getObjectValue());
+ }
+ }
+ return allHeaders.toArray(new Header[allHeaders.size()]);
+ }
+
protected void disconnect(HttpURLConnection conn) {
if (conn != null) {
String connection = conn.getHeaderField(HTTPConstants.HEADER_CONNECTION);
Modified: jmeter/trunk/test/src/org/apache/jmeter/protocol/http/control/TestCacheManagerBase.java
URL: http://svn.apache.org/viewvc/jmeter/trunk/test/src/org/apache/jmeter/protocol/http/control/TestCacheManagerBase.java?rev=1801859&r1=1801858&r2=1801859&view=diff
==============================================================================
--- jmeter/trunk/test/src/org/apache/jmeter/protocol/http/control/TestCacheManagerBase.java (original)
+++ jmeter/trunk/test/src/org/apache/jmeter/protocol/http/control/TestCacheManagerBase.java Thu Jul 13 18:23:05 2017
@@ -18,6 +18,12 @@
package org.apache.jmeter.protocol.http.control;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
import java.lang.reflect.Field;
import java.net.URL;
import java.text.SimpleDateFormat;
@@ -33,8 +39,6 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.*;
-
public abstract class TestCacheManagerBase extends JMeterTestCase {
protected static final String LOCAL_HOST = "http://localhost/";
protected static final String EXPECTED_ETAG = "0xCAFEBABEDEADBEEF";
@@ -157,25 +161,75 @@ public abstract class TestCacheManagerBa
@Test
public void testCacheVarySomething() throws Exception {
- testCacheVary("Something");
+ String varyHeader = "Something";
+ testCacheVary(varyHeader, new Header[] { new Header(varyHeader, "value") },
+ new Header[] {
+ new Header(varyHeader, "something completely different") });
}
@Test
public void testCacheVaryAcceptEncoding() throws Exception {
- testCacheVary("Accept-Encoding");
+ String varyHeader = "Accept-Encoding";
+ testCacheVary(varyHeader,
+ new Header[] { new Header(varyHeader, "value") }, new Header[] {
+ new Header(varyHeader, "something completely different") });
+ }
+
+ @Test
+ public void testCacheMultiValueVaryHeaders() throws Exception {
+ String varyHeader = "Accept-Encoding";
+ testCacheVary(varyHeader,
+ new Header[] { new Header(varyHeader, "value"),
+ new Header(varyHeader, "another value") },
+ new Header[] { new Header(varyHeader,
+ "something completely different") });
}
- private void testCacheVary(String vary) throws Exception {
+ @Test
+ public void testCacheMultipleVaryHeaders() throws Exception {
+ String varyHeaderOne = "Accept-Encoding";
+ String varyHeaderTwo = "Something";
+ testCacheVary(varyHeaderOne + "," + varyHeaderTwo,
+ new Header[] { new Header(varyHeaderOne, "first value"),
+ new Header(varyHeaderTwo, "another value") },
+ new Header[] { new Header(varyHeaderOne,
+ "first") });
+ }
+
+ @Test
+ public void testCacheMultipleMultiVaryHeaders() throws Exception {
+ String varyHeaderOne = "Accept-Encoding";
+ String varyHeaderTwo = "Something";
+ testCacheVary(varyHeaderOne + "," + varyHeaderTwo,
+ new Header[] { new Header(varyHeaderOne, "first value"),
+ new Header(varyHeaderOne, "second value"),
+ new Header(varyHeaderTwo, "another value") },
+ new Header[] { new Header(varyHeaderOne, "first value"),
+ new Header(varyHeaderOne, "another value") });
+ }
+
+ private String asString(Header[] headers) {
+ StringBuilder result = new StringBuilder();
+ for (Header header: headers) {
+ result.append(header.getName()).append(": ").append(header.getValue()).append("\n");
+ }
+ return result.toString();
+ }
+
+ private void testCacheVary(String vary, Header[] origHeaders, Header[] differentHeaders) throws Exception {
this.cacheManager.setUseExpires(true);
this.cacheManager.testIterationStart(null);
assertNull("Should not find entry", getThreadCacheEntry(LOCAL_HOST));
- assertFalse("Should not find valid entry", this.cacheManager.inCache(url));
+ assertFalse("Should not find valid entry", this.cacheManager.inCache(url, origHeaders));
setExpires(makeDate(new Date(System.currentTimeMillis())));
setCacheControl("public, max-age=5");
+ sampleResultOK.setRequestHeaders(asString(origHeaders));
this.vary = vary;
cacheResult(sampleResultOK);
- assertNull("Should not find entry", getThreadCacheEntry(LOCAL_HOST));
- assertFalse("Should not find valid entry", this.cacheManager.inCache(url));
+ assertNotNull("Should find entry with vary header", getThreadCacheEntry(LOCAL_HOST).getVaryHeader());
+ assertFalse("Should not find valid entry without headers", this.cacheManager.inCache(url));
+ assertTrue("Should find valid entry with headers", this.cacheManager.inCache(url, origHeaders));
+ assertFalse("Should not find valid entry with different header", this.cacheManager.inCache(url, differentHeaders));
this.vary = null;
}
Modified: jmeter/trunk/xdocs/changes.xml
URL: http://svn.apache.org/viewvc/jmeter/trunk/xdocs/changes.xml?rev=1801859&r1=1801858&r2=1801859&view=diff
==============================================================================
--- jmeter/trunk/xdocs/changes.xml [utf-8] (original)
+++ jmeter/trunk/xdocs/changes.xml [utf-8] Thu Jul 13 18:23:05 2017
@@ -109,6 +109,7 @@ Summary
<h3>Timers, Assertions, Config, Pre- & Post-Processors</h3>
<ul>
+ <li><bug>61176</bug><pr>298</pr>Cache responses that have <code>vary</code> header in the <code>CacheManager</code>.</li>
</ul>
<h3>Functions</h3>
Modified: jmeter/trunk/xdocs/usermanual/component_reference.xml
URL: http://svn.apache.org/viewvc/jmeter/trunk/xdocs/usermanual/component_reference.xml?rev=1801859&r1=1801858&r2=1801859&view=diff
==============================================================================
--- jmeter/trunk/xdocs/usermanual/component_reference.xml (original)
+++ jmeter/trunk/xdocs/usermanual/component_reference.xml Thu Jul 13 18:23:05 2017
@@ -3805,7 +3805,6 @@ If the requested document has not change
Likewise if the <code>Expires</code> date is in the future.
This may cause problems for Assertions.
</note>
-<note>Responses with a <code>Vary</code> header will not be cached.</note>
</description>
<properties>
<property name="Name" required="No">Descriptive name for this element that is shown in the tree. </property>