You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2014/12/01 15:31:22 UTC
svn commit: r1642691 - in /lucene/dev/branches/branch_5x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/core/
solr/core/src/java/org/apache/solr/store/blockcache/
Author: markrmiller
Date: Mon Dec 1 14:31:22 2014
New Revision: 1642691
URL: http://svn.apache.org/r1642691
Log:
SOLR-6752: Buffer Cache allocate/lost metrics should be exposed.
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/blockcache/BufferStore.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1642691&r1=1642690&r2=1642691&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Mon Dec 1 14:31:22 2014
@@ -419,6 +419,9 @@ Other Changes
* SOLR-6370: Allow tests to report/fail on many ZK watches being parallelly
requested on the same data (Ramkumar Aiyengar via Timothy Potter)
+* SOLR-6752: Buffer Cache allocate/lost metrics should be exposed.
+ (Mike Drob via Mark Miller)
+
================== 4.10.3 ==================
Bug Fixes
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java?rev=1642691&r1=1642690&r2=1642691&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java Mon Dec 1 14:31:22 2014
@@ -209,7 +209,7 @@ public class HdfsDirectoryFactory extend
private BlockCache createBlockCache(int numberOfBlocksPerBank, int blockSize,
int bankCount, boolean directAllocation, int slabSize, int bufferSize,
int bufferCount) {
- BufferStore.initNewBuffer(bufferSize, bufferCount);
+ BufferStore.initNewBuffer(bufferSize, bufferCount, metrics);
long totalMemory = (long) bankCount * (long) numberOfBlocksPerBank
* (long) blockSize;
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/blockcache/BufferStore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/blockcache/BufferStore.java?rev=1642691&r1=1642690&r2=1642691&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/blockcache/BufferStore.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/blockcache/BufferStore.java Mon Dec 1 14:31:22 2014
@@ -21,6 +21,7 @@ import java.util.concurrent.ArrayBlockin
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLong;
/**
* @lucene.experimental
@@ -45,7 +46,14 @@ public class BufferStore implements Stor
private final int bufferSize;
+ private final AtomicLong shardBuffercacheAllocate;
+ private final AtomicLong shardBuffercacheLost;
+
public synchronized static void initNewBuffer(int bufferSize, long totalAmount) {
+ initNewBuffer(bufferSize, totalAmount, null);
+ }
+
+ public synchronized static void initNewBuffer(int bufferSize, long totalAmount, Metrics metrics) {
if (totalAmount == 0) {
return;
}
@@ -55,13 +63,21 @@ public class BufferStore implements Stor
if (count > Integer.MAX_VALUE) {
count = Integer.MAX_VALUE;
}
- BufferStore store = new BufferStore(bufferSize, (int) count);
+ AtomicLong shardBuffercacheLost = new AtomicLong(0);
+ AtomicLong shardBuffercacheAllocate = new AtomicLong(0);
+ if (metrics != null) {
+ shardBuffercacheLost = metrics.shardBuffercacheLost;
+ shardBuffercacheAllocate = metrics.shardBuffercacheAllocate;
+ }
+ BufferStore store = new BufferStore(bufferSize, (int) count, shardBuffercacheAllocate, shardBuffercacheLost);
bufferStores.put(bufferSize, store);
}
}
- private BufferStore(int bufferSize, int count) {
+ private BufferStore(int bufferSize, int count, AtomicLong shardBuffercacheAllocate, AtomicLong shardBuffercacheLost) {
this.bufferSize = bufferSize;
+ this.shardBuffercacheAllocate = shardBuffercacheAllocate;
+ this.shardBuffercacheLost = shardBuffercacheLost;
buffers = setupBuffers(bufferSize, count);
}
@@ -102,14 +118,17 @@ public class BufferStore implements Stor
checkReturn(buffers.offer(buffer));
}
- private void checkReturn(boolean offer) {
-
+ private void checkReturn(boolean accepted) {
+ if (!accepted) {
+ shardBuffercacheLost.incrementAndGet();
+ }
}
private byte[] newBuffer(byte[] buf) {
if (buf != null) {
return buf;
}
+ shardBuffercacheAllocate.incrementAndGet();
return new byte[bufferSize];
}
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java?rev=1642691&r1=1642690&r2=1642691&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java Mon Dec 1 14:31:22 2014
@@ -49,9 +49,7 @@ public class Metrics implements Updater
public AtomicLong recordWrites = new AtomicLong(0);
public AtomicLong queriesExternal = new AtomicLong(0);
public AtomicLong queriesInternal = new AtomicLong(0);
- public AtomicLong shardBuffercacheAllocate1024 = new AtomicLong(0);
- public AtomicLong shardBuffercacheAllocate8192 = new AtomicLong(0);
- public AtomicLong shardBuffercacheAllocateOther = new AtomicLong(0);
+ public AtomicLong shardBuffercacheAllocate = new AtomicLong(0);
public AtomicLong shardBuffercacheLost = new AtomicLong(0);
public Map<String,MethodCall> methodCalls = new ConcurrentHashMap<>();
@@ -101,6 +99,8 @@ public class Metrics implements Updater
metricsRecord.setMetric("record.writes", getPerSecond(recordWrites.getAndSet(0), seconds));
metricsRecord.setMetric("query.external", getPerSecond(queriesExternal.getAndSet(0), seconds));
metricsRecord.setMetric("query.internal", getPerSecond(queriesInternal.getAndSet(0), seconds));
+ metricsRecord.setMetric("buffercache.allocations", getPerSecond(shardBuffercacheAllocate.getAndSet(0), seconds));
+ metricsRecord.setMetric("buffercache.lost", getPerSecond(shardBuffercacheLost.getAndSet(0), seconds));
for (Entry<String,MethodCall> entry : methodCalls.entrySet()) {
String key = entry.getKey();
MethodCall value = entry.getValue();