You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 04:22:17 UTC
svn commit: r1181579 - in
/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase: io/hfile/
regionserver/ regionserver/metrics/
Author: nspiegelberg
Date: Tue Oct 11 02:22:16 2011
New Revision: 1181579
URL: http://svn.apache.org/viewvc?rev=1181579&view=rev
Log:
Per CF stats and per blocktype stats for memstore size, cache size, etc
Summary:
Added stats for the following, per column family:
1) memstore size
2) storefile index size
3) storefile size
4) bytes flushed
5) # of storefiles
6) Block Cache usage by Block Type (index blocks vs. bloom blocks vs. data blocks)
7) Block Cache usage by Column Family
8) Total Size of Blooms by Column Family
Number 4 was removed by accident but readded. Also, stopped use of reset() persistent metrics in favor of non-persistent ones that are deleted when read. Testing is now underway; code should be stable pending discovery of unexpected behavior.
Task ID: #530815
Blame Rev:
Reviewers: kannan
CC:
Test Plan: Currently I am testing by editing the metrics configuration file to export into a file and reading the metrics there (they should be prefixed by a column family name). The better way to test is on a dev cluster using JMX, although hopefully both should yield the same information. Consistency checks should be made that, ie the number of files for each column family adds up to the total. If discrepancies are found they should be investigated.
Perhaps some performance tests should be done, and or the performance ramifications of the code should be thought about more thoroughly.
Revert Plan: Just revert the codebase and restart.
Tags:
Differential Revision: 270111
Added:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockInfo.java
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java?rev=1181579&r1=1181578&r2=1181579&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java Tue Oct 11 02:22:16 2011
@@ -26,6 +26,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -43,6 +45,8 @@ import org.apache.hadoop.io.Writable;
*/
public abstract class AbstractHFileWriter implements HFile.Writer {
+ private static final Log LOG = LogFactory.getLog(AbstractHFileWriter.class);
+
/** Key previously appended. Becomes the last key in the file. */
protected byte[] lastKeyBuffer = null;
@@ -91,6 +95,13 @@ public abstract class AbstractHFileWrite
/** May be null if we were passed a stream. */
protected final Path path;
+ // table qualified cfName for this HFile.
+ // This is used to report stats on a per-table/CF basis
+
+ // Note that this is gotten from the path, which can be null, so this can
+ // remain unknown
+ public String cfName = "cf.unknown";
+
/** Whether to cache key/value data blocks on write */
protected final boolean cacheDataBlocksOnWrite;
@@ -131,6 +142,20 @@ public abstract class AbstractHFileWrite
if (cacheDataBlocksOnWrite || cacheIndexBlocksOnWrite)
initBlockCache();
+
+ if (path != null)
+ parsePath(path.toString());
+ }
+
+ public void parsePath(String path) {
+ String splits[] = path.split("/");
+ if (splits.length < 2) {
+ LOG.warn("Could not determine the table and column family of the " +
+ "HFile path " + path);
+ return;
+ }
+
+ cfName = "cf." + splits[splits.length - 2];
}
/**
@@ -238,6 +263,11 @@ public abstract class AbstractHFileWrite
}
@Override
+ public String getColumnFamilyName() {
+ return cfName;
+ }
+
+ @Override
public String toString() {
return "writer=" + (path != null ? path.toString() : null) + ", name="
+ name + ", compression=" + compressAlgo.getName();
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java?rev=1181579&r1=1181578&r2=1181579&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java Tue Oct 11 02:22:16 2011
@@ -37,49 +37,55 @@ public enum BlockType {
// Scanned block section
/** Data block, both versions */
- DATA("DATABLK*"),
+ DATA("DATABLK*", BlockCategory.DATA),
/** Version 2 leaf index block. Appears in the data block section */
- LEAF_INDEX("IDXLEAF2"),
+ LEAF_INDEX("IDXLEAF2", BlockCategory.INDEX),
/** Bloom filter block, version 2 */
- BLOOM_CHUNK("BLMFBLK2"),
+ BLOOM_CHUNK("BLMFBLK2", BlockCategory.BLOOM),
// Non-scanned block section
/** Meta blocks */
- META("METABLKc"),
+ META("METABLKc", BlockCategory.META),
/** Intermediate-level version 2 index in the non-data block section */
- INTERMEDIATE_INDEX("IDXINTE2"),
+ INTERMEDIATE_INDEX("IDXINTE2", BlockCategory.INDEX),
// Load-on-open section.
/** Root index block, also used for the single-level meta index, version 2 */
- ROOT_INDEX("IDXROOT2"),
+ ROOT_INDEX("IDXROOT2", BlockCategory.INDEX),
/** File info, version 2 */
- FILE_INFO("FILEINF2"),
+ FILE_INFO("FILEINF2", BlockCategory.META),
/** Bloom filter metadata, version 2 */
- BLOOM_META("BLMFMET2"),
+ BLOOM_META("BLMFMET2", BlockCategory.BLOOM),
// Trailer
/** Fixed file trailer, both versions (always just a magic string) */
- TRAILER("TRABLK\"$"),
+ TRAILER("TRABLK\"$", BlockCategory.META),
// Legacy blocks
/** Block index magic string in version 1 */
- INDEX_V1("IDXBLK)+");
+ INDEX_V1("IDXBLK)+", BlockCategory.INDEX);
+
+ public enum BlockCategory {
+ DATA, META, INDEX, BLOOM
+ }
public static final int MAGIC_LENGTH = 8;
private final byte[] magic;
+ private final BlockCategory metricCat;
- private BlockType(String magicStr) {
+ private BlockType(String magicStr, BlockCategory metricCat) {
magic = magicStr.getBytes();
+ this.metricCat = metricCat;
assert magic.length == MAGIC_LENGTH;
}
@@ -95,6 +101,10 @@ public enum BlockType {
buf.put(magic);
}
+ public String getMetricName(){
+ return metricCat.toString();
+ }
+
public static BlockType parse(byte[] buf, int offset, int length)
throws IOException {
if (length != MAGIC_LENGTH) {
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1181579&r1=1181578&r2=1181579&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Tue Oct 11 02:22:16 2011
@@ -206,6 +206,8 @@ public class HFile {
/** @return the path to this {@link HFile} */
Path getPath();
+ String getColumnFamilyName();
+
void appendMetaBlock(String bloomFilterMetaKey, Writable metaWriter);
/**
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java?rev=1181579&r1=1181578&r2=1181579&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java Tue Oct 11 02:22:16 2011
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.hbase.io.DoubleOutputStream;
import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.hfile.HFileBlockInfo;
import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
@@ -71,7 +72,7 @@ import static org.apache.hadoop.hbase.io
* The version 2 block representation in the block cache is the same as above,
* except that the data section is always uncompressed in the cache.
*/
-public class HFileBlock implements HeapSize {
+public class HFileBlock implements HeapSize, HFileBlockInfo {
/** The size of a version 2 {@link HFile} block header */
public static final int HEADER_SIZE = MAGIC_LENGTH + 2 * Bytes.SIZEOF_INT
@@ -134,6 +135,16 @@ public class HFileBlock implements HeapS
this.offset = offset;
}
+ private String cfName = "cf.unknown";
+
+ public String getColumnFamilyName() {
+ return this.cfName;
+ }
+
+ public void setColumnFamilyName(String cfName) {
+ this.cfName = cfName;
+ }
+
/**
* Creates a block from an existing buffer starting with a header. Rewinds
* and takes ownership of the buffer. By definition of rewind, ignores the
@@ -1419,5 +1430,4 @@ public class HFileBlock implements HeapS
public int getNextBlockOnDiskSizeWithHeader() {
return nextBlockOnDiskSizeWithHeader;
}
-
}
Added: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockInfo.java?rev=1181579&view=auto
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockInfo.java (added)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockInfo.java Tue Oct 11 02:22:16 2011
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import org.apache.hadoop.hbase.io.hfile.BlockType;
+import org.apache.hadoop.hbase.io.hfile.HFileBlock.Writer;
+import org.apache.hadoop.hbase.io.HeapSize;
+
+/**
+ * An interface that exposes methods to retrieve the column type and BlockType
+ * of a particular cached block. This is more information than that which is
+ * required by most cache implementations, but is used for more specific
+ * metrics, for example. Used by implementations of HeapSize, such as
+ * {@link HFileBlock}
+ */
+public interface HFileBlockInfo {
+ /**
+ * @return Column family name of this cached item.
+ */
+ public String getColumnFamilyName();
+
+ /**
+ * @return BlockType descriptor of this cached item. Indicates the type of
+ * data, such as a data block or an index one.
+ */
+ public BlockType getBlockType();
+}
\ No newline at end of file
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java?rev=1181579&r1=1181578&r2=1181579&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java Tue Oct 11 02:22:16 2011
@@ -235,6 +235,7 @@ public class HFileReaderV1 extends Abstr
HFileBlock hfileBlock = fsBlockReader.readBlockData(offset,
nextOffset - offset, metaBlockIndexReader.getRootBlockDataSize(block),
true);
+ hfileBlock.setColumnFamilyName(this.getColumnFamilyName());
hfileBlock.expectType(BlockType.META);
long delta = System.currentTimeMillis() - now;
@@ -322,6 +323,7 @@ public class HFileReaderV1 extends Abstr
HFileBlock hfileBlock = fsBlockReader.readBlockData(offset, nextOffset
- offset, dataBlockIndexReader.getRootBlockDataSize(block), pread);
+ hfileBlock.setColumnFamilyName(this.getColumnFamilyName());
hfileBlock.expectType(BlockType.DATA);
ByteBuffer buf = hfileBlock.getBufferWithoutHeader();
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java?rev=1181579&r1=1181578&r2=1181579&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java Tue Oct 11 02:22:16 2011
@@ -189,7 +189,7 @@ public class HFileReaderV2 extends Abstr
HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,
blockSize, -1, true);
-
+ metaBlock.setColumnFamilyName(this.getColumnFamilyName());
long delta = System.currentTimeMillis() - now;
HRegion.incrTimeVaryingMetric(fsReadTimeMetric, delta);
HFile.readTime += delta;
@@ -282,6 +282,7 @@ public class HFileReaderV2 extends Abstr
long now = System.currentTimeMillis();
HFileBlock dataBlock = fsBlockReader.readBlockData(dataBlockOffset,
onDiskBlockSize, -1, pread);
+ dataBlock.setColumnFamilyName(this.getColumnFamilyName());
long delta = System.currentTimeMillis() - now;
HFile.readTime += delta;
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java?rev=1181579&r1=1181578&r2=1181579&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java Tue Oct 11 02:22:16 2011
@@ -198,10 +198,11 @@ public class HFileWriterV1 extends Abstr
if (cacheDataBlocksOnWrite) {
baosDos.flush();
byte[] bytes = baos.toByteArray();
- blockCache.cacheBlock(HFile.getBlockCacheKey(name, blockBegin),
- new HFileBlock(BlockType.DATA,
- (int) (outputStream.getPos() - blockBegin), bytes.length, -1,
- ByteBuffer.wrap(bytes, 0, bytes.length), true, blockBegin));
+ HFileBlock cBlock = new HFileBlock(BlockType.DATA,
+ (int) (outputStream.getPos() - blockBegin), bytes.length, -1,
+ ByteBuffer.wrap(bytes, 0, bytes.length), true, blockBegin);
+ cBlock.setColumnFamilyName(this.getColumnFamilyName());
+ blockCache.cacheBlock(HFile.getBlockCacheKey(name, blockBegin),cBlock);
baosDos.close();
}
blockNumber++;
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java?rev=1181579&r1=1181578&r2=1181579&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java Tue Oct 11 02:22:16 2011
@@ -210,8 +210,10 @@ public class HFileWriterV2 extends Abstr
HFile.writeOps++;
if (cacheDataBlocksOnWrite) {
+ HFileBlock cBlock = fsBlockWriter.getBlockForCaching();
+ cBlock.setColumnFamilyName(this.getColumnFamilyName());
blockCache.cacheBlock(HFile.getBlockCacheKey(name, lastDataBlockOffset),
- fsBlockWriter.getBlockForCaching());
+ cBlock);
}
}
@@ -229,8 +231,10 @@ public class HFileWriterV2 extends Abstr
if (cacheThisBlock) {
// Cache this block on write.
+ HFileBlock cBlock = fsBlockWriter.getBlockForCaching();
+ cBlock.setColumnFamilyName(this.getColumnFamilyName());
blockCache.cacheBlock(HFile.getBlockCacheKey(name, offset),
- fsBlockWriter.getBlockForCaching());
+ cBlock);
}
}
}
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java?rev=1181579&r1=1181578&r2=1181579&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java Tue Oct 11 02:22:16 2011
@@ -30,7 +30,9 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.io.hfile.HFileBlockInfo;
import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
@@ -249,7 +251,7 @@ public class LruBlockCache implements Bl
throw new RuntimeException("Cached an already cached block");
}
cb = new CachedBlock(blockName, buf, count.incrementAndGet(), inMemory);
- long newSize = size.addAndGet(cb.heapSize());
+ long newSize = updateSizeMetrics(cb, false);
map.put(blockName, cb);
elements.incrementAndGet();
if(newSize > acceptableSize() && !evictionInProgress) {
@@ -272,6 +274,30 @@ public class LruBlockCache implements Bl
}
/**
+ * Helper function that updates the local size counter and also updates any
+ * per-cf or per-blocktype metrics it can discern from given
+ * {@link CachedBlock}
+ *
+ * @param cb
+ * @param evict
+ */
+ protected long updateSizeMetrics(CachedBlock cb, boolean evict) {
+ long heapsize = cb.heapSize();
+ if (evict) {
+ heapsize *= -1;
+ }
+ if (cb.getBuffer() instanceof HFileBlockInfo) {
+ HFileBlockInfo cb_hfbi = (HFileBlockInfo) cb.getBuffer();
+ HRegion.incrNumericPersistentMetric(cb_hfbi.getColumnFamilyName()
+ + ".blockCacheSize", heapsize);
+ HRegion.incrNumericPersistentMetric("bt."
+ + cb_hfbi.getBlockType().getMetricName() + ".blockCacheSize",
+ heapsize);
+ }
+ return size.addAndGet(heapsize);
+ }
+
+ /**
* Get the buffer of the block with the specified name.
* @param blockName block name
* @return buffer of specified block name, or null if not in cache
@@ -317,7 +343,7 @@ public class LruBlockCache implements Bl
protected long evictBlock(CachedBlock block) {
map.remove(block.getName());
- size.addAndGet(-1 * block.heapSize());
+ updateSizeMetrics(block, true);
elements.decrementAndGet();
stats.evicted();
return block.heapSize();
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1181579&r1=1181578&r2=1181579&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue Oct 11 02:22:16 2011
@@ -250,6 +250,11 @@ public class HRegion implements HeapSize
public static final ConcurrentMap<String, AtomicLong>
numericMetrics = new ConcurrentHashMap<String, AtomicLong>();
+ // for simple numeric metrics (current block cache size)
+ // These ones are not reset to zero when queried, unlike the previous.
+ public static final ConcurrentMap<String, AtomicLong>
+ numericPersistentMetrics = new ConcurrentHashMap<String, AtomicLong>();
+
// Used for metrics where we want track a metrics (such as latency)
// over a number of operations.
public static final ConcurrentMap<String,
@@ -315,11 +320,18 @@ public class HRegion implements HeapSize
oldVal.addAndGet(amount);
}
- static long getNumericMetric(String key) {
- AtomicLong m = numericMetrics.get(key);
- if (m == null)
- return 0;
- return m.get();
+ public static void setNumericMetric(String key, long amount) {
+ numericMetrics.put(key, new AtomicLong(amount));
+ }
+
+ public static void incrNumericPersistentMetric(String key, long amount) {
+ AtomicLong oldVal = numericPersistentMetrics.get(key);
+ if (oldVal == null) {
+ oldVal = numericPersistentMetrics.putIfAbsent(key, new AtomicLong(amount));
+ if (oldVal == null)
+ return;
+ }
+ oldVal.addAndGet(amount);
}
public static void incrTimeVaryingMetric(String key, long amount) {
@@ -335,6 +347,20 @@ public class HRegion implements HeapSize
oldVal.getSecond().incrementAndGet(); // increment ops by 1
}
+ static long getNumericMetric(String key) {
+ AtomicLong m = numericMetrics.get(key);
+ if (m == null)
+ return 0;
+ return m.get();
+ }
+
+ static long getNumericPersistentMetric(String key) {
+ AtomicLong m = numericPersistentMetrics.get(key);
+ if (m == null)
+ return 0;
+ return m.get();
+ }
+
public static final long getWriteOps() {
return writeOps.getAndSet(0);
}
@@ -1158,7 +1184,9 @@ public class HRegion implements HeapSize
// rows then)
this.updatesLock.writeLock().lock();
final long currentMemStoreSize = this.memstoreSize.get();
- List<StoreFlusher> storeFlushers = new ArrayList<StoreFlusher>(stores.size());
+ //copy the array of per column family memstore values
+ List<StoreFlusher> storeFlushers = new ArrayList<StoreFlusher>(
+ stores.size());
try {
sequenceId = (wal == null)? myseqid: wal.startCacheFlush();
completeSequenceId = this.getCompleteCacheFlushSequenceId(sequenceId);
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1181579&r1=1181578&r2=1181579&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Tue Oct 11 02:22:16 2011
@@ -31,11 +31,13 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.SortedMap;
@@ -55,6 +57,7 @@ import org.apache.commons.cli.CommandLin
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.commons.lang.mutable.MutableDouble;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -992,6 +995,24 @@ public class HRegionServer implements HR
}
}
+ /**
+ * Help function for metrics() that increments a map value if it exists.
+ *
+ * @param map
+ * The map to work with
+ * @param key
+ * the string key
+ * @param val
+ * the value to add or set the map key to
+ */
+ protected void incrMap(Map<String, MutableDouble> map, String key, double val) {
+ if (map.get(key) != null) {
+ map.get(key).add(val);
+ } else {
+ map.put(key, new MutableDouble(val));
+ }
+ }
+
protected void metrics() {
this.metrics.regions.set(this.onlineRegions.size());
this.metrics.incrementRequests(this.requestCount.get());
@@ -1004,6 +1025,21 @@ public class HRegionServer implements HR
long storefileIndexSize = 0;
long totalStaticIndexSize = 0;
long totalStaticBloomSize = 0;
+
+ long tmpfiles;
+ long tmpindex;
+ long tmpfilesize;
+ long tmpbloomsize;
+ long tmpstaticsize;
+ String cfname;
+
+ // Note that this is a map of Doubles instead of Longs. This is because we
+ // do effective integer division, which would perhaps truncate more than it
+ // should because we do it only on one part of our sum at a time. Rather
+ // than dividing at the end, where it is difficult to know the proper
+ // factor, everything is exact then truncated.
+ Map<String, MutableDouble> tempVals = new HashMap<String, MutableDouble>();
+
synchronized (this.onlineRegions) {
for (Map.Entry<Integer, HRegion> e: this.onlineRegions.entrySet()) {
HRegion r = e.getValue();
@@ -1012,14 +1048,37 @@ public class HRegionServer implements HR
stores += r.stores.size();
for(Map.Entry<byte [], Store> ee: r.stores.entrySet()) {
Store store = ee.getValue();
- storefiles += store.getStorefilesCount();
- storefileIndexSize += store.getStorefilesIndexSize();
- totalStaticIndexSize += store.getTotalStaticIndexSize();
- totalStaticBloomSize += store.getTotalStaticBloomSize();
+ tmpfiles = store.getStorefilesCount();
+ tmpindex = store.getStorefilesIndexSize();
+ tmpfilesize = store.getStorefilesSize();
+ tmpbloomsize = store.getTotalStaticBloomSize();
+ tmpstaticsize = store.getTotalStaticIndexSize();
+
+ // Note that there is only one store per CF so setting is safe
+ cfname = "cf." + store.toString();
+ this.incrMap(tempVals, cfname + ".storeFileCount", tmpfiles);
+ this.incrMap(tempVals, cfname + ".storeFileIndexSizeMB",
+ (tmpindex / (1024.0 * 1024)));
+ this.incrMap(tempVals, cfname + ".storeFileSizeMB",
+ (tmpfilesize / (1024.0 * 1024)));
+ this.incrMap(tempVals, cfname + ".staticBloomSizeKB",
+ (tmpbloomsize / 1024.0));
+ this.incrMap(tempVals, cfname + ".memstoreSizeMB",
+ (store.getMemStoreSize() / (1024.0 * 1024)));
+ this.incrMap(tempVals, cfname + ".staticIndexSizeKB",
+ tmpstaticsize / 1024.0);
+
+ storefiles += tmpfiles;
+ storefileIndexSize += tmpindex;
+ totalStaticIndexSize += tmpstaticsize;
+ totalStaticBloomSize += tmpbloomsize;
}
}
}
}
+ for (Entry<String, MutableDouble> e : tempVals.entrySet()) {
+ HRegion.setNumericMetric(e.getKey(), e.getValue().longValue());
+ }
this.metrics.stores.set(stores);
this.metrics.storefiles.set(storefiles);
this.metrics.memstoreSizeMB.set((int)(memstoreSize/(1024*1024)));
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1181579&r1=1181578&r2=1181579&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Tue Oct 11 02:22:16 2011
@@ -499,6 +499,13 @@ public class Store implements HeapSize {
this.conf, this.family.getBloomFilterType(), this.inMemory);
StoreFile.Reader r = sf.createReader();
this.storeSize += r.length();
+ // This increments the metrics associated with total flushed bytes for this
+ // family. The overall flush count is stored in the static metrics and
+ // retrieved from HRegion.recentFlushes, which is set within
+ // HRegion.internalFlushcache, which indirectly calls this to actually do
+ // the flushing through the StoreFlusherImpl class
+ HRegion.incrNumericPersistentMetric("cf." + this.toString() + ".flushSize",
+ flushed);
if(LOG.isInfoEnabled()) {
LOG.info("Added " + sf + ", entries=" + r.getEntries() +
", sequenceid=" + logCacheFlushId +
@@ -1548,6 +1555,13 @@ public class Store implements HeapSize {
}
/**
+ * @return The size of this store's memstore, in bytes
+ */
+ long getMemStoreSize() {
+ return this.memstore.heapSize();
+ }
+
+ /**
* @return The priority that this store should have in the compaction queue
*/
public int getCompactPriority() {
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1181579&r1=1181578&r2=1181579&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Tue Oct 11 02:22:16 2011
@@ -1197,6 +1197,10 @@ public class StoreFile {
return reader.indexSize();
}
+ public String getColumnFamilyName() {
+ return reader.getColumnFamilyName();
+ }
+
public BloomType getBloomFilterType() {
return this.bloomFilterType;
}
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java?rev=1181579&r1=1181578&r2=1181579&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java Tue Oct 11 02:22:16 2011
@@ -135,6 +135,11 @@ public class RegionServerDynamicMetrics
for (Entry<String, AtomicLong> entry : HRegion.numericMetrics.entrySet()) {
this.setNumericMetric(entry.getKey(), entry.getValue().getAndSet(0));
}
+ /* get dynamically created numeric metrics, and push the metrics.
+ * These ones aren't to be reset; they are cumulative. */
+ for (Entry<String, AtomicLong> entry : HRegion.numericPersistentMetrics.entrySet()) {
+ this.setNumericMetric(entry.getKey(), entry.getValue().get());
+ }
/* get dynamically created time varying metrics, and push the metrics */
for (Entry<String, Pair<AtomicLong, AtomicInteger>> entry :
HRegion.timeVaryingMetrics.entrySet()) {