You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@bookkeeper.apache.org by GitBox <gi...@apache.org> on 2018/12/16 08:20:37 UTC
[GitHub] sijie closed pull request #1875: [stats][doc] Add @StatsDoc
annotation for garbage collector stats
sijie closed pull request #1875: [stats][doc] Add @StatsDoc annotation for garbage collector stats
URL: https://github.com/apache/bookkeeper/pull/1875
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
index ca02651e87..006592646c 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
@@ -21,14 +21,6 @@
package org.apache.bookkeeper.bookie;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ACTIVE_ENTRY_LOG_COUNT;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ACTIVE_ENTRY_LOG_SPACE_BYTES;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.MAJOR_COMPACTION_COUNT;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.MINOR_COMPACTION_COUNT;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.RECLAIMED_COMPACTION_SPACE_BYTES;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.RECLAIMED_DELETION_SPACE_BYTES;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.THREAD_RUNTIME;
-
import com.google.common.annotations.VisibleForTesting;
import io.netty.util.concurrent.DefaultThreadFactory;
@@ -46,11 +38,9 @@
import java.util.concurrent.atomic.AtomicLong;
import org.apache.bookkeeper.bookie.GarbageCollector.GarbageCleaner;
+import org.apache.bookkeeper.bookie.stats.GarbageCollectorStats;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.meta.LedgerManager;
-import org.apache.bookkeeper.stats.Counter;
-import org.apache.bookkeeper.stats.Gauge;
-import org.apache.bookkeeper.stats.OpStatsLogger;
import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.bookkeeper.util.MathUtils;
import org.apache.bookkeeper.util.SafeRunnable;
@@ -92,12 +82,7 @@
final AbstractLogCompactor compactor;
// Stats loggers for garbage collection operations
- final StatsLogger statsLogger;
- private final Counter minorCompactionCounter;
- private final Counter majorCompactionCounter;
- private final Counter reclaimedSpaceViaDeletes;
- private final Counter reclaimedSpaceViaCompaction;
- private final OpStatsLogger gcThreadRuntime;
+ private final GarbageCollectorStats gcStats;
private volatile long totalEntryLogSize;
private volatile int numActiveEntryLogs;
@@ -124,7 +109,7 @@
// Boolean to disable minor compaction, when disk is full
final AtomicBoolean suspendMinorCompaction = new AtomicBoolean(false);
- final GarbageCollector garbageCollector;
+ final ScanAndCompareGarbageCollector garbageCollector;
final GarbageCleaner garbageCleaner;
final ServerConfiguration conf;
@@ -162,52 +147,28 @@ public GarbageCollectorThread(ServerConfiguration conf,
this.ledgerStorage = ledgerStorage;
this.gcWaitTime = conf.getGcWaitTime();
+ this.numActiveEntryLogs = 0;
+ this.totalEntryLogSize = 0L;
+ this.garbageCollector = new ScanAndCompareGarbageCollector(ledgerManager, ledgerStorage, conf, statsLogger);
+ this.gcStats = new GarbageCollectorStats(
+ statsLogger,
+ () -> numActiveEntryLogs,
+ () -> totalEntryLogSize,
+ () -> garbageCollector.getNumActiveLedgers()
+ );
+
this.garbageCleaner = ledgerId -> {
try {
if (LOG.isDebugEnabled()) {
LOG.debug("delete ledger : " + ledgerId);
}
+ gcStats.getDeletedLedgerCounter().inc();
ledgerStorage.deleteLedger(ledgerId);
} catch (IOException e) {
LOG.error("Exception when deleting the ledger index file on the Bookie: ", e);
}
};
- // Stat state initialization
- this.statsLogger = statsLogger;
-
- this.minorCompactionCounter = statsLogger.getCounter(MINOR_COMPACTION_COUNT);
- this.majorCompactionCounter = statsLogger.getCounter(MAJOR_COMPACTION_COUNT);
- this.reclaimedSpaceViaCompaction = statsLogger.getCounter(RECLAIMED_COMPACTION_SPACE_BYTES);
- this.reclaimedSpaceViaDeletes = statsLogger.getCounter(RECLAIMED_DELETION_SPACE_BYTES);
- this.gcThreadRuntime = statsLogger.getOpStatsLogger(THREAD_RUNTIME);
- this.numActiveEntryLogs = 0;
- statsLogger.registerGauge(ACTIVE_ENTRY_LOG_COUNT, new Gauge<Integer>() {
- @Override
- public Integer getDefaultValue() {
- return 0;
- }
-
- @Override
- public Integer getSample() {
- return numActiveEntryLogs;
- }
- });
- this.totalEntryLogSize = 0L;
- statsLogger.registerGauge(ACTIVE_ENTRY_LOG_SPACE_BYTES, new Gauge<Long>() {
- @Override
- public Long getDefaultValue() {
- return 0L;
- }
-
- @Override
- public Long getSample() {
- return totalEntryLogSize;
- }
- });
-
- this.garbageCollector = new ScanAndCompareGarbageCollector(ledgerManager, ledgerStorage, conf, statsLogger);
-
// compaction parameters
minorCompactionThreshold = conf.getMinorCompactionThreshold();
minorCompactionInterval = conf.getMinorCompactionInterval() * SECOND;
@@ -389,7 +350,7 @@ public void runWithFlags(boolean force, boolean suspendMajor, boolean suspendMin
lastMajorCompactionTime = System.currentTimeMillis();
// and also move minor compaction time
lastMinorCompactionTime = lastMajorCompactionTime;
- majorCompactionCounter.inc();
+ gcStats.getMajorCompactionCounter().inc();
majorCompacting.set(false);
} else if (enableMinorCompaction && (!suspendMinor)
&& (force || curTime - lastMinorCompactionTime > minorCompactionInterval)) {
@@ -398,7 +359,7 @@ public void runWithFlags(boolean force, boolean suspendMajor, boolean suspendMin
minorCompacting.set(true);
doCompactEntryLogs(minorCompactionThreshold);
lastMinorCompactionTime = System.currentTimeMillis();
- minorCompactionCounter.inc();
+ gcStats.getMinorCompactionCounter().inc();
minorCompacting.set(false);
}
@@ -408,7 +369,7 @@ public void runWithFlags(boolean force, boolean suspendMajor, boolean suspendMin
.currentThread().getName());
}
}
- this.gcThreadRuntime.registerSuccessfulEvent(
+ gcStats.getGcThreadRuntime().registerSuccessfulEvent(
MathUtils.nowInNano() - threadStart, TimeUnit.NANOSECONDS);
}
@@ -434,7 +395,7 @@ private void doGcEntryLogs() {
// We can remove this entry log file now.
LOG.info("Deleting entryLogId " + entryLogId + " as it has no active ledgers!");
removeEntryLog(entryLogId);
- this.reclaimedSpaceViaDeletes.add(meta.getTotalSize());
+ gcStats.getReclaimedSpaceViaDeletes().add(meta.getTotalSize());
}
totalEntryLogSizeAcc.getAndAdd(meta.getRemainingSize());
@@ -492,7 +453,7 @@ void doCompactEntryLogs(double threshold) {
long priorRemainingSize = meta.getRemainingSize();
compactEntryLog(meta);
- this.reclaimedSpaceViaCompaction.add(meta.getTotalSize() - priorRemainingSize);
+ gcStats.getReclaimedSpaceViaCompaction().add(meta.getTotalSize() - priorRemainingSize);
if (!running) { // if gc thread is not running, stop compaction
return;
@@ -621,8 +582,8 @@ public GarbageCollectionStatus getGarbageCollectionStatus() {
.minorCompacting(minorCompacting.get())
.lastMajorCompactionTime(lastMajorCompactionTime)
.lastMinorCompactionTime(lastMinorCompactionTime)
- .majorCompactionCounter(majorCompactionCounter.get())
- .minorCompactionCounter(minorCompactionCounter.get())
+ .majorCompactionCounter(gcStats.getMajorCompactionCounter().get())
+ .minorCompactionCounter(gcStats.getMinorCompactionCounter().get())
.build();
}
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java
index 6af5d15b02..24c5c97e9c 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java
@@ -21,8 +21,6 @@
package org.apache.bookkeeper.bookie;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ACTIVE_LEDGER_COUNT;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.DELETED_LEDGER_COUNT;
import static org.apache.bookkeeper.common.concurrent.FutureUtils.result;
import com.google.common.collect.Sets;
@@ -42,8 +40,6 @@
import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager;
import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase;
import org.apache.bookkeeper.net.BookieSocketAddress;
-import org.apache.bookkeeper.stats.Counter;
-import org.apache.bookkeeper.stats.Gauge;
import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.bookkeeper.util.ZkUtils;
import org.apache.bookkeeper.zookeeper.ZooKeeperClient;
@@ -69,7 +65,7 @@
*
* <p>TODO: eliminate the direct usage of zookeeper here {@link https://github.com/apache/bookkeeper/issues/1331}
*/
-public class ScanAndCompareGarbageCollector implements GarbageCollector{
+public class ScanAndCompareGarbageCollector implements GarbageCollector {
static final Logger LOG = LoggerFactory.getLogger(ScanAndCompareGarbageCollector.class);
static final int MAX_CONCURRENT_ZK_REQUESTS = 1000;
@@ -86,7 +82,6 @@
private final String zkLedgersRootPath;
private final boolean verifyMetadataOnGc;
private int activeLedgerCounter;
- private Counter deletedLedgerCounter;
public ScanAndCompareGarbageCollector(LedgerManager ledgerManager, CompactableLedgerStorage ledgerStorage,
ServerConfiguration conf, StatsLogger statsLogger) throws IOException {
@@ -106,20 +101,11 @@ public ScanAndCompareGarbageCollector(LedgerManager ledgerManager, CompactableLe
verifyMetadataOnGc = conf.getVerifyMetadataOnGC();
- this.deletedLedgerCounter = statsLogger.getCounter(DELETED_LEDGER_COUNT);
-
this.activeLedgerCounter = 0;
- statsLogger.registerGauge(ACTIVE_LEDGER_COUNT, new Gauge<Integer>() {
- @Override
- public Integer getDefaultValue() {
- return 0;
- }
+ }
- @Override
- public Integer getSample() {
- return activeLedgerCounter;
- }
- });
+ public int getNumActiveLedgers() {
+ return activeLedgerCounter;
}
@Override
@@ -190,7 +176,6 @@ public void gc(GarbageCleaner garbageCleaner) {
continue;
}
}
- deletedLedgerCounter.inc();
garbageCleaner.clean(bkLid);
}
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/GarbageCollectorStats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/GarbageCollectorStats.java
new file mode 100644
index 0000000000..1c9475608f
--- /dev/null
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/GarbageCollectorStats.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bookkeeper.bookie.stats;
+
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ACTIVE_ENTRY_LOG_COUNT;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ACTIVE_ENTRY_LOG_SPACE_BYTES;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ACTIVE_LEDGER_COUNT;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_SCOPE;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.CATEGORY_SERVER;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.DELETED_LEDGER_COUNT;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.MAJOR_COMPACTION_COUNT;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.MINOR_COMPACTION_COUNT;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.RECLAIMED_COMPACTION_SPACE_BYTES;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.RECLAIMED_DELETION_SPACE_BYTES;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.THREAD_RUNTIME;
+
+import java.util.function.Supplier;
+import lombok.Getter;
+import org.apache.bookkeeper.stats.Counter;
+import org.apache.bookkeeper.stats.Gauge;
+import org.apache.bookkeeper.stats.OpStatsLogger;
+import org.apache.bookkeeper.stats.StatsLogger;
+import org.apache.bookkeeper.stats.annotations.StatsDoc;
+
+/**
+ * A umbrella class for gc stats.
+ */
+@StatsDoc(
+ name = BOOKIE_SCOPE,
+ category = CATEGORY_SERVER,
+ help = "Garbage Collector related stats"
+)
+@Getter
+public class GarbageCollectorStats {
+
+ final StatsLogger statsLogger;
+ @StatsDoc(
+ name = MINOR_COMPACTION_COUNT,
+ help = "Number of minor compactions"
+ )
+ private final Counter minorCompactionCounter;
+ @StatsDoc(
+ name = MAJOR_COMPACTION_COUNT,
+ help = "Number of major compactions"
+ )
+ private final Counter majorCompactionCounter;
+ @StatsDoc(
+ name = RECLAIMED_DELETION_SPACE_BYTES,
+ help = "Number of disk space bytes reclaimed via deleting entry log files"
+ )
+ private final Counter reclaimedSpaceViaDeletes;
+ @StatsDoc(
+ name = RECLAIMED_COMPACTION_SPACE_BYTES,
+ help = "Number of disk space bytes reclaimed via compacting entry log files"
+ )
+ private final Counter reclaimedSpaceViaCompaction;
+ @StatsDoc(
+ name = DELETED_LEDGER_COUNT,
+ help = "Number of ledgers deleted by garbage collection"
+ )
+ private final Counter deletedLedgerCounter;
+ @StatsDoc(
+ name = THREAD_RUNTIME,
+ help = "Operation stats of garbage collections"
+ )
+ private final OpStatsLogger gcThreadRuntime;
+ @StatsDoc(
+ name = ACTIVE_ENTRY_LOG_COUNT,
+ help = "Current number of active entry log files"
+ )
+ private final Gauge<Integer> activeEntryLogCountGauge;
+ @StatsDoc(
+ name = ACTIVE_ENTRY_LOG_SPACE_BYTES,
+ help = "Current number of active entry log space bytes"
+ )
+ private final Gauge<Long> activeEntryLogSpaceBytesGauge;
+ @StatsDoc(
+ name = ACTIVE_LEDGER_COUNT,
+ help = "Current number of active ledgers"
+ )
+ private final Gauge<Integer> activeLedgerCountGauge;
+
+ public GarbageCollectorStats(StatsLogger statsLogger,
+ Supplier<Integer> activeEntryLogCountSupplier,
+ Supplier<Long> activeEntryLogSpaceBytesSupplier,
+ Supplier<Integer> activeLedgerCountSupplier) {
+ this.statsLogger = statsLogger;
+
+ this.minorCompactionCounter = statsLogger.getCounter(MINOR_COMPACTION_COUNT);
+ this.majorCompactionCounter = statsLogger.getCounter(MAJOR_COMPACTION_COUNT);
+ this.reclaimedSpaceViaCompaction = statsLogger.getCounter(RECLAIMED_COMPACTION_SPACE_BYTES);
+ this.reclaimedSpaceViaDeletes = statsLogger.getCounter(RECLAIMED_DELETION_SPACE_BYTES);
+ this.gcThreadRuntime = statsLogger.getOpStatsLogger(THREAD_RUNTIME);
+ this.deletedLedgerCounter = statsLogger.getCounter(DELETED_LEDGER_COUNT);
+
+ this.activeEntryLogCountGauge = new Gauge<Integer>() {
+ @Override
+ public Integer getDefaultValue() {
+ return 0;
+ }
+
+ @Override
+ public Integer getSample() {
+ return activeEntryLogCountSupplier.get();
+ }
+ };
+ statsLogger.registerGauge(ACTIVE_ENTRY_LOG_COUNT, activeEntryLogCountGauge);
+ this.activeEntryLogSpaceBytesGauge = new Gauge<Long>() {
+ @Override
+ public Long getDefaultValue() {
+ return 0L;
+ }
+
+ @Override
+ public Long getSample() {
+ return activeEntryLogSpaceBytesSupplier.get();
+ }
+ };
+ statsLogger.registerGauge(ACTIVE_ENTRY_LOG_SPACE_BYTES, activeEntryLogSpaceBytesGauge);
+ this.activeLedgerCountGauge = new Gauge<Integer>() {
+ @Override
+ public Integer getDefaultValue() {
+ return 0;
+ }
+
+ @Override
+ public Integer getSample() {
+ return activeLedgerCountSupplier.get();
+ }
+ };
+ statsLogger.registerGauge(ACTIVE_LEDGER_COUNT, activeLedgerCountGauge);
+ }
+
+}
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/GcLedgersTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/GcLedgersTest.java
index d2735657e1..828956d4f3 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/GcLedgersTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/GcLedgersTest.java
@@ -21,8 +21,6 @@
package org.apache.bookkeeper.meta;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ACTIVE_LEDGER_COUNT;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.DELETED_LEDGER_COUNT;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -173,7 +171,7 @@ public void testGarbageCollectLedgers() throws Exception {
final CountDownLatch endLatch = new CountDownLatch(2);
final CompactableLedgerStorage mockLedgerStorage = new MockLedgerStorage();
TestStatsProvider stats = new TestStatsProvider();
- final GarbageCollector garbageCollector = new ScanAndCompareGarbageCollector(getLedgerManager(),
+ final ScanAndCompareGarbageCollector garbageCollector = new ScanAndCompareGarbageCollector(getLedgerManager(),
mockLedgerStorage, baseConf, stats.getStatsLogger("gc"));
Thread gcThread = new Thread() {
@Override
@@ -235,12 +233,9 @@ public void run() {
for (Long ledger : createdLedgers) {
assertTrue(activeLedgers.containsKey(ledger));
}
- assertTrue(
- "Wrong DELETED_LEDGER_COUNT",
- stats.getCounter("gc." + DELETED_LEDGER_COUNT).get() == removedLedgers.size());
assertTrue(
"Wrong ACTIVE_LEDGER_COUNT",
- stats.getGauge("gc." + ACTIVE_LEDGER_COUNT).getSample().intValue() == createdLedgers.size());
+ garbageCollector.getNumActiveLedgers() == createdLedgers.size());
}
@Test
@@ -253,7 +248,7 @@ public void testGcLedgersOutsideRange() throws Exception {
MockLedgerStorage mockLedgerStorage = new MockLedgerStorage();
TestStatsProvider stats = new TestStatsProvider();
- final GarbageCollector garbageCollector = new ScanAndCompareGarbageCollector(getLedgerManager(),
+ final ScanAndCompareGarbageCollector garbageCollector = new ScanAndCompareGarbageCollector(getLedgerManager(),
mockLedgerStorage, baseConf, stats.getStatsLogger("gc"));
GarbageCollector.GarbageCleaner cleaner = new GarbageCollector.GarbageCleaner() {
@Override
@@ -271,37 +266,26 @@ public void clean(long ledgerId) {
garbageCollector.gc(cleaner);
assertNull("Should have cleaned nothing", cleaned.poll());
- assertTrue(
- "Wrong DELETED_LEDGER_COUNT",
- stats.getCounter("gc." + DELETED_LEDGER_COUNT).get() == 0);
assertTrue(
"Wrong ACTIVE_LEDGER_COUNT",
- stats.getGauge(
- "gc." + ACTIVE_LEDGER_COUNT).getSample().intValue() == numLedgers);
+ garbageCollector.getNumActiveLedgers() == numLedgers);
long last = createdLedgers.last();
removeLedger(last);
garbageCollector.gc(cleaner);
assertNotNull("Should have cleaned something", cleaned.peek());
assertEquals("Should have cleaned last ledger" + last, (long) last, (long) cleaned.poll());
- assertTrue(
- "Wrong DELETED_LEDGER_COUNT",
- stats.getCounter("gc." + DELETED_LEDGER_COUNT).get() == 1);
long first = createdLedgers.first();
removeLedger(first);
garbageCollector.gc(cleaner);
assertNotNull("Should have cleaned something", cleaned.peek());
assertEquals("Should have cleaned first ledger" + first, (long) first, (long) cleaned.poll());
- assertTrue(
- "Wrong DELETED_LEDGER_COUNT",
- stats.getCounter("gc." + DELETED_LEDGER_COUNT).get() == 2);
garbageCollector.gc(cleaner);
assertTrue(
"Wrong ACTIVE_LEDGER_COUNT",
- stats.getGauge(
- "gc." + ACTIVE_LEDGER_COUNT).getSample().intValue() == (numLedgers - 2));
+ garbageCollector.getNumActiveLedgers() == (numLedgers - 2));
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services