You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by si...@apache.org on 2018/11/27 10:16:39 UTC
[bookkeeper] branch branch-4.8 updated: Add rest endpoint
trigger_gc to trigger GC on Bookie
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-4.8
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/branch-4.8 by this push:
new 0c98b40 Add rest endpoint trigger_gc to trigger GC on Bookie
0c98b40 is described below
commit 0c98b400221b5e508807dd96f71c5bec5da89c65
Author: Jia Zhai <ji...@users.noreply.github.com>
AuthorDate: Tue Nov 27 18:16:19 2018 +0800
Add rest endpoint trigger_gc to trigger GC on Bookie
Descriptions of the changes in this PR:
Add rest endpoint trigger_gc to trigger GC on Bookie
### Motivation
Some times user would like to trigger GC manually instead of waiting to the timeout or disk full.
Reviewers: Enrico Olivelli <eo...@gmail.com>, Sijie Guo <si...@apache.org>
This closes #1838 from jiazhai/rest_force_gc
(cherry picked from commit 641db3b4ff29d8e04216fee84625546707631e8e)
Signed-off-by: Sijie Guo <si...@apache.org>
---
.../org/apache/bookkeeper/http/HttpRouter.java | 2 +
.../org/apache/bookkeeper/http/HttpServer.java | 1 +
.../bookkeeper/bookie/GarbageCollectorThread.java | 7 +++
.../bookie/InterleavedLedgerStorage.java | 5 ++
.../apache/bookkeeper/bookie/LedgerStorage.java | 6 ++
.../bookkeeper/bookie/SortedLedgerStorage.java | 5 ++
.../bookie/storage/ldb/DbLedgerStorage.java | 4 ++
.../ldb/SingleDirectoryDbLedgerStorage.java | 5 ++
.../server/http/BKHttpServiceProvider.java | 3 +
.../server/http/service/TriggerGCService.java | 71 ++++++++++++++++++++++
.../bookkeeper/server/http/TestHttpService.java | 38 ++++++++++++
site/docs/latest/admin/http.md | 11 ++++
12 files changed, 158 insertions(+)
diff --git a/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpRouter.java b/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpRouter.java
index a856c72..5384df6 100644
--- a/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpRouter.java
+++ b/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpRouter.java
@@ -45,6 +45,7 @@ public abstract class HttpRouter<Handler> {
public static final String LAST_LOG_MARK = "/api/v1/bookie/last_log_mark";
public static final String LIST_DISK_FILE = "/api/v1/bookie/list_disk_file";
public static final String EXPAND_STORAGE = "/api/v1/bookie/expand_storage";
+ public static final String GC = "/api/v1/bookie/gc";
// autorecovery
public static final String RECOVERY_BOOKIE = "/api/v1/autorecovery/bookie";
public static final String LIST_UNDER_REPLICATED_LEDGER = "/api/v1/autorecovery/list_under_replicated_ledger";
@@ -73,6 +74,7 @@ public abstract class HttpRouter<Handler> {
this.endpointHandlers.put(LAST_LOG_MARK, handlerFactory.newHandler(HttpServer.ApiType.LAST_LOG_MARK));
this.endpointHandlers.put(LIST_DISK_FILE, handlerFactory.newHandler(HttpServer.ApiType.LIST_DISK_FILE));
this.endpointHandlers.put(EXPAND_STORAGE, handlerFactory.newHandler(HttpServer.ApiType.EXPAND_STORAGE));
+ this.endpointHandlers.put(GC, handlerFactory.newHandler(HttpServer.ApiType.GC));
// autorecovery
this.endpointHandlers.put(RECOVERY_BOOKIE, handlerFactory.newHandler(HttpServer.ApiType.RECOVERY_BOOKIE));
diff --git a/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpServer.java b/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpServer.java
index 30e4d05..5e9f509 100644
--- a/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpServer.java
+++ b/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpServer.java
@@ -77,6 +77,7 @@ public interface HttpServer {
LAST_LOG_MARK,
LIST_DISK_FILE,
EXPAND_STORAGE,
+ GC,
// autorecovery
RECOVERY_BOOKIE,
LIST_UNDER_REPLICATED_LEDGER,
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
index ad752de..c4d9414 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
@@ -382,6 +382,13 @@ public class GarbageCollectorThread extends SafeRunnable {
lastMinorCompactionTime = System.currentTimeMillis();
minorCompactionCounter.inc();
}
+
+ if (force) {
+ if (forceGarbageCollection.compareAndSet(true, false)) {
+ LOG.info("{} Set forceGarbageCollection to false after force GC to make it forceGC-able again.", Thread
+ .currentThread().getName());
+ }
+ }
this.gcThreadRuntime.registerSuccessfulEvent(
MathUtils.nowInNano() - threadStart, TimeUnit.NANOSECONDS);
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java
index 8ab6517..d7d4977 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java
@@ -192,6 +192,11 @@ public class InterleavedLedgerStorage implements CompactableLedgerStorage, Entry
}
@Override
+ public void forceGC() {
+ gcThread.enableForceGC();
+ }
+
+ @Override
public void start() {
gcThread.start();
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerStorage.java
index 34e32b9..ef64ff9 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerStorage.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerStorage.java
@@ -175,4 +175,10 @@ public interface LedgerStorage {
return this;
}
+ /**
+ * Force trigger Garbage Collection.
+ */
+ default void forceGC() {
+ return;
+ }
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/SortedLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/SortedLedgerStorage.java
index e4b137f..dd7b373 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/SortedLedgerStorage.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/SortedLedgerStorage.java
@@ -327,4 +327,9 @@ public class SortedLedgerStorage
public LedgerStorage getUnderlyingLedgerStorage() {
return interleavedLedgerStorage;
}
+
+ @Override
+ public void forceGC() {
+ interleavedLedgerStorage.forceGC();
+ }
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
index 8753363..831de53 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
@@ -348,4 +348,8 @@ public class DbLedgerStorage implements LedgerStorage {
}
}
+ @Override
+ public void forceGC() {
+ ledgerStorageList.stream().forEach(SingleDirectoryDbLedgerStorage::forceGC);
+ }
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
index e31b44e..27b4214 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
@@ -264,6 +264,11 @@ public class SingleDirectoryDbLedgerStorage implements CompactableLedgerStorage
}
@Override
+ public void forceGC() {
+ gcThread.enableForceGC();
+ }
+
+ @Override
public void shutdown() throws InterruptedException {
try {
flush();
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/BKHttpServiceProvider.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/BKHttpServiceProvider.java
index 052b50e..102b0eb 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/BKHttpServiceProvider.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/BKHttpServiceProvider.java
@@ -53,6 +53,7 @@ import org.apache.bookkeeper.server.http.service.MetricsService;
import org.apache.bookkeeper.server.http.service.ReadLedgerEntryService;
import org.apache.bookkeeper.server.http.service.RecoveryBookieService;
import org.apache.bookkeeper.server.http.service.TriggerAuditService;
+import org.apache.bookkeeper.server.http.service.TriggerGCService;
import org.apache.bookkeeper.server.http.service.WhoIsAuditorService;
import org.apache.bookkeeper.stats.StatsProvider;
import org.apache.bookkeeper.zookeeper.ZooKeeperClient;
@@ -208,6 +209,8 @@ public class BKHttpServiceProvider implements HttpServiceProvider {
return new ListDiskFilesService(configuration);
case EXPAND_STORAGE:
return new ExpandStorageService(configuration);
+ case GC:
+ return new TriggerGCService(configuration, bookieServer);
// autorecovery
case RECOVERY_BOOKIE:
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/TriggerGCService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/TriggerGCService.java
new file mode 100644
index 0000000..185965a
--- /dev/null
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/TriggerGCService.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.server.http.service;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.bookkeeper.common.util.JsonUtil;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.http.HttpServer;
+import org.apache.bookkeeper.http.service.HttpEndpointService;
+import org.apache.bookkeeper.http.service.HttpServiceRequest;
+import org.apache.bookkeeper.http.service.HttpServiceResponse;
+import org.apache.bookkeeper.proto.BookieServer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * HttpEndpointService that handle force trigger GC requests.
+ *
+ * <p>The PUT method will force trigger GC on current bookie, and make GC run at backend.
+ */
+public class TriggerGCService implements HttpEndpointService {
+
+ static final Logger LOG = LoggerFactory.getLogger(TriggerGCService.class);
+
+ protected ServerConfiguration conf;
+ protected BookieServer bookieServer;
+
+ public TriggerGCService(ServerConfiguration conf, BookieServer bookieServer) {
+ checkNotNull(conf);
+ checkNotNull(bookieServer);
+ this.conf = conf;
+ this.bookieServer = bookieServer;
+ }
+
+ @Override
+ public HttpServiceResponse handle(HttpServiceRequest request) throws Exception {
+ HttpServiceResponse response = new HttpServiceResponse();
+ // PUT
+ if (HttpServer.Method.PUT == request.getMethod()) {
+ bookieServer.getBookie().getLedgerStorage().forceGC();
+
+ String output = "Triggered GC on BookieServer: " + bookieServer.toString();
+ String jsonResponse = JsonUtil.toJson(output);
+ LOG.debug("output body:" + jsonResponse);
+ response.setBody(jsonResponse);
+ response.setCode(HttpServer.StatusCode.OK);
+ return response;
+ } else {
+ response.setCode(HttpServer.StatusCode.NOT_FOUND);
+ response.setBody("Not found method. Should be PUT method to trigger GC.");
+ return response;
+ }
+ }
+}
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java
index 5d87ca9..c3859d6 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java
@@ -749,4 +749,42 @@ public class TestHttpService extends BookKeeperClusterTestCase {
stopAuditorElector();
}
+ /**
+ * Create ledgers, then test Delete Ledger service.
+ */
+ @Test
+ public void testTriggerGCService() throws Exception {
+ baseConf.setMetadataServiceUri(zkUtil.getMetadataServiceUri());
+ BookKeeper.DigestType digestType = BookKeeper.DigestType.CRC32;
+ int numLedgers = 4;
+ int numMsgs = 100;
+ LedgerHandle[] lh = new LedgerHandle[numLedgers];
+ // create ledgers
+ for (int i = 0; i < numLedgers; i++) {
+ lh[i] = bkc.createLedger(digestType, "".getBytes());
+ }
+ String content = "Apache BookKeeper is cool!";
+ // add entries
+ for (int i = 0; i < numMsgs; i++) {
+ for (int j = 0; j < numLedgers; j++) {
+ lh[j].addEntry(content.getBytes());
+ }
+ }
+ // close ledgers
+ for (int i = 0; i < numLedgers; i++) {
+ lh[i].close();
+ }
+ HttpEndpointService triggerGCService = bkHttpServiceProvider
+ .provideHttpEndpointService(HttpServer.ApiType.GC);
+
+ //1, GET, should return NOT_FOUND
+ HttpServiceRequest request1 = new HttpServiceRequest(null, HttpServer.Method.GET, null);
+ HttpServiceResponse response1 = triggerGCService.handle(request1);
+ assertEquals(HttpServer.StatusCode.NOT_FOUND.getValue(), response1.getStatusCode());
+
+ //2, PUT, should return OK
+ HttpServiceRequest request2 = new HttpServiceRequest(null, HttpServer.Method.PUT, null);
+ HttpServiceResponse response2 = triggerGCService.handle(request2);
+ assertEquals(HttpServer.StatusCode.OK.getValue(), response2.getStatusCode());
+ }
}
diff --git a/site/docs/latest/admin/http.md b/site/docs/latest/admin/http.md
index dc64744..7c1abea 100644
--- a/site/docs/latest/admin/http.md
+++ b/site/docs/latest/admin/http.md
@@ -271,6 +271,17 @@ Currently all the HTTP endpoints could be divided into these 4 components:
|403 | Permission denied |
|404 | Not found |
+### Endpoint: /api/v1/bookie/gc
+1. Method: PUT
+ * Description: trigger gc for this bookie.
+ * Response:
+
+ | Code | Description |
+ |:-------|:------------|
+ |200 | Successful operation |
+ |403 | Permission denied |
+ |404 | Not found |
+
## Auto recovery
### Endpoint: /api/v1/autorecovery/bookie/