You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by uc...@apache.org on 2016/07/25 13:21:05 UTC
flink git commit: [FLINK-4150] [runtime] Don't clean up BlobStore on
BlobServer shut down
Repository: flink
Updated Branches:
refs/heads/master 2818ee009 -> 32130160b
[FLINK-4150] [runtime] Don't clean up BlobStore on BlobServer shut down
The `BlobServer` acts as a local cache for uploaded BLOBs. The life-cycle of
each BLOB is bound to the life-cycle of the `BlobServer`. If the BlobServer
shuts down (on JobManager shut down), all local files will be removed.
With HA, BLOBs are persisted to another file system (e.g. HDFS) via the
`BlobStore` in order to have BLOBs available after a JobManager failure (or
shut down). These BLOBs are only allowed to be removed when the job that
requires them enters a globally terminal state (`FINISHED`, `CANCELLED`,
`FAILED`).
This commit removes the `BlobStore` clean up call from the `BlobServer`
shutdown. The `BlobStore` files will only be cleaned up via the
`BlobLibraryCacheManager`'s' clean up task (periodically or on
BlobLibraryCacheManager shutdown). This means that there is a chance that
BLOBs will linger around after the job has terminated, if the job manager
fails before the clean up.
This closes #2256.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/32130160
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/32130160
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/32130160
Branch: refs/heads/master
Commit: 32130160bcc5ef2edb7d03c398dd506ffb4291c6
Parents: 2818ee0
Author: Ufuk Celebi <uc...@apache.org>
Authored: Thu Jul 14 16:29:49 2016 +0200
Committer: Ufuk Celebi <uc...@apache.org>
Committed: Mon Jul 25 15:20:26 2016 +0200
----------------------------------------------------------------------
.../org/apache/flink/runtime/blob/BlobServer.java | 3 ---
.../flink/runtime/blob/BlobServerConnection.java | 18 +++++++++++-------
.../flink/runtime/blob/FileSystemBlobStore.java | 12 +++++++++++-
.../librarycache/BlobLibraryCacheManager.java | 9 +++++++--
.../flink/runtime/blob/BlobRecoveryITCase.java | 6 ++++++
.../BlobLibraryCacheRecoveryITCase.java | 6 ++++++
.../TestingLeaderElectionService.java | 5 +++++
7 files changed, 46 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/32130160/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java
index ed3a68b..b4b6812 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java
@@ -301,9 +301,6 @@ public class BlobServer extends Thread implements BlobService {
LOG.error("BLOB server failed to properly clean up its storage directory.");
}
- // Clean up the recovery directory
- blobStore.cleanUp();
-
// Remove shutdown hook to prevent resource leaks, unless this is invoked by the
// shutdown hook itself
if (shutdownHook != null && shutdownHook != Thread.currentThread()) {
http://git-wip-us.apache.org/repos/asf/flink/blob/32130160/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServerConnection.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServerConnection.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServerConnection.java
index d7bba8f..321fc67 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServerConnection.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServerConnection.java
@@ -64,6 +64,9 @@ class BlobServerConnection extends Thread {
/** The BLOB server. */
private final BlobServer blobServer;
+ /** The HA blob store. */
+ private final BlobStore blobStore;
+
/**
* Creates a new BLOB connection for a client request
*
@@ -80,6 +83,7 @@ class BlobServerConnection extends Thread {
this.clientSocket = clientSocket;
this.blobServer = blobServer;
+ this.blobStore = blobServer.getBlobStore();
}
// --------------------------------------------------------------------------------------------
@@ -182,7 +186,7 @@ class BlobServerConnection extends Thread {
blobFile = this.blobServer.getStorageLocation(jobID, key);
if (!blobFile.exists()) {
- blobServer.getBlobStore().get(jobID, key, blobFile);
+ blobStore.get(jobID, key, blobFile);
}
}
else if (contentAddressable == CONTENT_ADDRESSABLE) {
@@ -190,7 +194,7 @@ class BlobServerConnection extends Thread {
blobFile = blobServer.getStorageLocation(key);
if (!blobFile.exists()) {
- blobServer.getBlobStore().get(key, blobFile);
+ blobStore.get(key, blobFile);
}
}
else {
@@ -320,7 +324,7 @@ class BlobServerConnection extends Thread {
Files.move(incomingFile, storageFile);
incomingFile = null;
- blobServer.getBlobStore().put(storageFile, jobID, key);
+ blobStore.put(storageFile, jobID, key);
outputStream.write(RETURN_OKAY);
}
@@ -330,7 +334,7 @@ class BlobServerConnection extends Thread {
Files.move(incomingFile, storageFile);
incomingFile = null;
- blobServer.getBlobStore().put(storageFile, blobKey);
+ blobStore.put(storageFile, blobKey);
// Return computed key to client for validation
outputStream.write(RETURN_OKAY);
@@ -390,7 +394,7 @@ class BlobServerConnection extends Thread {
throw new IOException("Cannot delete BLOB file " + blobFile.getAbsolutePath());
}
- blobServer.getBlobStore().delete(key);
+ blobStore.delete(key);
}
else if (type == NAME_ADDRESSABLE) {
byte[] jidBytes = new byte[JobID.SIZE];
@@ -404,7 +408,7 @@ class BlobServerConnection extends Thread {
throw new IOException("Cannot delete BLOB file " + blobFile.getAbsolutePath());
}
- blobServer.getBlobStore().delete(jobID, key);
+ blobStore.delete(jobID, key);
}
else if (type == JOB_ID_SCOPE) {
byte[] jidBytes = new byte[JobID.SIZE];
@@ -413,7 +417,7 @@ class BlobServerConnection extends Thread {
blobServer.deleteJobDirectory(jobID);
- blobServer.getBlobStore().deleteAll(jobID);
+ blobStore.deleteAll(jobID);
}
else {
throw new IOException("Unrecognized addressing type: " + type);
http://git-wip-us.apache.org/repos/asf/flink/blob/32130160/flink-runtime/src/main/java/org/apache/flink/runtime/blob/FileSystemBlobStore.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/FileSystemBlobStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/FileSystemBlobStore.java
index 226b063..5641d87 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/FileSystemBlobStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/FileSystemBlobStore.java
@@ -143,7 +143,17 @@ class FileSystemBlobStore implements BlobStore {
try {
LOG.debug("Deleting {}.", blobPath);
- FileSystem.get(new URI(blobPath)).delete(new Path(blobPath), true);
+ FileSystem fs = FileSystem.get(new URI(blobPath));
+ Path path = new Path(blobPath);
+
+ fs.delete(path, true);
+
+ // send a call to delete the directory containing the file. This will
+ // fail (and be ignored) when some files still exist.
+ try {
+ fs.delete(path.getParent(), false);
+ fs.delete(new Path(basePath), false);
+ } catch (IOException ignored) {}
}
catch (Exception e) {
LOG.warn("Failed to delete blob at " + blobPath);
http://git-wip-us.apache.org/repos/asf/flink/blob/32130160/flink-runtime/src/main/java/org/apache/flink/runtime/execution/librarycache/BlobLibraryCacheManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/librarycache/BlobLibraryCacheManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/librarycache/BlobLibraryCacheManager.java
index 12be6f4..d1fbc70 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/librarycache/BlobLibraryCacheManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/librarycache/BlobLibraryCacheManager.java
@@ -77,7 +77,7 @@ public final class BlobLibraryCacheManager extends TimerTask implements LibraryC
// Initializing the clean up task
this.cleanupTimer = new Timer(true);
- this.cleanupTimer.schedule(this, cleanupInterval);
+ this.cleanupTimer.schedule(this, cleanupInterval, cleanupInterval);
}
// --------------------------------------------------------------------------------------------
@@ -200,6 +200,12 @@ public final class BlobLibraryCacheManager extends TimerTask implements LibraryC
@Override
public void shutdown() throws IOException{
+ try {
+ run();
+ } catch (Throwable t) {
+ LOG.warn("Failed to run clean up task before shutdown", t);
+ }
+
blobService.shutdown();
cleanupTimer.cancel();
}
@@ -210,7 +216,6 @@ public final class BlobLibraryCacheManager extends TimerTask implements LibraryC
@Override
public void run() {
synchronized (lockObject) {
-
Iterator<Map.Entry<BlobKey, Integer>> entryIter = blobKeyReferenceCounters.entrySet().iterator();
while (entryIter.hasNext()) {
http://git-wip-us.apache.org/repos/asf/flink/blob/32130160/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobRecoveryITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobRecoveryITCase.java
index 2eff4fc..528a20c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobRecoveryITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobRecoveryITCase.java
@@ -139,6 +139,12 @@ public class BlobRecoveryITCase {
assertEquals(expected[i], actual[j]);
}
}
+
+ // Remove again
+ client.delete(keys[0]);
+ client.delete(keys[1]);
+ client.delete(jobId[0], testKey[0]);
+ client.delete(jobId[1], testKey[1]);
}
finally {
for (BlobServer s : server) {
http://git-wip-us.apache.org/repos/asf/flink/blob/32130160/flink-runtime/src/test/java/org/apache/flink/runtime/execution/librarycache/BlobLibraryCacheRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/execution/librarycache/BlobLibraryCacheRecoveryITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/execution/librarycache/BlobLibraryCacheRecoveryITCase.java
index 6068322..f1021e6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/execution/librarycache/BlobLibraryCacheRecoveryITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/execution/librarycache/BlobLibraryCacheRecoveryITCase.java
@@ -136,6 +136,12 @@ public class BlobLibraryCacheRecoveryITCase {
assertEquals(0, fis.available());
}
+
+ // Remove blobs again
+ try (BlobClient client = new BlobClient(serverAddress[1])) {
+ client.delete(keys.get(0));
+ client.delete(keys.get(1));
+ }
}
finally {
for (BlobServer s : server) {
http://git-wip-us.apache.org/repos/asf/flink/blob/32130160/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderElectionService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderElectionService.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderElectionService.java
index 4e119fe..16779fa 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderElectionService.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderElectionService.java
@@ -58,4 +58,9 @@ public class TestingLeaderElectionService implements LeaderElectionService {
hasLeadership = false;
contender.revokeLeadership();
}
+
+ public void reset() {
+ contender = null;
+ hasLeadership = false;
+ }
}