You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/10/31 22:27:05 UTC
[06/12] git commit: ACCUMULO-3213 Archive files that the garbage
collector would otherwise delete.
ACCUMULO-3213 Archive files that the garbage collector would otherwise delete.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7ce1a649
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7ce1a649
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7ce1a649
Branch: refs/heads/1.6
Commit: 7ce1a649fe22120833c9c0505c4a1a4c41986eca
Parents: 1890bea
Author: Josh Elser <el...@apache.org>
Authored: Tue Oct 28 21:52:21 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri Oct 31 14:59:47 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 1 +
.../apache/accumulo/server/ServerConstants.java | 8 ++-
.../accumulo/gc/SimpleGarbageCollector.java | 75 ++++++++++++++++++--
3 files changed, 78 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ce1a649/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index ad83454..56f3d9c 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -289,6 +289,7 @@ public enum Property {
GC_PORT("gc.port.client", "50091", PropertyType.PORT, "The listening port for the garbage collector's monitor service"),
GC_DELETE_THREADS("gc.threads.delete", "16", PropertyType.COUNT, "The number of threads used to delete files"),
GC_TRASH_IGNORE("gc.trash.ignore", "false", PropertyType.BOOLEAN, "Do not use the Trash, even if it is configured"),
+ GC_FILE_ARCHIVE("gc.file.archive", "false", PropertyType.BOOLEAN, "Archive any files/directories instead of moving to the HDFS trash or deleting"),
// properties that are specific to the monitor server behavior
MONITOR_PREFIX("monitor.", null, PropertyType.PREFIX, "Properties in this category affect the behavior of the monitor web server."),
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ce1a649/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index 880e2db..d4c0b32 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -112,6 +112,8 @@ public class ServerConstants {
public static final String TABLE_DIR = "tables";
public static final String RECOVERY_DIR = "recovery";
public static final String WAL_DIR = "wal";
+ public static final String WALOG_ARCHIVE_DIR = "walogArchive";
+ public static final String FILE_ARCHIVE_DIR = "fileArchive";
public static String[] getTablesDirs() {
return VolumeConfiguration.prefix(getBaseUris(), TABLE_DIR);
@@ -126,7 +128,11 @@ public class ServerConstants {
}
public static String[] getWalogArchives() {
- return VolumeConfiguration.prefix(getBaseUris(), "walogArchive");
+ return VolumeConfiguration.prefix(getBaseUris(), WALOG_ARCHIVE_DIR);
+ }
+
+ public static String[] getFileArchives() {
+ return VolumeConfiguration.prefix(getBaseUris(), FILE_ARCHIVE_DIR);
}
public static Path getInstanceIdLocation(Volume v) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ce1a649/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 9b4af58..308d7b9 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -70,6 +70,7 @@ import org.apache.accumulo.core.util.ServerServices;
import org.apache.accumulo.core.util.ServerServices.Service;
import org.apache.accumulo.core.util.SslConnectionParams;
import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
@@ -225,6 +226,15 @@ public class SimpleGarbageCollector implements Iface {
}
/**
+ * Should files be archived (as opposed to preserved in trash)
+ *
+ * @return True if files should be archived, false otherwise
+ */
+ boolean shouldArchiveFiles() {
+ return config.getBoolean(Property.GC_FILE_ARCHIVE);
+ }
+
+ /**
* Initializes this garbage collector.
*
* @param fs
@@ -597,13 +607,68 @@ public class SimpleGarbageCollector implements Iface {
* if the volume manager encountered a problem
*/
boolean moveToTrash(Path path) throws IOException {
- if (!isUsingTrash())
- return false;
- try {
- return fs.moveToTrash(path);
- } catch (FileNotFoundException ex) {
+ if (shouldArchiveFiles()) {
+ return archiveFile(path);
+ } else {
+ if (!isUsingTrash())
+ return false;
+ try {
+ return fs.moveToTrash(path);
+ } catch (FileNotFoundException ex) {
+ return false;
+ }
+ }
+ }
+
+ /**
+ * Move a file, that would otherwise be deleted, to the archive directory for files
+ *
+ * @param fileToArchive
+ * Path to file that is to be archived
+ * @return True if the file was successfully moved to the file archive directory, false otherwise
+ */
+ boolean archiveFile(Path fileToArchive) throws IOException {
+ // Figure out what the base path this volume uses on this FileSystem
+ Volume sourceVolume = fs.getVolumeByPath(fileToArchive);
+ String sourceVolumeBasePath = sourceVolume.getBasePath();
+
+ log.debug("Base path for volume: " + sourceVolumeBasePath);
+
+ // Get the path for the file we want to archive
+ String sourcePathBasePath = fileToArchive.toUri().getPath();
+
+ // Strip off the common base path for the file to archive
+ String relativeVolumePath = sourcePathBasePath.substring(sourceVolumeBasePath.length());
+ if (Path.SEPARATOR_CHAR == relativeVolumePath.charAt(0)) {
+ if (relativeVolumePath.length() > 1) {
+ relativeVolumePath = relativeVolumePath.substring(1);
+ } else {
+ relativeVolumePath = "";
+ }
+ }
+
+ log.debug("Computed relative path for file to archive: " + relativeVolumePath);
+
+ // The file archive path on this volume (we can't archive this file to a different volume)
+ Path archivePath = new Path(sourceVolumeBasePath, ServerConstants.FILE_ARCHIVE_DIR);
+
+ log.debug("File archive path: " + archivePath);
+
+ fs.mkdirs(archivePath);
+
+ // Preserve the path beneath the Volume's base directory (e.g. tables/1/A_0000001.rf)
+ Path fileArchivePath = new Path(archivePath, relativeVolumePath);
+
+ log.debug("Create full path of " + fileArchivePath + " from " + archivePath + " and " + relativeVolumePath);
+
+ // Make sure that it doesn't already exist, something is wrong.
+ if (fs.exists(fileArchivePath)) {
+ log.warn("Tried to archive file, but it already exists: " + fileArchivePath);
return false;
}
+
+ log.debug("Moving " + fileToArchive + " to " + fileArchivePath);
+ return fs.rename(fileToArchive, fileArchivePath);
}
private void getZooLock(HostAndPort addr) throws KeeperException, InterruptedException {