You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by bh...@apache.org on 2014/04/07 23:15:07 UTC
[2/3] git commit: ACCUMULO-2558 Add unit tests for server/gc
ACCUMULO-2558 Add unit tests for server/gc
Code changes accompanying unit tests:
- visibility of methods under test changed from private to package
- some methods refactored to take arguments, to permit testability
- GarbageCollectWriteAheadLogs.isUUID() improved to check for null and to check
length of potential UUID, since UUID.fromString() will accept UUIDs with extraneous
leading zeroes in components
- SimpleGarbageCollector.getZooLock() finds ZK root using instance field of class
instead of calling HdfsZooInstance.getInstance() again
- SimpleGarbageCollector.init() no longer declares throwing IOException
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/88f24d23
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/88f24d23
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/88f24d23
Branch: refs/heads/master
Commit: 88f24d2323efd1aa33de1bf2cd5a0241cbd78306
Parents: 8d6a4cf
Author: Bill Havanki <bh...@cloudera.com>
Authored: Thu Mar 27 11:27:15 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Mon Apr 7 17:05:03 2014 -0400
----------------------------------------------------------------------
server/gc/pom.xml | 5 +
.../gc/GarbageCollectWriteAheadLogs.java | 89 ++++++-
.../accumulo/gc/SimpleGarbageCollector.java | 134 +++++++++-
.../gc/GarbageCollectWriteAheadLogsTest.java | 264 +++++++++++++++++++
.../gc/SimpleGarbageCollectorOptsTest.java | 37 +++
.../accumulo/gc/SimpleGarbageCollectorTest.java | 146 ++++++++++
6 files changed, 657 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/88f24d23/server/gc/pom.xml
----------------------------------------------------------------------
diff --git a/server/gc/pom.xml b/server/gc/pom.xml
index 6f95273..33cc596 100644
--- a/server/gc/pom.xml
+++ b/server/gc/pom.xml
@@ -73,6 +73,11 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/88f24d23/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index ab2ab42..ae850af 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@ -64,12 +64,44 @@ public class GarbageCollectWriteAheadLogs {
private boolean useTrash;
+ /**
+ * Creates a new GC WAL object.
+ *
+ * @param instance instance to use
+ * @param fs volume manager to use
+ * @param useTrash true to move files to trash rather than delete them
+ */
GarbageCollectWriteAheadLogs(Instance instance, VolumeManager fs, boolean useTrash) throws IOException {
this.instance = instance;
this.fs = fs;
this.useTrash = useTrash;
}
+ /**
+ * Gets the instance used by this object.
+ *
+ * @return instance
+ */
+ Instance getInstance() {
+ return instance;
+ }
+ /**
+ * Gets the volume manager used by this object.
+ *
+ * @return volume manager
+ */
+ VolumeManager getVolumeManager() {
+ return fs;
+ }
+ /**
+ * Checks if the volume manager should move files to the trash rather than
+ * delete them.
+ *
+ * @return true if trash is used
+ */
+ boolean isUsingTrash() {
+ return useTrash;
+ }
public void collect(GCStatus status) {
Span span = Trace.start("scanServers");
@@ -205,14 +237,29 @@ public class GarbageCollectWriteAheadLogs {
return 0;
}
- private List<String> paths2strings(ArrayList<Path> paths) {
+ /**
+ * Converts a list of paths to their corresponding strings.
+ *
+ * @param paths list of paths
+ * @return string forms of paths
+ */
+ static List<String> paths2strings(List<Path> paths) {
List<String> result = new ArrayList<String>(paths.size());
for (Path path : paths)
result.add(path.toString());
return result;
}
- private static Map<String,ArrayList<Path>> mapServersToFiles(Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) {
+ /**
+ * Reverses the given mapping of file paths to servers. The returned map
+ * provides a list of file paths for each server. Any path whose name is not
+ * in the mapping of file names to paths is skipped.
+ *
+ * @param fileToServerMap map of file paths to servers
+ * @param nameToFileMap map of file names to paths
+ * @return map of servers to lists of file paths
+ */
+ static Map<String,ArrayList<Path>> mapServersToFiles(Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) {
Map<String,ArrayList<Path>> result = new HashMap<String,ArrayList<Path>>();
for (Entry<Path,String> fileServer : fileToServerMap.entrySet()) {
if (!nameToFileMap.containsKey(fileServer.getKey().getName()))
@@ -251,11 +298,23 @@ public class GarbageCollectWriteAheadLogs {
return count;
}
+ private int scanServers(Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) throws Exception {
+ return scanServers(ServerConstants.getWalDirs(), fileToServerMap, nameToFileMap);
+ }
//TODO Remove deprecation warning suppression when Hadoop1 support is dropped
@SuppressWarnings("deprecation")
- private int scanServers(Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) throws Exception {
+ /**
+ * Scans write-ahead log directories for logs. The maps passed in are
+ * populated with scan information.
+ *
+ * @param walDirs write-ahead log directories
+ * @param fileToServerMap map of file paths to servers
+ * @param nameToFileMap map of file names to paths
+ * @return number of servers located (including those with no logs present)
+ */
+ int scanServers(String[] walDirs, Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) throws Exception {
Set<String> servers = new HashSet<String>();
- for (String walDir : ServerConstants.getWalDirs()) {
+ for (String walDir : walDirs) {
Path walRoot = new Path(walDir);
FileStatus[] listing = null;
try {
@@ -290,9 +349,18 @@ public class GarbageCollectWriteAheadLogs {
}
private Map<String, Path> getSortedWALogs() throws IOException {
+ return getSortedWALogs(ServerConstants.getRecoveryDirs());
+ }
+ /**
+ * Looks for write-ahead logs in recovery directories.
+ *
+ * @param recoveryDirs recovery directories
+ * @return map of log file names to paths
+ */
+ Map<String, Path> getSortedWALogs(String[] recoveryDirs) throws IOException {
Map<String, Path> result = new HashMap<String, Path>();
- for (String dir : ServerConstants.getRecoveryDirs()) {
+ for (String dir : recoveryDirs) {
Path recoveryDir = new Path(dir);
if (fs.exists(recoveryDir)) {
@@ -309,7 +377,16 @@ public class GarbageCollectWriteAheadLogs {
return result;
}
- static private boolean isUUID(String name) {
+ /**
+ * Checks if a string is a valid UUID.
+ *
+ * @param name string to check
+ * @return true if string is a UUID
+ */
+ static boolean isUUID(String name) {
+ if (name == null || name.length() != 36) {
+ return false;
+ }
try {
UUID.fromString(name);
return true;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/88f24d23/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 89925b4..39716eb 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -108,6 +108,9 @@ import com.google.common.net.HostAndPort;
public class SimpleGarbageCollector implements Iface {
private static final Text EMPTY_TEXT = new Text();
+ /**
+ * Options for the garbage collector.
+ */
static class Opts extends ServerOpts {
@Parameter(names = {"-v", "--verbose"}, description = "extra information will get printed to stdout also")
boolean verbose = false;
@@ -115,8 +118,11 @@ public class SimpleGarbageCollector implements Iface {
boolean safeMode = false;
}
- // how much of the JVM's available memory should it use gathering candidates
- private static final float CANDIDATE_MEMORY_PERCENTAGE = 0.75f;
+ /**
+ * A fraction representing how much of the JVM's available memory should be
+ * used for gathering candidates.
+ */
+ static final float CANDIDATE_MEMORY_PERCENTAGE = 0.75f;
private static final Logger log = Logger.getLogger(SimpleGarbageCollector.class);
@@ -148,18 +154,100 @@ public class SimpleGarbageCollector implements Iface {
gc.run();
}
+ /**
+ * Creates a new garbage collector.
+ *
+ * @param opts options
+ */
public SimpleGarbageCollector(Opts opts) {
this.opts = opts;
}
- public void init(VolumeManager fs, Instance instance, Credentials credentials, boolean noTrash) throws IOException {
+ /**
+ * Gets the credentials used by this GC.
+ *
+ * @return credentials
+ */
+ Credentials getCredentials() {
+ return credentials;
+ }
+ /**
+ * Gets the delay before the first collection.
+ *
+ * @return start delay, in milliseconds
+ */
+ long getStartDelay() {
+ return gcStartDelay;
+ }
+ /**
+ * Gets the volume manager used by this GC.
+ *
+ * @return volume manager
+ */
+ VolumeManager getVolumeManager() {
+ return fs;
+ }
+ /**
+ * Checks if the volume manager should move files to the trash rather than
+ * delete them.
+ *
+ * @return true if trash is used
+ */
+ boolean isUsingTrash() {
+ return useTrash;
+ }
+ /**
+ * Gets the options for this garbage collector.
+ */
+ Opts getOpts() {
+ return opts;
+ }
+ /**
+ * Gets the number of threads used for deleting files.
+ *
+ * @return number of delete threads
+ */
+ int getNumDeleteThreads() {
+ return numDeleteThreads;
+ }
+ /**
+ * Gets the instance used by this GC.
+ *
+ * @return instance
+ */
+ Instance getInstance() {
+ return instance;
+ }
+
+ /**
+ * Initializes this garbage collector with the current system configuration.
+ *
+ * @param fs volume manager
+ * @param instance instance
+ * @param credentials credentials
+ * @param noTrash true to not move files to trash instead of deleting
+ */
+ public void init(VolumeManager fs, Instance instance, Credentials credentials, boolean noTrash) {
+ init(fs, instance, credentials, noTrash, ServerConfiguration.getSystemConfiguration(instance));
+ }
+
+ /**
+ * Initializes this garbage collector.
+ *
+ * @param fs volume manager
+ * @param instance instance
+ * @param credentials credentials
+ * @param noTrash true to not move files to trash instead of deleting
+ * @param systemConfig system configuration
+ */
+ public void init(VolumeManager fs, Instance instance, Credentials credentials, boolean noTrash, AccumuloConfiguration systemConfig) {
this.fs = fs;
this.credentials = credentials;
this.instance = instance;
- gcStartDelay = ServerConfiguration.getSystemConfiguration(instance).getTimeInMillis(Property.GC_CYCLE_START);
- long gcDelay = ServerConfiguration.getSystemConfiguration(instance).getTimeInMillis(Property.GC_CYCLE_DELAY);
- numDeleteThreads = ServerConfiguration.getSystemConfiguration(instance).getCount(Property.GC_DELETE_THREADS);
+ gcStartDelay = systemConfig.getTimeInMillis(Property.GC_CYCLE_START);
+ long gcDelay = systemConfig.getTimeInMillis(Property.GC_CYCLE_DELAY);
+ numDeleteThreads = systemConfig.getCount(Property.GC_DELETE_THREADS);
log.info("start delay: " + gcStartDelay + " milliseconds");
log.info("time delay: " + gcDelay + " milliseconds");
log.info("safemode: " + opts.safeMode);
@@ -194,7 +282,7 @@ public class SimpleGarbageCollector implements Iface {
for (Entry<Key,Value> entry : scanner) {
String cand = entry.getKey().getRow().toString().substring(MetadataSchema.DeletesSection.getRowPrefix().length());
result.add(cand);
- if (almostOutOfMemory()) {
+ if (almostOutOfMemory(Runtime.getRuntime())) {
log.info("List of delete candidates has exceeded the memory threshold. Attempting to delete what has been gathered so far.");
break;
}
@@ -506,7 +594,16 @@ public class SimpleGarbageCollector implements Iface {
}
}
- private boolean moveToTrash(Path path) throws IOException {
+ /**
+ * Moves a file to trash. If this garbage collector is not using trash, this
+ * method returns false and leaves the file alone. If the file is missing,
+ * this method returns false as opposed to throwing an exception.
+ *
+ * @param path
+ * @return true if the file was moved to trash
+ * @throws IOException if the volume manager encountered a problem
+ */
+ boolean moveToTrash(Path path) throws IOException {
if (!useTrash)
return false;
try {
@@ -517,7 +614,7 @@ public class SimpleGarbageCollector implements Iface {
}
private void getZooLock(HostAndPort addr) throws KeeperException, InterruptedException {
- String path = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZGC_LOCK;
+ String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
LockWatcher lockWatcher = new LockWatcher() {
@Override
@@ -563,8 +660,14 @@ public class SimpleGarbageCollector implements Iface {
}
}
- static public boolean almostOutOfMemory() {
- Runtime runtime = Runtime.getRuntime();
+ /**
+ * Checks if the system is almost out of memory.
+ *
+ * @param runtime Java runtime
+ * @return true if system is almost out of memory
+ * @see #CANDIDATE_MEMORY_PERCENTAGE
+ */
+ static boolean almostOutOfMemory(Runtime runtime) {
return runtime.totalMemory() - runtime.freeMemory() > CANDIDATE_MEMORY_PERCENTAGE * runtime.maxMemory();
}
@@ -576,7 +679,14 @@ public class SimpleGarbageCollector implements Iface {
writer.addMutation(m);
}
- private boolean isDir(String delete) {
+ /**
+ * Checks if the given string is a directory.
+ *
+ * @param delete possible directory
+ * @return true if string is a directory
+ */
+ static boolean isDir(String delete) {
+ if (delete == null) { return false; }
int slashCount = 0;
for (int i = 0; i < delete.length(); i++)
if (delete.charAt(i) == '/')
http://git-wip-us.apache.org/repos/asf/accumulo/blob/88f24d23/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
new file mode 100644
index 0000000..f90b965
--- /dev/null
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.gc;
+
+import java.io.FileNotFoundException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+
+public class GarbageCollectWriteAheadLogsTest {
+ private static final long BLOCK_SIZE = 64000000L;
+
+ private static final Path DIR_1_PATH = new Path("/dir1");
+ private static final Path DIR_2_PATH = new Path("/dir2");
+ private static final Path DIR_3_PATH = new Path("/dir3");
+ private static final String UUID1 = UUID.randomUUID().toString();
+ private static final String UUID2 = UUID.randomUUID().toString();
+ private static final String UUID3 = UUID.randomUUID().toString();
+
+ private Instance instance;
+ private VolumeManager volMgr;
+ private GarbageCollectWriteAheadLogs gcwal;
+ private long modTime;
+
+ @Before
+ public void setUp() throws Exception {
+ instance = createMock(Instance.class);
+ volMgr = createMock(VolumeManager.class);
+ gcwal = new GarbageCollectWriteAheadLogs(instance, volMgr, false);
+ modTime = System.currentTimeMillis();
+ }
+
+ @Test
+ public void testGetters() {
+ assertSame(instance, gcwal.getInstance());
+ assertSame(volMgr, gcwal.getVolumeManager());
+ assertFalse(gcwal.isUsingTrash());
+ }
+
+ @Test
+ public void testPathsToStrings() {
+ ArrayList<Path> paths = new ArrayList<Path>();
+ paths.add(new Path(DIR_1_PATH, "file1"));
+ paths.add(DIR_2_PATH);
+ paths.add(new Path(DIR_3_PATH, "file3"));
+ List<String> strings = GarbageCollectWriteAheadLogs.paths2strings(paths);
+ int len = 3;
+ assertEquals(len, strings.size());
+ for (int i = 0; i < len; i++) {
+ assertEquals(paths.get(i).toString(), strings.get(i));
+ }
+ }
+
+ @Test
+ public void testMapServersToFiles() {
+ /*
+ * Test fileToServerMap:
+ * /dir1/server1/uuid1 -> server1 (new-style)
+ * /dir1/uuid2 -> "" (old-style)
+ * /dir3/server3/uuid3 -> server3 (new-style)
+ */
+ Map<Path,String> fileToServerMap = new java.util.HashMap<Path,String>();
+ Path path1 = new Path(new Path(DIR_1_PATH, "server1"), UUID1);
+ fileToServerMap.put(path1, "server1"); // new-style
+ Path path2 = new Path(DIR_1_PATH, UUID2);
+ fileToServerMap.put(path2, ""); // old-style
+ Path path3 = new Path(new Path(DIR_3_PATH, "server3"), UUID3);
+ fileToServerMap.put(path3, "server3"); // old-style
+ /*
+ * Test nameToFileMap:
+ * uuid1 -> /dir1/server1/uuid1
+ * uuid3 -> /dir3/server3/uuid3
+ */
+ Map<String,Path> nameToFileMap = new java.util.HashMap<String,Path>();
+ nameToFileMap.put(UUID1, path1);
+ nameToFileMap.put(UUID3, path3);
+
+ /**
+ * Expected map:
+ * server1 -> [ /dir1/server1/uuid1 ]
+ * server3 -> [ /dir3/server3/uuid3 ]
+ */
+ Map<String,ArrayList<Path>> result = GarbageCollectWriteAheadLogs.mapServersToFiles(fileToServerMap, nameToFileMap);
+ assertEquals(2, result.size());
+ ArrayList<Path> list1 = result.get("server1");
+ assertEquals(1, list1.size());
+ assertTrue(list1.contains(path1));
+ ArrayList<Path> list3 = result.get("server3");
+ assertEquals(1, list3.size());
+ assertTrue(list3.contains(path3));
+ }
+
+ private FileStatus makeFileStatus(int size, Path path) {
+ boolean isDir = (size == 0);
+ return new FileStatus(size, isDir, 3, BLOCK_SIZE, modTime, path);
+ }
+ private void mockListStatus(Path dir, FileStatus... fileStatuses) throws Exception {
+ expect(volMgr.listStatus(dir)).andReturn(fileStatuses);
+ }
+
+ @Test
+ public void testScanServers_NewStyle() throws Exception {
+ String[] walDirs = new String[] {"/dir1", "/dir2", "/dir3"};
+ /*
+ * Test directory layout:
+ * /dir1/
+ * server1/
+ * uuid1
+ * file2
+ * subdir2/
+ * /dir2/ missing
+ * /dir3/
+ * server3/
+ * uuid3
+ */
+ Path serverDir1Path = new Path(DIR_1_PATH, "server1");
+ FileStatus serverDir1 = makeFileStatus(0, serverDir1Path);
+ Path subDir2Path = new Path(DIR_1_PATH, "subdir2");
+ FileStatus serverDir2 = makeFileStatus(0, subDir2Path);
+ mockListStatus(DIR_1_PATH, serverDir1, serverDir2);
+ Path path1 = new Path(serverDir1Path, UUID1);
+ FileStatus file1 = makeFileStatus(100, path1);
+ FileStatus file2 = makeFileStatus(200, new Path(serverDir1Path, "file2"));
+ mockListStatus(serverDir1Path, file1, file2);
+ mockListStatus(subDir2Path);
+ expect(volMgr.listStatus(DIR_2_PATH)).andThrow(new FileNotFoundException());
+ Path serverDir3Path = new Path(DIR_3_PATH, "server3");
+ FileStatus serverDir3 = makeFileStatus(0, serverDir3Path);
+ mockListStatus(DIR_3_PATH, serverDir3);
+ Path path3 = new Path(serverDir3Path, UUID3);
+ FileStatus file3 = makeFileStatus(300, path3);
+ mockListStatus(serverDir3Path, file3);
+ replay(volMgr);
+
+ Map<Path,String> fileToServerMap = new java.util.HashMap<Path,String>();
+ Map<String,Path> nameToFileMap = new java.util.HashMap<String,Path>();
+ int count = gcwal.scanServers(walDirs, fileToServerMap, nameToFileMap);
+ assertEquals(3, count);
+ /*
+ * Expected fileToServerMap:
+ * /dir1/server1/uuid1 -> server1
+ * /dir3/server3/uuid3 -> server3
+ */
+ assertEquals(2, fileToServerMap.size());
+ assertEquals("server1", fileToServerMap.get(path1));
+ assertEquals("server3", fileToServerMap.get(path3));
+ /*
+ * Expected nameToFileMap:
+ * uuid1 -> /dir1/server1/uuid1
+ * uuid3 -> /dir3/server3/uuid3
+ */
+ assertEquals(2, nameToFileMap.size());
+ assertEquals(path1, nameToFileMap.get(UUID1));
+ assertEquals(path3, nameToFileMap.get(UUID3));
+ }
+
+ @Test
+ public void testScanServers_OldStyle() throws Exception {
+ /*
+ * Test directory layout:
+ * /dir1/
+ * uuid1
+ * /dir3/
+ * uuid3
+ */
+ String[] walDirs = new String[] {"/dir1", "/dir3"};
+ Path serverFile1Path = new Path(DIR_1_PATH, UUID1);
+ FileStatus serverFile1 = makeFileStatus(100, serverFile1Path);
+ mockListStatus(DIR_1_PATH, serverFile1);
+ Path serverFile3Path = new Path(DIR_3_PATH, UUID3);
+ FileStatus serverFile3 = makeFileStatus(300, serverFile3Path);
+ mockListStatus(DIR_3_PATH, serverFile3);
+ replay(volMgr);
+
+ Map<Path,String> fileToServerMap = new java.util.HashMap<Path,String>();
+ Map<String,Path> nameToFileMap = new java.util.HashMap<String,Path>();
+ int count = gcwal.scanServers(walDirs, fileToServerMap, nameToFileMap);
+ /*
+ * Expected fileToServerMap:
+ * /dir1/uuid1 -> ""
+ * /dir3/uuid3 -> ""
+ */
+ assertEquals(2, count);
+ assertEquals(2, fileToServerMap.size());
+ assertEquals("", fileToServerMap.get(serverFile1Path));
+ assertEquals("", fileToServerMap.get(serverFile3Path));
+ /*
+ * Expected nameToFileMap: empty
+ */
+ assertEquals(0, nameToFileMap.size());
+ }
+
+ @Test
+ public void testGetSortedWALogs() throws Exception {
+ String[] recoveryDirs = new String[] {"/dir1", "/dir2", "/dir3"};
+ /*
+ * Test directory layout:
+ * /dir1/
+ * uuid1
+ * file2
+ * /dir2/ missing
+ * /dir3/
+ * uuid3
+ */
+ expect(volMgr.exists(DIR_1_PATH)).andReturn(true);
+ expect(volMgr.exists(DIR_2_PATH)).andReturn(false);
+ expect(volMgr.exists(DIR_3_PATH)).andReturn(true);
+ Path path1 = new Path(DIR_1_PATH, UUID1);
+ FileStatus file1 = makeFileStatus(100, path1);
+ FileStatus file2 = makeFileStatus(200, new Path(DIR_1_PATH, "file2"));
+ mockListStatus(DIR_1_PATH, file1, file2);
+ Path path3 = new Path(DIR_3_PATH, UUID3);
+ FileStatus file3 = makeFileStatus(300, path3);
+ mockListStatus(DIR_3_PATH, file3);
+ replay(volMgr);
+
+ Map<String,Path> sortedWalogs = gcwal.getSortedWALogs(recoveryDirs);
+ /**
+ * Expected map:
+ * uuid1 -> /dir1/uuid1
+ * uuid3 -> /dir3/uuid3
+ */
+ assertEquals(2, sortedWalogs.size());
+ assertEquals(path1, sortedWalogs.get(UUID1));
+ assertEquals(path3, sortedWalogs.get(UUID3));
+ }
+
+ @Test
+ public void testIsUUID() {
+ assertTrue(GarbageCollectWriteAheadLogs.isUUID(UUID.randomUUID().toString()));
+ assertFalse(GarbageCollectWriteAheadLogs.isUUID("foo"));
+ assertFalse(GarbageCollectWriteAheadLogs.isUUID("0" + UUID.randomUUID().toString()));
+ assertFalse(GarbageCollectWriteAheadLogs.isUUID(null));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/88f24d23/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorOptsTest.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorOptsTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorOptsTest.java
new file mode 100644
index 0000000..d484741
--- /dev/null
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorOptsTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.gc;
+
+import org.apache.accumulo.gc.SimpleGarbageCollector.Opts;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertFalse;
+
+public class SimpleGarbageCollectorOptsTest {
+ private Opts opts;
+
+ @Before
+ public void setUp() {
+ opts = new Opts();
+ }
+
+ @Test
+ public void testIt() {
+ assertFalse(opts.verbose);
+ assertFalse(opts.safeMode);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/88f24d23/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
new file mode 100644
index 0000000..532eeba
--- /dev/null
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.gc;
+
+import java.io.FileNotFoundException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.gc.SimpleGarbageCollector.Opts;
+import static org.apache.accumulo.gc.SimpleGarbageCollector.CANDIDATE_MEMORY_PERCENTAGE;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.hadoop.fs.Path;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+
+public class SimpleGarbageCollectorTest {
+ private VolumeManager volMgr;
+ private Instance instance;
+ private Credentials credentials;
+ private Opts opts;
+ private SimpleGarbageCollector gc;
+ private AccumuloConfiguration systemConfig;
+
+ @Before
+ public void setUp() {
+ volMgr = createMock(VolumeManager.class);
+ instance = createMock(Instance.class);
+ credentials = createMock(Credentials.class);
+
+ opts = new Opts();
+ gc = new SimpleGarbageCollector(opts);
+ systemConfig = mockSystemConfig();
+ }
+
+ @Test
+ public void testConstruction() {
+ assertSame(opts, gc.getOpts());
+ assertNotNull(gc.getStatus(createMock(TInfo.class), createMock(TCredentials.class)));
+ }
+
+ private AccumuloConfiguration mockSystemConfig() {
+ AccumuloConfiguration systemConfig = createMock(AccumuloConfiguration.class);
+ expect(systemConfig.getTimeInMillis(Property.GC_CYCLE_START)).andReturn(1000L);
+ expect(systemConfig.getTimeInMillis(Property.GC_CYCLE_DELAY)).andReturn(20000L);
+ expect(systemConfig.getCount(Property.GC_DELETE_THREADS)).andReturn(2);
+ replay(systemConfig);
+ return systemConfig;
+ }
+
+ @Test
+ public void testInit() throws Exception {
+ gc.init(volMgr, instance, credentials, false, systemConfig);
+ assertSame(volMgr, gc.getVolumeManager());
+ assertSame(instance, gc.getInstance());
+ assertSame(credentials, gc.getCredentials());
+ assertTrue(gc.isUsingTrash());
+ assertEquals(1000L, gc.getStartDelay());
+ assertEquals(2, gc.getNumDeleteThreads());
+ }
+
+ @Test
+ public void testMoveToTrash_UsingTrash() throws Exception {
+ gc.init(volMgr, instance, credentials, false, systemConfig);
+ Path path = createMock(Path.class);
+ expect(volMgr.moveToTrash(path)).andReturn(true);
+ replay(volMgr);
+ assertTrue(gc.moveToTrash(path));
+ verify(volMgr);
+ }
+
+ @Test
+ public void testMoveToTrash_UsingTrash_VolMgrFailure() throws Exception {
+ gc.init(volMgr, instance, credentials, false, systemConfig);
+ Path path = createMock(Path.class);
+ expect(volMgr.moveToTrash(path)).andThrow(new FileNotFoundException());
+ replay(volMgr);
+ assertFalse(gc.moveToTrash(path));
+ verify(volMgr);
+ }
+
+ @Test
+ public void testMoveToTrash_NotUsingTrash() throws Exception {
+ gc.init(volMgr, instance, credentials, true, systemConfig);
+ Path path = createMock(Path.class);
+ assertFalse(gc.moveToTrash(path));
+ }
+
+ @Test
+ public void testAlmostOutOfMemory_Pass() {
+ testAlmostOutOfMemory(1.0f - (CANDIDATE_MEMORY_PERCENTAGE - 0.05f), false);
+ }
+
+ @Test
+ public void testAlmostOutOfMemory_Fail() {
+ testAlmostOutOfMemory(1.0f - (CANDIDATE_MEMORY_PERCENTAGE + 0.05f), true);
+ }
+
+ private void testAlmostOutOfMemory(float freeFactor, boolean expected) {
+ Runtime runtime = createMock(Runtime.class);
+ expect(runtime.totalMemory()).andReturn(1000L);
+ expectLastCall().anyTimes();
+ expect(runtime.maxMemory()).andReturn(1000L);
+ expectLastCall().anyTimes();
+ expect(runtime.freeMemory()).andReturn((long) (freeFactor * 1000.0f));
+ expectLastCall().anyTimes();
+ replay(runtime);
+
+ assertEquals(expected, SimpleGarbageCollector.almostOutOfMemory(runtime));
+ }
+
+ @Test
+ public void testIsDir() {
+ assertTrue(SimpleGarbageCollector.isDir("/dir1"));
+ assertFalse(SimpleGarbageCollector.isDir("file1"));
+ assertFalse(SimpleGarbageCollector.isDir("/dir1/file1"));
+ assertFalse(SimpleGarbageCollector.isDir(""));
+ assertFalse(SimpleGarbageCollector.isDir(null));
+ }
+}