You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2013/12/21 04:24:53 UTC
[1/2] git commit: ACCUMULO-2048 Removed unnecessary path manipulation
in walog GC and added sanity check
Updated Branches:
refs/heads/1.6.0-SNAPSHOT 66c7848a1 -> 3aace9806
ACCUMULO-2048 Removed unnecessary path manipulation in walog GC and
added sanity check
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/bed80715
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/bed80715
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/bed80715
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: bed80715607d6b9b74ad96e3f2a7020e8ad9f149
Parents: 66c7848
Author: Keith Turner <kt...@apache.org>
Authored: Fri Dec 20 19:56:44 2013 -0500
Committer: Keith Turner <kt...@apache.org>
Committed: Fri Dec 20 22:13:49 2013 -0500
----------------------------------------------------------------------
.../gc/GarbageCollectWriteAheadLogs.java | 28 +++++++++++---------
1 file changed, 16 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bed80715/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index c4c69b4..111d1e7 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@ -42,7 +42,6 @@ import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
-import org.apache.accumulo.server.fs.VolumeManager.FileType;
import org.apache.accumulo.server.security.SystemCredentials;
import org.apache.accumulo.server.util.MetadataTableUtil;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
@@ -231,20 +230,19 @@ public class GarbageCollectWriteAheadLogs {
InterruptedException {
int count = 0;
Iterator<LogEntry> iterator = MetadataTableUtil.getLogEntries(SystemCredentials.get());
+
while (iterator.hasNext()) {
for (String entry : iterator.next().logSet) {
- String parts[] = entry.split("/", 2);
- String filename = parts[1];
- Path path;
- if (filename.contains(":"))
- path = new Path(filename);
- else
- path = fs.getFullPath(FileType.WAL, filename);
-
- Path pathFromNN = nameToFileMap.remove(path.getName());
+ String uuid = new Path(entry).getName();
+ if (!isUUID(uuid)) {
+ // fully expect this to be a uuid, if its not then something is wrong and walog GC should not proceed!
+ throw new IllegalArgumentException("Expected uuid, but got " + uuid + " from " + entry);
+ }
+
+ Path pathFromNN = nameToFileMap.remove(uuid);
if (pathFromNN != null) {
status.currentLog.inUse++;
- sortedWALogs.remove(path.getName());
+ sortedWALogs.remove(uuid);
}
count++;
}
@@ -258,7 +256,13 @@ public class GarbageCollectWriteAheadLogs {
Set<String> servers = new HashSet<String>();
for (String walDir : ServerConstants.getWalDirs()) {
Path walRoot = new Path(walDir);
- FileStatus[] listing = fs.listStatus(walRoot);
+ FileStatus[] listing = null;
+ try {
+ listing = fs.listStatus(walRoot);
+ } catch (FileNotFoundException e) {
+ // ignore dir
+ }
+
if (listing == null)
continue;
for (FileStatus status : listing) {
[2/2] git commit: ACCUMULO-2080 made tserver use uuid when checking
if a walog to be removed is in use
Posted by kt...@apache.org.
ACCUMULO-2080 made tserver use uuid when checking if a walog to be removed is in use
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3aace980
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3aace980
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3aace980
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 3aace980689967414aa2fd390c4e00dff1db11ca
Parents: bed8071
Author: Keith Turner <kt...@apache.org>
Authored: Fri Dec 20 22:12:58 2013 -0500
Committer: Keith Turner <kt...@apache.org>
Committed: Fri Dec 20 22:20:01 2013 -0500
----------------------------------------------------------------------
.../java/org/apache/accumulo/tserver/Tablet.java | 4 ++--
.../org/apache/accumulo/tserver/TabletServer.java | 18 +++++++++++-------
.../accumulo/tserver/log/TabletServerLogger.java | 6 +++---
3 files changed, 16 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3aace980/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
index d904037..f0fb524 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
@@ -3682,11 +3682,11 @@ public class Tablet {
private Set<DfsLogger> currentLogs = new HashSet<DfsLogger>();
- public Set<String> getCurrentLogs() {
+ public Set<String> getCurrentLogFiles() {
Set<String> result = new HashSet<String>();
synchronized (currentLogs) {
for (DfsLogger log : currentLogs) {
- result.add(log.toString());
+ result.add(log.getFileName());
}
}
return result;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3aace980/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 322a1a5..d2b01ac 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -2451,20 +2451,24 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
String myname = getClientAddressString();
myname = myname.replace(':', '+');
Set<String> loggers = new HashSet<String>();
- logger.getLoggers(loggers);
+ logger.getLogFiles(loggers);
+ Set<String> loggerUUIDs = new HashSet<String>();
+ for (String logger : loggers)
+ loggerUUIDs.add(new Path(logger).getName());
+
nextFile: for (String filename : filenames) {
+ String uuid = new Path(filename).getName();
// skip any log we're currently using
- for (String logger : loggers) {
- if (logger.contains(filename))
- continue nextFile;
- }
+ if (loggerUUIDs.contains(uuid))
+ continue nextFile;
+
List<Tablet> onlineTabletsCopy = new ArrayList<Tablet>();
synchronized (onlineTablets) {
onlineTabletsCopy.addAll(onlineTablets.values());
}
for (Tablet tablet : onlineTabletsCopy) {
- for (String current : tablet.getCurrentLogs()) {
- if (current.contains(filename)) {
+ for (String current : tablet.getCurrentLogFiles()) {
+ if (current.contains(uuid)) {
log.info("Attempted to delete " + filename + " from tablet " + tablet.getExtent());
continue nextFile;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3aace980/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
index 7fd2766..a276a97 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
@@ -36,9 +36,9 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.tserver.Tablet;
+import org.apache.accumulo.tserver.Tablet.CommitSession;
import org.apache.accumulo.tserver.TabletMutations;
import org.apache.accumulo.tserver.TabletServer;
-import org.apache.accumulo.tserver.Tablet.CommitSession;
import org.apache.accumulo.tserver.log.DfsLogger.LoggerOperation;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
@@ -161,11 +161,11 @@ public class TabletServerLogger {
return result[0];
}
- public void getLoggers(Set<String> loggersOut) {
+ public void getLogFiles(Set<String> loggersOut) {
logSetLock.readLock().lock();
try {
for (DfsLogger logger : loggers) {
- loggersOut.add(logger.toString());
+ loggersOut.add(logger.getFileName());
}
} finally {
logSetLock.readLock().unlock();