You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/11/13 17:38:25 UTC
git commit: ACCUMULO-1831 ACCUMULO-1888 use uuids to confirm WALog GC;
remove WALog entries in the !METADATA table with the correct mutation
Updated Branches:
refs/heads/1.6.0-SNAPSHOT ed11bd3d0 -> 0e63755d5
ACCUMULO-1831 ACCUMULO-1888 use uuids to confirm WALog GC; remove WALog entries in the !METADATA table with the correct mutation
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0e63755d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0e63755d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0e63755d
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 0e63755d5e3a9e441bae51d2f28402a3768820dc
Parents: ed11bd3
Author: Eric Newton <er...@gmail.com>
Authored: Wed Nov 13 11:38:34 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Wed Nov 13 11:38:34 2013 -0500
----------------------------------------------------------------------
.../server/util/MasterMetadataUtil.java | 3 +-
.../accumulo/server/util/MetadataTableUtil.java | 2 +-
.../gc/GarbageCollectWriteAheadLogs.java | 34 ++++++++++++--------
.../apache/accumulo/tserver/log/DfsLogger.java | 5 ++-
4 files changed, 26 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0e63755d/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
index 987eba9..e936b97 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
@@ -36,7 +36,6 @@ import org.apache.accumulo.core.data.PartialKey;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
@@ -243,7 +242,7 @@ public class MasterMetadataUtil {
*/
public static void updateTabletDataFile(KeyExtent extent, FileRef path, FileRef mergeFile, DataFileValue dfv, String time, Credentials credentials,
Set<FileRef> filesInUseByScans, String address, ZooLock zooLock, Set<String> unusedWalLogs, TServerInstance lastLocation, long flushId) {
- if (extent.equals(RootTable.EXTENT)) {
+ if (extent.isRootTablet()) {
if (unusedWalLogs != null) {
IZooReaderWriter zk = ZooReaderWriter.getInstance();
// unusedWalLogs will contain the location/name of each log in a log set
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0e63755d/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index cbe42b7..76ea3b1 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -686,7 +686,7 @@ public class MetadataTableUtil {
}
} else {
Mutation m = new Mutation(entry.extent.getMetadataEntry());
- m.putDelete(LogColumnFamily.NAME, new Text(entry.server + "/" + entry.filename));
+ m.putDelete(LogColumnFamily.NAME, new Text(entry.toString()));
update(SystemCredentials.get(), zooLock, m, entry.extent);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0e63755d/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 98b7bd6..6eed80d 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@ -79,7 +79,8 @@ public class GarbageCollectWriteAheadLogs {
status.currentLog.started = System.currentTimeMillis();
Map<Path,String> fileToServerMap = new HashMap<Path,String>();
- int count = scanServers(fileToServerMap);
+ Map<String,Path> nameToFileMap = new HashMap<String, Path>();
+ int count = scanServers(fileToServerMap, nameToFileMap);
long fileScanStop = System.currentTimeMillis();
log.info(String.format("Fetched %d files from %d servers in %.2f seconds", fileToServerMap.size(), count,
(fileScanStop - status.currentLog.started) / 1000.));
@@ -88,7 +89,7 @@ public class GarbageCollectWriteAheadLogs {
span = Trace.start("removeMetadataEntries");
try {
- count = removeMetadataEntries(fileToServerMap, sortedWALogs, status);
+ count = removeMetadataEntries(nameToFileMap, sortedWALogs, status);
} catch (Exception ex) {
log.error("Unable to scan metadata table", ex);
return;
@@ -100,9 +101,9 @@ public class GarbageCollectWriteAheadLogs {
log.info(String.format("%d log entries scanned in %.2f seconds", count, (logEntryScanStop - fileScanStop) / 1000.));
span = Trace.start("removeFiles");
- Map<String,ArrayList<Path>> serverToFileMap = mapServersToFiles(fileToServerMap);
+ Map<String,ArrayList<Path>> serverToFileMap = mapServersToFiles(fileToServerMap, nameToFileMap);
- count = removeFiles(serverToFileMap, sortedWALogs, status);
+ count = removeFiles(nameToFileMap, serverToFileMap, sortedWALogs, status);
long removeStop = System.currentTimeMillis();
log.info(String.format("%d total logs removed from %d servers in %.2f seconds", count, serverToFileMap.size(), (removeStop - logEntryScanStop) / 1000.));
@@ -131,7 +132,7 @@ public class GarbageCollectWriteAheadLogs {
}
}
- private int removeFiles(Map<String,ArrayList<Path>> serverToFileMap, Set<Path> sortedWALogs, final GCStatus status) {
+ private int removeFiles(Map<String,Path> nameToFileMap, Map<String,ArrayList<Path>> serverToFileMap, Set<Path> sortedWALogs, final GCStatus status) {
AccumuloConfiguration conf = instance.getConfiguration();
for (Entry<String,ArrayList<Path>> entry : serverToFileMap.entrySet()) {
if (entry.getKey().isEmpty()) {
@@ -210,9 +211,11 @@ public class GarbageCollectWriteAheadLogs {
return result;
}
- private static Map<String,ArrayList<Path>> mapServersToFiles(Map<Path,String> fileToServerMap) {
+ private static Map<String,ArrayList<Path>> mapServersToFiles(Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) {
Map<String,ArrayList<Path>> result = new HashMap<String,ArrayList<Path>>();
for (Entry<Path,String> fileServer : fileToServerMap.entrySet()) {
+ if (!nameToFileMap.containsKey(fileServer.getKey().getName()))
+ continue;
ArrayList<Path> files = result.get(fileServer.getValue());
if (files == null) {
files = new ArrayList<Path>();
@@ -223,7 +226,7 @@ public class GarbageCollectWriteAheadLogs {
return result;
}
- private int removeMetadataEntries(Map<Path,String> fileToServerMap, Set<Path> sortedWALogs, GCStatus status) throws IOException, KeeperException,
+ private int removeMetadataEntries(Map<String,Path> nameToFileMap, Set<Path> sortedWALogs, GCStatus status) throws IOException, KeeperException,
InterruptedException {
int count = 0;
Iterator<LogEntry> iterator = MetadataTableUtil.getLogEntries(SystemCredentials.get());
@@ -237,11 +240,11 @@ public class GarbageCollectWriteAheadLogs {
else
path = fs.getFullPath(FileType.WAL, filename);
- if (fileToServerMap.remove(path) != null)
+ Path pathFromNN = nameToFileMap.remove(path.getName());
+ if (pathFromNN != null) {
status.currentLog.inUse++;
-
- sortedWALogs.remove(path);
-
+ sortedWALogs.remove(pathFromNN);
+ }
count++;
}
}
@@ -250,7 +253,7 @@ public class GarbageCollectWriteAheadLogs {
//TODO Remove deprecation warning suppression when Hadoop1 support is dropped
@SuppressWarnings("deprecation")
- private int scanServers(Map<Path,String> fileToServerMap) throws Exception {
+ private int scanServers(Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) throws Exception {
Set<String> servers = new HashSet<String>();
for (String walDir : ServerConstants.getWalDirs()) {
Path walRoot = new Path(walDir);
@@ -262,9 +265,10 @@ public class GarbageCollectWriteAheadLogs {
servers.add(server);
if (status.isDir()) {
for (FileStatus file : fs.listStatus(new Path(walRoot, server))) {
- if (isUUID(file.getPath().getName()))
+ if (isUUID(file.getPath().getName())) {
fileToServerMap.put(file.getPath(), server);
- else {
+ nameToFileMap.put(file.getPath().getName(), file.getPath());
+ } else {
log.info("Ignoring file " + file.getPath() + " because it doesn't look like a uuid");
}
}
@@ -276,6 +280,8 @@ public class GarbageCollectWriteAheadLogs {
}
}
}
+ log.debug("fileToServerMap " + fileToServerMap);
+ log.debug("nameToFileMap " + nameToFileMap);
return servers.size();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0e63755d/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
index 612678a..eee3f68 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
@@ -407,7 +407,10 @@ public class DfsLogger {
@Override
public String toString() {
- return getLogger() + "/" + getFileName();
+ String fileName = getFileName();
+ if (fileName.contains(":"))
+ return getLogger() + "/" + getFileName();
+ return fileName;
}
public String getFileName() {