You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/03/11 19:25:30 UTC
[21/23] git commit: ACCUMULO-2061 Remove implicit assumptions about
'/accumulo' directory that all volumes would have.
ACCUMULO-2061 Remove implicit assumptions about '/accumulo' directory that all volumes would have.
Added some extra logging at trace to help if this needs to be revisited.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/039989ad
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/039989ad
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/039989ad
Branch: refs/heads/ACCUMULO-2061
Commit: 039989ad44bb9b46d620a3e302e15c32b3b3840a
Parents: c3214e0
Author: Josh Elser <el...@apache.org>
Authored: Tue Mar 11 13:57:48 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Mar 11 13:57:48 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/server/ServerConstants.java | 4 +-
.../accumulo/server/fs/VolumeManager.java | 3 +-
.../apache/accumulo/server/fs/VolumeUtil.java | 20 ++++-
.../accumulo/server/fs/VolumeUtilTest.java | 89 ++++++++++++++++++--
4 files changed, 103 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index 6c27897..7dd0a08 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -187,8 +187,8 @@ public class ServerConstants {
HashSet<Path> baseDirs = new HashSet<Path>();
for (String baseDir : getBaseUris()) {
- // normalize using path and remove accumulo dir
- baseDirs.add(new Path(baseDir).getParent());
+ // normalize using path
+ baseDirs.add(new Path(baseDir));
}
for (Pair<Path,Path> pair : ret)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
index c345ede..9b8fb98 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
@@ -48,9 +48,10 @@ public interface VolumeManager {
}
private static int endOfVolumeIndex(String path, String dir) {
+ // Strip off the suffix that starts with the FileType (e.g. tables, wal, etc)
int dirIndex = path.indexOf('/' + dir);
if (dirIndex != -1) {
- return path.lastIndexOf('/', dirIndex - 1);
+ return dirIndex;
}
if (path.contains(":"))
http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index 110f1e4..bcfb008 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -81,8 +81,10 @@ public class VolumeUtil {
}
public static String switchVolume(String path, FileType ft, List<Pair<Path,Path>> replacements) {
- if (replacements.size() == 0)
+ if (replacements.size() == 0) {
+ log.trace("Not switching volume because there are no replacements");
return null;
+ }
Path p = new Path(path);
@@ -92,10 +94,15 @@ public class VolumeUtil {
for (Pair<Path,Path> pair : replacements) {
Path key = removeTrailingSlash(pair.getFirst());
- if (key.equals(volume))
- return new Path(pair.getSecond(), ft.removeVolume(p)).toString();
+ if (key.equals(volume)) {
+ String replacement = new Path(pair.getSecond(), ft.removeVolume(p)).toString();
+ log.trace("Replacing " + path + " with " + replacement);
+ return replacement;
+ }
}
+ log.trace("Could not find replacement for " + ft + " at " + path);
+
return null;
}
@@ -119,13 +126,17 @@ public class VolumeUtil {
}
- if (numSwitched == 0)
+ if (numSwitched == 0) {
+ log.trace("Did not switch " + le);
return null;
+ }
LogEntry newLogEntry = new LogEntry(le);
newLogEntry.filename = switchedPath;
newLogEntry.logSet = switchedLogs;
+ log.trace("Switched " + le + " to " + newLogEntry);
+
return newLogEntry;
}
@@ -165,6 +176,7 @@ public class VolumeUtil {
*/
public static TabletFiles updateTabletVolumes(ZooLock zooLock, VolumeManager vm, KeyExtent extent, TabletFiles tabletFiles) throws IOException {
List<Pair<Path,Path>> replacements = ServerConstants.getVolumeReplacements();
+ log.trace("Using volume replacements: " + replacements);
List<LogEntry> logsToRemove = new ArrayList<LogEntry>();
List<LogEntry> logsToAdd = new ArrayList<LogEntry>();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
index c85be45..3b905c9 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
@@ -43,9 +43,9 @@ public class VolumeUtilTest {
@Test
public void testSwitchVolume() {
List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/"), new Path("viewfs:/b")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/b/accumulo")));
Assert.assertEquals("viewfs:/a/accumulo/tables/t-00000/C000.rf",
VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
@@ -57,9 +57,9 @@ public class VolumeUtilTest {
Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
replacements.clear();
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/"), new Path("viewfs:/b")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/b/accumulo")));
Assert.assertEquals("viewfs:/a/accumulo/tables/t-00000/C000.rf",
VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
@@ -73,6 +73,70 @@ public class VolumeUtilTest {
}
@Test
+ public void testSwitchVolumesDifferentSourceDepths() {
+ List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/b")));
+
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/b/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/a/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+
+ replacements.clear();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/b")));
+
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/b/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/d2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/a/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ }
+
+ @Test
+ public void testSwitchVolumesDifferentTargetDepths() {
+ List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/path3")));
+
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path3/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/path1/path2/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+
+ replacements.clear();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/path3")));
+
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path3/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/d2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/path1/path2/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ }
+
+ @Test
public void testSame() throws Exception {
FileSystem fs = FileSystem.getLocal(new Configuration());
@@ -133,6 +197,19 @@ public class VolumeUtilTest {
}
+ @Test
+ public void testRootTableReplacement() throws IOException {
+ List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
+ replacements.add(new Pair<Path,Path>(new Path("file:/foo/v1"), new Path("file:/foo/v8")));
+ replacements.add(new Pair<Path,Path>(new Path("file:/foo/v2"), new Path("file:/foo/v9")));
+
+ FileType ft = FileType.TABLE;
+
+ Assert.assertEquals("file:/foo/v8/tables/+r/root_tablet",
+ VolumeUtil.switchVolume("file:/foo/v1/tables/+r/root_tablet",
+ ft, replacements));
+ }
+
private void writeFile(FileSystem fs, Path dir, String filename, String data) throws IOException {
FSDataOutputStream out = fs.create(new Path(dir, filename));
try {