You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/03/21 03:13:04 UTC
[01/25] ACCUMULO-2061 Initial implementation to remove
instance.dfs.dir from usage of instance.volumes
Repository: accumulo
Updated Branches:
refs/heads/1.6.0-SNAPSHOT ef5dc4a1f -> cb2f4b580
refs/heads/master 866422d27 -> 2dbc14fa9
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java
index 205a793..ad29c19 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java
@@ -27,28 +27,50 @@ import org.junit.Test;
public class FileTypeTest {
@Test
public void testVolumeExtraction() {
- Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:/a/accumulo"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:///a/accumulo"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:///a/accumulo"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:/a/accumulo"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
+ // Having an 'accumulo' directory is not a requirement
+ Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:/a/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:/a/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:///a/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:///a/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:///accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:/a"), FileType.WAL.getVolume(new Path("file:/a/accumulo/wal/1.2.3.4/aaa-bbb-ccc-ddd")));
+ // Having an 'accumulo' directory is not a requirement
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:/a/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:///a/tables/2b/t-001/C00.rf")));
+
+ Assert.assertEquals(new Path("file:/accumulo"), FileType.TABLE.getVolume(new Path("file:/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:/accumulo"), FileType.TABLE.getVolume(new Path("file:///accumulo/tables/2b/t-001/C00.rf")));
+
+ // Having an 'accumulo' directory is not a requirement
+ Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:///tables/2b/t-001/C00.rf")));
+
+ Assert.assertEquals(new Path("file:/a"), FileType.WAL.getVolume(new Path("file:/a/wal/1.2.3.4/aaa-bbb-ccc-ddd")));
Assert.assertNull(FileType.WAL.getVolume(new Path("1.2.3.4/aaa-bbb-ccc-ddd")));
Assert.assertNull(FileType.TABLE.getVolume(new Path("../2b/t-001/C00.rf")));
Assert.assertNull(FileType.TABLE.getVolume(new Path("/t-001/C00.rf")));
- Assert.assertEquals(new Path("hdfs://nn1/"), FileType.TABLE.getVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("hdfs://nn1/a/"), FileType.TABLE.getVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("hdfs://nn1/accumulo"), FileType.TABLE.getVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("hdfs://nn1/a/accumulo"), FileType.TABLE.getVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf")));
+
+ // Having an 'accumulo' directory is not a requirement
+ Assert.assertEquals(new Path("hdfs://nn1/"), FileType.TABLE.getVolume(new Path("hdfs://nn1/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("hdfs://nn1/a"), FileType.TABLE.getVolume(new Path("hdfs://nn1/a/tables/2b/t-001/C00.rf")));
+
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf")));
+ // Having an 'accumulo' directory is not a requirement
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/a/tables/2b/t-001/C00.rf")));
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
index c85be45..3b905c9 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
@@ -43,9 +43,9 @@ public class VolumeUtilTest {
@Test
public void testSwitchVolume() {
List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/"), new Path("viewfs:/b")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/b/accumulo")));
Assert.assertEquals("viewfs:/a/accumulo/tables/t-00000/C000.rf",
VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
@@ -57,9 +57,9 @@ public class VolumeUtilTest {
Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
replacements.clear();
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/"), new Path("viewfs:/b")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/b/accumulo")));
Assert.assertEquals("viewfs:/a/accumulo/tables/t-00000/C000.rf",
VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
@@ -73,6 +73,70 @@ public class VolumeUtilTest {
}
@Test
+ public void testSwitchVolumesDifferentSourceDepths() {
+ List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/b")));
+
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/b/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/a/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+
+ replacements.clear();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/b")));
+
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/b/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/d2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/a/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ }
+
+ @Test
+ public void testSwitchVolumesDifferentTargetDepths() {
+ List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/path3")));
+
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path3/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/path1/path2/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+
+ replacements.clear();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/path3")));
+
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path3/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/d2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/path1/path2/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ }
+
+ @Test
public void testSame() throws Exception {
FileSystem fs = FileSystem.getLocal(new Configuration());
@@ -133,6 +197,19 @@ public class VolumeUtilTest {
}
+ @Test
+ public void testRootTableReplacement() throws IOException {
+ List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
+ replacements.add(new Pair<Path,Path>(new Path("file:/foo/v1"), new Path("file:/foo/v8")));
+ replacements.add(new Pair<Path,Path>(new Path("file:/foo/v2"), new Path("file:/foo/v9")));
+
+ FileType ft = FileType.TABLE;
+
+ Assert.assertEquals("file:/foo/v8/tables/+r/root_tablet",
+ VolumeUtil.switchVolume("file:/foo/v1/tables/+r/root_tablet",
+ ft, replacements));
+ }
+
private void writeFile(FileSystem fs, Path dir, String filename, String data) throws IOException {
FSDataOutputStream out = fs.create(new Path(dir, filename));
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
index 4e2a878..1e9c1dd 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
@@ -335,6 +335,71 @@ public class GarbageCollectionTest {
assertRemoved(gce);
}
+
+ @Test
+ public void testCustomDirectories() throws Exception {
+ TestGCE gce = new TestGCE();
+
+ gce.candidates.add("/4/t-0");
+ gce.candidates.add("/4/t-0/F002.rf");
+ gce.candidates.add("hdfs://foo.com:6000/user/foo/tables/5/t-0");
+ gce.candidates.add("/6/t-0");
+ gce.candidates.add("hdfs://foo:6000/user/foo/tables/7/t-0/");
+ gce.candidates.add("/8/t-0");
+ gce.candidates.add("hdfs://foo:6000/user/foo/tables/9/t-0");
+ gce.candidates.add("/a/t-0");
+ gce.candidates.add("hdfs://foo:6000/user/foo/tables/b/t-0");
+ gce.candidates.add("/c/t-0");
+ gce.candidates.add("hdfs://foo:6000/user/foo/tables/d/t-0");
+
+ gce.addDirReference("4", null, "/t-0");
+ gce.addDirReference("5", null, "/t-0");
+ gce.addDirReference("6", null, "hdfs://foo.com:6000/user/foo/tables/6/t-0");
+ gce.addDirReference("7", null, "hdfs://foo.com:6000/user/foo/tables/7/t-0");
+
+ gce.addFileReference("8", "m", "/t-0/F00.rf");
+ gce.addFileReference("9", "m", "/t-0/F00.rf");
+
+ gce.addFileReference("a", "m", "hdfs://foo.com:6000/user/foo/tables/a/t-0/F00.rf");
+ gce.addFileReference("b", "m", "hdfs://foo.com:6000/user/foo/tables/b/t-0/F00.rf");
+
+ gce.addFileReference("e", "m", "../c/t-0/F00.rf");
+ gce.addFileReference("f", "m", "../d/t-0/F00.rf");
+
+ GarbageCollectionAlgorithm gca = new GarbageCollectionAlgorithm();
+
+ // A directory reference does not preclude a candidate file beneath that directory from deletion
+ gca.collect(gce);
+ assertRemoved(gce, "/4/t-0/F002.rf");
+
+ // Removing the dir reference for a table will delete all tablet directories
+ gce.removeDirReference("5", null);
+ gca.collect(gce);
+ assertRemoved(gce, "hdfs://foo.com:6000/user/foo/tables/5/t-0");
+
+ gce.removeDirReference("4", null);
+ gca.collect(gce);
+ assertRemoved(gce, "/4/t-0");
+
+ gce.removeDirReference("6", null);
+ gce.removeDirReference("7", null);
+ gca.collect(gce);
+ assertRemoved(gce, "/6/t-0", "hdfs://foo:6000/user/foo/tables/7/t-0/");
+
+ gce.removeFileReference("8", "m", "/t-0/F00.rf");
+ gce.removeFileReference("9", "m", "/t-0/F00.rf");
+ gce.removeFileReference("a", "m", "hdfs://foo.com:6000/user/foo/tables/a/t-0/F00.rf");
+ gce.removeFileReference("b", "m", "hdfs://foo.com:6000/user/foo/tables/b/t-0/F00.rf");
+ gce.removeFileReference("e", "m", "../c/t-0/F00.rf");
+ gce.removeFileReference("f", "m", "../d/t-0/F00.rf");
+ gca.collect(gce);
+ assertRemoved(gce, "/8/t-0", "hdfs://foo:6000/user/foo/tables/9/t-0", "/a/t-0", "hdfs://foo:6000/user/foo/tables/b/t-0", "/c/t-0",
+ "hdfs://foo:6000/user/foo/tables/d/t-0");
+
+ gca.collect(gce);
+ assertRemoved(gce);
+ }
+
private void badRefTest(String ref) {
TestGCE gce = new TestGCE();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
index 36bbb53..eeb9b16 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
@@ -152,7 +152,7 @@ class WriteExportFiles extends MasterRepo {
public static void exportTable(VolumeManager fs, Connector conn, String tableName, String tableID, String exportDir) throws Exception {
fs.mkdirs(new Path(exportDir));
- Path exportMetaFilePath = fs.getFileSystemByPath(new Path(exportDir)).makeQualified(new Path(exportDir, Constants.EXPORT_FILE));
+ Path exportMetaFilePath = fs.getVolumeByPath(new Path(exportDir)).getFileSystem().makeQualified(new Path(exportDir, Constants.EXPORT_FILE));
FSDataOutputStream fileOut = fs.create(exportMetaFilePath, false);
ZipOutputStream zipOut = new ZipOutputStream(fileOut);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
index 7e84c55..cd59b78 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
@@ -427,7 +427,7 @@ class ImportPopulateZookeeper extends MasterRepo {
Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
try {
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
return TableOperationsImpl.getExportedProps(ns, path);
} catch (IOException ioe) {
throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
index 942f866..31e63ed 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
@@ -33,11 +33,11 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.conf.DefaultConfiguration;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
import org.apache.accumulo.core.util.Duration;
import org.apache.accumulo.core.util.NumUtil;
import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.monitor.Monitor;
import org.apache.accumulo.monitor.ZooKeeperStatus;
import org.apache.accumulo.monitor.ZooKeeperStatus.ZooKeeperState;
@@ -268,9 +268,9 @@ public class DefaultServlet extends BasicServlet {
long totalHdfsBytesUsed = 0l;
try {
- for (String baseDir : VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration())) {
+ for (String baseDir : VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())) {
final Path basePath = new Path(baseDir);
- final FileSystem fs = vm.getFileSystemByPath(basePath);
+ final FileSystem fs = vm.getVolumeByPath(basePath).getFileSystem();
try {
// Calculate the amount of space used by Accumulo on the FileSystem
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
index e9f1083..6b31af1 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.trace.TraceFileSystem;
import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor;
@@ -32,35 +33,35 @@ import org.apache.log4j.Logger;
* Copy failed bulk imports.
*/
public class BulkFailedCopyProcessor implements Processor {
-
+
private static final Logger log = Logger.getLogger(BulkFailedCopyProcessor.class);
-
+
@Override
public Processor newProcessor() {
return new BulkFailedCopyProcessor();
}
-
+
@Override
public void process(String workID, byte[] data) {
-
+
String paths[] = new String(data, Constants.UTF8).split(",");
-
+
Path orig = new Path(paths[0]);
Path dest = new Path(paths[1]);
Path tmp = new Path(dest.getParent(), dest.getName() + ".tmp");
-
+
try {
- FileSystem fs = TraceFileSystem.wrap(org.apache.accumulo.core.file.VolumeConfiguration.getDefaultFilesystem(CachedConfiguration.getInstance(),
- ServerConfiguration.getSiteConfiguration()));
-
+ FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(CachedConfiguration.getInstance(),
+ ServerConfiguration.getSiteConfiguration()).getFileSystem());
+
FileUtil.copy(fs, orig, fs, tmp, false, true, CachedConfiguration.getInstance());
fs.rename(tmp, dest);
log.debug("copied " + orig + " to " + dest);
} catch (IOException ex) {
try {
- FileSystem fs = TraceFileSystem.wrap(org.apache.accumulo.core.file.VolumeConfiguration.getDefaultFilesystem(CachedConfiguration.getInstance(),
- ServerConfiguration.getSiteConfiguration()));
-
+ FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(CachedConfiguration.getInstance(),
+ ServerConfiguration.getSiteConfiguration()).getFileSystem());
+
fs.create(dest).close();
log.warn(" marked " + dest + " failed", ex);
} catch (IOException e) {
@@ -69,5 +70,5 @@ public class BulkFailedCopyProcessor implements Processor {
}
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
index 151db6e..822171c 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
@@ -336,7 +336,7 @@ public class Compactor implements Callable<CompactionStats> {
thread = Thread.currentThread();
try {
FileOperations fileFactory = FileOperations.getInstance();
- FileSystem ns = this.fs.getFileSystemByPath(outputFile.path());
+ FileSystem ns = this.fs.getVolumeByPath(outputFile.path()).getFileSystem();
mfw = fileFactory.openWriter(outputFile.path().toString(), ns, ns.getConf(), acuTableConf);
Map<String,Set<ByteSequence>> lGroups;
@@ -421,7 +421,7 @@ public class Compactor implements Callable<CompactionStats> {
try {
FileOperations fileFactory = FileOperations.getInstance();
- FileSystem fs = this.fs.getFileSystemByPath(mapFile.path());
+ FileSystem fs = this.fs.getVolumeByPath(mapFile.path()).getFileSystem();
FileSKVIterator reader;
reader = fileFactory.openReader(mapFile.path().toString(), false, fs, conf, acuTableConf);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
index bb95532..8bf2517 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
@@ -309,7 +309,7 @@ public class FileManager {
if (!file.contains(":"))
throw new IllegalArgumentException("Expected uri, got : " + file);
Path path = new Path(file);
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
//log.debug("Opening "+file + " path " + path);
FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), false, ns, ns.getConf(), conf.getTableConfiguration(table.toString()),
dataCache, indexCache);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
index cc4b68d..3fe60b7 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
@@ -1260,7 +1260,7 @@ public class Tablet {
long rtime = Long.MIN_VALUE;
for (FileRef ref : datafiles.keySet()) {
Path path = ref.path();
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), tabletServer.getTableConfiguration(extent));
long maxTime = -1;
try {
@@ -2975,7 +2975,7 @@ public class Tablet {
FileOperations fileFactory = FileOperations.getInstance();
for (Entry<FileRef,DataFileValue> entry : allFiles.entrySet()) {
FileRef file = entry.getKey();
- FileSystem ns = fs.getFileSystemByPath(file.path());
+ FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
FileSKVIterator openReader = fileFactory.openReader(file.path().toString(), true, ns, ns.getConf(), this.getTableConfiguration());
try {
Key first = openReader.getFirstKey();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 475621b..6d73125 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -969,7 +969,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
Map<FileRef,MapFileInfo> fileRefMap = new HashMap<FileRef,MapFileInfo>();
for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
Path path = new Path(mapping.getKey());
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
path = ns.makeQualified(path);
fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
index 3bbb476..900600f 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
@@ -81,7 +81,7 @@ public class MajorCompactionRequest implements Cloneable {
// @TODO verify the file isn't some random file in HDFS
// @TODO ensure these files are always closed?
FileOperations fileFactory = FileOperations.getInstance();
- FileSystem ns = volumeManager.getFileSystemByPath(ref.path());
+ FileSystem ns = volumeManager.getVolumeByPath(ref.path()).getFileSystem();
FileSKVIterator openReader = fileFactory.openReader(ref.path().toString(), true, ns, ns.getConf(), tableConfig);
return openReader;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
index 8f783c3..bb8e3c7 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
@@ -160,7 +160,7 @@ public class LogSorter {
private void writeBuffer(String destPath, ArrayList<Pair<LogFileKey,LogFileValue>> buffer, int part) throws IOException {
Path path = new Path(destPath, String.format("part-r-%05d", part++));
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
@SuppressWarnings("deprecation")
MapFile.Writer output = new MapFile.Writer(ns.getConf(), ns, path.toString(), LogFileKey.class, LogFileValue.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java
index a28bac4..541f075 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java
@@ -97,7 +97,7 @@ public class MultiReader {
foundFinish = true;
continue;
}
- FileSystem ns = fs.getFileSystemByPath(child.getPath());
+ FileSystem ns = fs.getVolumeByPath(child.getPath()).getFileSystem();
heap.add(new Index(new Reader(ns, child.getPath().toString(), ns.getConf())));
}
if (!foundFinish)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java
index 1cd8f12..7a1c84b 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java
@@ -116,6 +116,7 @@ public class RootFilesTest {
ConfigurationCopy conf = new ConfigurationCopy();
conf.set(Property.INSTANCE_DFS_URI, "file:///");
+ conf.set(Property.INSTANCE_DFS_DIR, "/");
VolumeManager vm = VolumeManagerImpl.get(conf);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
index 50c8b31..dad9a75 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
@@ -22,6 +22,8 @@ import java.util.Map;
import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.volume.VolumeImpl;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
@@ -44,7 +46,7 @@ public class TabletServerSyncCheckTest {
conf.set(DFS_DURABLE_SYNC, "false");
FileSystem fs = new TestFileSystem(conf);
- TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.of("foo", fs));
+ TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.<String,Volume> of("foo", new VolumeImpl(fs, "/")));
vm.ensureSyncIsEnabled();
}
@@ -56,7 +58,7 @@ public class TabletServerSyncCheckTest {
FileSystem fs1 = new TestFileSystem(conf1);
FileSystem fs2 = new TestFileSystem(conf2);
- TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.of("bar", fs2, "foo", fs1));
+ TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.<String,Volume> of("bar", new VolumeImpl(fs2, "/"), "foo", new VolumeImpl(fs1, "/")));
vm.ensureSyncIsEnabled();
}
@@ -67,7 +69,7 @@ public class TabletServerSyncCheckTest {
conf.set(DFS_SUPPORT_APPEND, "false");
FileSystem fs = new TestFileSystem(conf);
- TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.of("foo", fs));
+ TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.<String,Volume> of("foo", new VolumeImpl(fs, "/")));
vm.ensureSyncIsEnabled();
}
@@ -89,8 +91,8 @@ public class TabletServerSyncCheckTest {
private class TestVolumeManagerImpl extends VolumeManagerImpl {
- public TestVolumeManagerImpl(Map<String,? extends FileSystem> volumes) {
- super(volumes, volumes.keySet().iterator().next(), new ConfigurationCopy(Collections.<String,String> emptyMap()));
+ public TestVolumeManagerImpl(Map<String,Volume> volumes) {
+ super(volumes, volumes.values().iterator().next(), new ConfigurationCopy(Collections.<String,String> emptyMap()));
}
@Override
@@ -149,7 +151,7 @@ public class TabletServerSyncCheckTest {
}
@Override
- public FileSystem getFileSystemByPath(Path path) {
+ public Volume getVolumeByPath(Path path) {
return null;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java
index c4d3dfb..a79e77e 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java
@@ -42,13 +42,13 @@ public class MultiReaderTest {
@Before
public void setUp() throws Exception {
- fs = VolumeManagerImpl.getLocal();
root.create();
- String path = root.getRoot().getAbsolutePath();
- Path root = new Path("file://" + path + "/manyMaps");
+ String path = root.getRoot().getAbsolutePath() + "/manyMaps";
+ fs = VolumeManagerImpl.getLocal(path);
+ Path root = new Path("file://" + path);
fs.mkdirs(root);
fs.create(new Path(root, "finished")).close();
- FileSystem ns = fs.getFileSystemByPath(root);
+ FileSystem ns = fs.getVolumeByPath(root).getFileSystem();
@SuppressWarnings("deprecation")
Writer oddWriter = new Writer(ns.getConf(), ns, new Path(root, "odd").toString(), IntWritable.class, BytesWritable.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
index 359bfa1..fffa15e 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
@@ -117,14 +117,15 @@ public class SortedLogRecoveryTest {
private static List<Mutation> recover(Map<String,KeyValue[]> logs, Set<String> files, KeyExtent extent) throws IOException {
TemporaryFolder root = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
root.create();
- final String workdir = "file://" + root.getRoot().getAbsolutePath() + "/workdir";
- VolumeManager fs = VolumeManagerImpl.getLocal();
- fs.deleteRecursively(new Path(workdir));
+ final String workdir = root.getRoot().getAbsolutePath() + "/workdir";
+ VolumeManager fs = VolumeManagerImpl.getLocal(workdir);
+ final Path workdirPath = new Path("file://" + workdir);
+ fs.deleteRecursively(workdirPath);
ArrayList<Path> dirs = new ArrayList<Path>();
try {
for (Entry<String,KeyValue[]> entry : logs.entrySet()) {
String path = workdir + "/" + entry.getKey();
- FileSystem ns = fs.getFileSystemByPath(new Path(path));
+ FileSystem ns = fs.getVolumeByPath(new Path(path)).getFileSystem();
@SuppressWarnings("deprecation")
Writer map = new MapFile.Writer(ns.getConf(), ns, path + "/log1", LogFileKey.class, LogFileValue.class);
for (KeyValue lfe : entry.getValue()) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
index af149fa..d6c23e3 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
@@ -61,10 +61,10 @@ public class TestUpgradePathForWALogs {
public void setUp() throws Exception {
// quiet log messages about compress.CodecPool
Logger.getRootLogger().setLevel(Level.ERROR);
- fs = VolumeManagerImpl.getLocal();
root.create();
- String path = root.getRoot().getAbsolutePath();
- Path manyMapsPath = new Path("file://" + path + "/manyMaps");
+ String path = root.getRoot().getAbsolutePath() + "/manyMaps";
+ fs = VolumeManagerImpl.getLocal(path);
+ Path manyMapsPath = new Path("file://" + path);
fs.mkdirs(manyMapsPath);
fs.create(new Path(manyMapsPath, "finished")).close();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
index 9a9cad7..d2c8d0f 100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
@@ -398,7 +398,7 @@ public class CollectTabletStats {
// assume it is a map file
status = fs.getFileStatus(new Path(file + "/data"));
}
- FileSystem ns = fs.getFileSystemByPath(file.path());
+ FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
BlockLocation[] locs = ns.getFileBlockLocations(status, 0, status.getLen());
System.out.println("\t\t\tBlocks for : " + file);
@@ -445,7 +445,7 @@ public class CollectTabletStats {
HashSet<ByteSequence> columnSet = createColumnBSS(columns);
for (FileRef file : files) {
- FileSystem ns = fs.getFileSystemByPath(file.path());
+ FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().openReader(file.path().toString(), false, ns, ns.getConf(), aconf);
Range range = new Range(ke.getPrevEndRow(), false, ke.getEndRow(), true);
reader.seek(range, columnSet, columnSet.size() == 0 ? false : true);
@@ -475,7 +475,7 @@ public class CollectTabletStats {
List<SortedKeyValueIterator<Key,Value>> readers = new ArrayList<SortedKeyValueIterator<Key,Value>>(files.size());
for (FileRef file : files) {
- FileSystem ns = fs.getFileSystemByPath(file.path());
+ FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
readers.add(FileOperations.getInstance().openReader(file.path().toString(), false, ns, ns.getConf(), aconf.getConfiguration()));
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index a0efe45..a7f7556 100644
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@ -23,6 +23,7 @@ import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -104,8 +105,9 @@ public class VolumeIT extends ConfigurableMacIT {
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
// Run MAC on two locations in the local file system
- cfg.setProperty(Property.INSTANCE_DFS_URI, v1.toString());
- cfg.setProperty(Property.INSTANCE_DFS_DIR, "/accumulo");
+ URI v1Uri = v1.toUri();
+ cfg.setProperty(Property.INSTANCE_DFS_DIR, v1Uri.getPath());
+ cfg.setProperty(Property.INSTANCE_DFS_URI, v1Uri.getScheme() + v1Uri.getHost());
cfg.setProperty(Property.INSTANCE_VOLUMES, v1.toString() + "," + v2.toString());
// use raw local file system so walogs sync and flush will work
@@ -114,7 +116,7 @@ public class VolumeIT extends ConfigurableMacIT {
super.configure(cfg, hadoopCoreSite);
}
- @Test
+ @Test(timeout = 2 * 60 * 1000)
public void test() throws Exception {
// create a table
Connector connector = getConnector();
@@ -176,7 +178,7 @@ public class VolumeIT extends ConfigurableMacIT {
Assert.assertEquals(expected, actual);
}
- @Test
+ @Test(timeout = 2 * 60 * 1000)
public void testRelativePaths() throws Exception {
List<String> expected = new ArrayList<String>();
@@ -292,9 +294,8 @@ public class VolumeIT extends ConfigurableMacIT {
// check that all volumes are initialized
for (Path volumePath : Arrays.asList(v1, v2, v3)) {
FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
- Path vp = new Path(volumePath, "accumulo");
- Path vpi = new Path(vp, ServerConstants.INSTANCE_ID_DIR);
- FileStatus[] iids = fs.listStatus(vpi);
+ Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
+ FileStatus[] iids = fs.listStatus(vp);
Assert.assertEquals(1, iids.length);
Assert.assertEquals(uuid, iids[0].getPath().getName());
}
@@ -390,7 +391,7 @@ public class VolumeIT extends ConfigurableMacIT {
Assert.assertEquals(200, sum);
}
- @Test
+ @Test(timeout = 5 * 60 * 1000)
public void testRemoveVolumes() throws Exception {
String[] tableNames = getTableNames(2);
@@ -447,12 +448,12 @@ public class VolumeIT extends ConfigurableMacIT {
File v1f = new File(v1.toUri());
File v8f = new File(new File(v1.getParent().toUri()), "v8");
- v1f.renameTo(v8f);
+ Assert.assertTrue("Failed to rename " + v1f + " to " + v8f, v1f.renameTo(v8f));
Path v8 = new Path(v8f.toURI());
File v2f = new File(v2.toUri());
File v9f = new File(new File(v2.getParent().toUri()), "v9");
- v2f.renameTo(v9f);
+ Assert.assertTrue("Failed to rename " + v2f + " to " + v9f, v2f.renameTo(v9f));
Path v9 = new Path(v9f.toURI());
Configuration conf = new Configuration(false);
@@ -494,12 +495,12 @@ public class VolumeIT extends ConfigurableMacIT {
verifyVolumesUsed(tableNames[2], true, v8, v9);
}
- @Test
+ @Test(timeout = 5 * 60 * 1000)
public void testCleanReplaceVolumes() throws Exception {
testReplaceVolume(true);
}
- @Test
+ @Test(timeout = 5 * 60 * 1000)
public void testDirtyReplaceVolumes() throws Exception {
testReplaceVolume(false);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
index c8023c0..eee093b 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
@@ -29,9 +29,9 @@ import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVWriter;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.file.rfile.RFile;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.trace.TraceFileSystem;
import org.apache.hadoop.conf.Configuration;
@@ -53,7 +53,7 @@ public class BulkFileIT extends SimpleMacIT {
c.tableOperations().addSplits(tableName, splits);
Configuration conf = new Configuration();
AccumuloConfiguration aconf = ServerConfiguration.getDefaultConfiguration();
- FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultFilesystem(conf, aconf));
+ FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem());
String dir = rootPath() + "/bulk_test_diff_files_89723987592_" + getTableNames(1)[0];
[08/25] git commit: ACCUMULO-2061 Add comments,
javadoc and other cleanup
Posted by el...@apache.org.
ACCUMULO-2061 Add comments, javadoc and other cleanup
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/550e5e61
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/550e5e61
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/550e5e61
Branch: refs/heads/master
Commit: 550e5e61affc490f0931a8991d40f283b0282115
Parents: d45b723
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 12:11:48 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:58:53 2014 -0400
----------------------------------------------------------------------
.../accumulo/core/file/rfile/PrintInfo.java | 9 +++++-
.../org/apache/accumulo/core/volume/Volume.java | 15 ++++------
.../core/volume/VolumeConfiguration.java | 30 ++++++++++++--------
.../apache/accumulo/core/volume/VolumeImpl.java | 22 +++++++-------
.../accumulo/server/fs/VolumeManager.java | 2 --
.../accumulo/server/fs/VolumeManagerImpl.java | 19 +++++++------
.../apache/accumulo/server/fs/VolumeUtil.java | 2 +-
.../accumulo/server/util/ChangeSecret.java | 7 +++--
.../accumulo/server/fs/VolumeUtilTest.java | 2 +-
9 files changed, 59 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 4e39fc7..7c0f067 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -31,10 +31,12 @@ import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
import com.beust.jcommander.Parameter;
public class PrintInfo {
+ private static final Logger log = Logger.getLogger(PrintInfo.class);
static class Opts extends Help {
@Parameter(names = {"-d", "--dump"}, description = "dump the key/value pairs")
@@ -50,6 +52,8 @@ public class PrintInfo {
@SuppressWarnings("deprecation")
AccumuloConfiguration aconf = AccumuloConfiguration.getSiteConfiguration();
+ // TODO This will only work for RFiles in HDFS when the filesystem is defined in the core-site.xml
+ // on the classpath if a path, and not a URI, is given
FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem();
FileSystem localFs = FileSystem.getLocal(conf);
Opts opts = new Opts();
@@ -68,8 +72,11 @@ public class PrintInfo {
FileSystem fs;
if (arg.contains(":"))
fs = path.getFileSystem(conf);
- else
+ else {
+ // Recommend a URI is given for the above todo reason
+ log.warn("Attempting to find file across filesystems. Consider providing URI instead of path");
fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
+ }
CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null, null, aconf);
Reader iter = new RFile.Reader(_rdr);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
index 08f61d4..17b2bf3 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
@@ -21,39 +21,36 @@ import org.apache.hadoop.fs.Path;
/**
* Encapsulates a {@link FileSystem} and a base {@link Path} within that filesystem. This
- * also avoid the necessity to pass around a Configuration.
+ * also avoid the necessity to pass around a Configuration.
*/
public interface Volume {
/**
* A {@link FileSystem} that Accumulo will use
- * @return
*/
public FileSystem getFileSystem();
/**
* The base path which Accumulo will use within the given {@link FileSystem}
- * @return
*/
public String getBasePath();
-
+
/**
* Convert the given Path into a Path that is relative to the base path for this Volume
- * @param p
- * @return
+ * @param p The suffix to use
+ * @return A Path for this Volume with the provided suffix
*/
public Path prefixChild(Path p);
/**
* Convert the given child path into a Path that is relative to the base path for this Volume
- * @param p
- * @return
+ * @param p The suffix to use
+ * @return A Path for this Volume with the provided suffix
*/
public Path prefixChild(String p);
/**
* Determine if the Path is valid on this Volume (contained by the basePath)
- * @param p
* @return True if path is contained within the basePath, false otherwise
*/
public boolean isValidPath(Path p);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
index 3005174..5db5bb2 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
@@ -30,10 +30,10 @@ import org.apache.hadoop.fs.Path;
import com.google.common.base.Preconditions;
public class VolumeConfiguration {
-
+
public static Volume getVolume(String path, Configuration conf, AccumuloConfiguration acuconf) throws IOException {
Preconditions.checkNotNull(path);
-
+
if (path.contains(":")) {
// An absolute path
return create(new Path(path), conf);
@@ -59,12 +59,15 @@ public class VolumeConfiguration {
}
}
+ /**
+ * @see org.apache.accumulo.core.volume.VolumeConfiguration#getVolumeUris(AccumuloConfiguration)
+ */
@Deprecated
public static String getConfiguredBaseDir(AccumuloConfiguration conf) {
String singleNamespace = conf.get(Property.INSTANCE_DFS_DIR);
String dfsUri = conf.get(Property.INSTANCE_DFS_URI);
String baseDir;
-
+
if (dfsUri == null || dfsUri.isEmpty()) {
Configuration hadoopConfig = CachedConfiguration.getInstance();
try {
@@ -82,14 +85,15 @@ public class VolumeConfiguration {
/**
* Compute the URIs to be used by Accumulo
+ *
* @param conf
* @return
*/
public static String[] getVolumeUris(AccumuloConfiguration conf) {
String ns = conf.get(Property.INSTANCE_VOLUMES);
-
+
String configuredBaseDirs[];
-
+
if (ns == null || ns.isEmpty()) {
// Fall back to using the old config values
configuredBaseDirs = new String[] {getConfiguredBaseDir(conf)};
@@ -101,7 +105,7 @@ public class VolumeConfiguration {
if (!namespace.contains(":")) {
throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_VOLUMES.getKey() + " got " + namespace);
}
-
+
try {
// pass through URI to unescape hex encoded chars (e.g. convert %2C to "," char)
configuredBaseDirs[i++] = new Path(new URI(namespace)).toString();
@@ -110,7 +114,7 @@ public class VolumeConfiguration {
}
}
}
-
+
return configuredBaseDirs;
}
@@ -126,24 +130,26 @@ public class VolumeConfiguration {
/**
* Create a Volume with the given FileSystem that writes to the default path
- * @param fs A FileSystem to write to
- * @return A Volume instance writing to the given FileSystem in the default path
+ *
+ * @param fs
+ * A FileSystem to write to
+ * @return A Volume instance writing to the given FileSystem in the default path
*/
@SuppressWarnings("deprecation")
public static <T extends FileSystem> Volume create(T fs, AccumuloConfiguration acuconf) {
String dfsDir = acuconf.get(Property.INSTANCE_DFS_DIR);
return new VolumeImpl(fs, null == dfsDir ? Property.INSTANCE_DFS_DIR.getDefaultValue() : dfsDir);
}
-
+
public static <T extends FileSystem> Volume create(T fs, String basePath) {
return new VolumeImpl(fs, basePath);
}
-
+
public static Volume create(String path, Configuration conf) throws IOException {
Preconditions.checkNotNull(path);
return create(new Path(path), conf);
}
-
+
public static Volume create(Path path, Configuration conf) throws IOException {
return new VolumeImpl(path, conf);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
index 0aaf482..55ccfbc 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
@@ -24,30 +24,30 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-
/**
- *
+ * Basic Volume implementation that contains a FileSystem and a base path
+ * that should be used within that filesystem.
*/
public class VolumeImpl implements Volume {
protected final FileSystem fs;
protected final String basePath;
-
+
public VolumeImpl(Path path, Configuration conf) throws IOException {
checkNotNull(path);
checkNotNull(conf);
-
+
this.fs = path.getFileSystem(conf);
this.basePath = path.toUri().getPath();
}
-
+
public VolumeImpl(FileSystem fs, String basePath) {
checkNotNull(fs);
checkNotNull(basePath);
-
+
this.fs = fs;
this.basePath = basePath;
}
-
+
@Override
public FileSystem getFileSystem() {
return fs;
@@ -66,20 +66,20 @@ public class VolumeImpl implements Volume {
@Override
public boolean isValidPath(Path p) {
checkNotNull(p);
-
+
return p.toUri().getPath().startsWith(basePath);
}
-
+
@Override
public boolean equals(Object o) {
if (o instanceof VolumeImpl) {
VolumeImpl other = (VolumeImpl) o;
return getFileSystem().equals(other.getFileSystem()) && getBasePath().equals(other.getBasePath());
}
-
+
return false;
}
-
+
@Override
public String toString() {
return getFileSystem() + " " + basePath;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
index 9b8fb98..cbfdb5e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
@@ -160,13 +160,11 @@ public interface VolumeManager {
/**
* Fetch the default Volume
- * @return
*/
public Volume getDefaultVolume();
/**
* Fetch the configured Volumes, excluding the default Volume
- * @return
*/
public Collection<Volume> getVolumes();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index ca5167d..b860f53 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -86,11 +86,12 @@ public class VolumeManagerImpl implements VolumeManager {
inverted.put(volume.getFileSystem(), volume);
}
}
-
+
public static org.apache.accumulo.server.fs.VolumeManager getLocal(String localBasePath) throws IOException {
AccumuloConfiguration accConf = DefaultConfiguration.getDefaultConfiguration();
Volume defaultLocalVolume = VolumeConfiguration.create(FileSystem.getLocal(CachedConfiguration.getInstance()), localBasePath);
-
+
+ // The default volume gets placed in the map, but local filesystem is only used for testing purposes
return new VolumeManagerImpl(Collections.singletonMap(DEFAULT, defaultLocalVolume), defaultLocalVolume, accConf);
}
@@ -112,16 +113,16 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public FSDataOutputStream create(Path path) throws IOException {
checkNotNull(path);
-
+
Volume v = getVolumeByPath(path);
-
+
return v.getFileSystem().create(path);
}
@Override
public FSDataOutputStream create(Path path, boolean overwrite) throws IOException {
checkNotNull(path);
-
+
Volume v = getVolumeByPath(path);
return v.getFileSystem().create(path, overwrite);
@@ -149,7 +150,7 @@ public class VolumeManagerImpl implements VolumeManager {
Volume v = getVolumeByPath(path);
FileSystem fs = v.getFileSystem();
-
+
if (bufferSize == 0) {
fs.getConf().getInt("io.file.buffer.size", 4096);
}
@@ -314,7 +315,7 @@ public class VolumeManagerImpl implements VolumeManager {
return defaultVolume;
}
-
+
log.debug("Could not determine volume for Path '" + path + "' from defined volumes");
} catch (IOException ex) {
throw new RuntimeException(ex);
@@ -404,7 +405,7 @@ public class VolumeManagerImpl implements VolumeManager {
// The "default" Volume for Accumulo (in case no volumes are specified)
for (String volumeUriOrDir : VolumeConfiguration.getVolumeUris(conf)) {
if (volumeUriOrDir.equals(DEFAULT))
- // Cannot re-define the default volume
+ // Cannot re-define the default volume
throw new IllegalArgumentException();
// We require a URI here, fail if it doesn't look like one
@@ -554,7 +555,7 @@ public class VolumeManagerImpl implements VolumeManager {
public Volume getDefaultVolume() {
return defaultVolume;
}
-
+
@Override
public Collection<Volume> getVolumes() {
return volumesByName.values();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index bcfb008..2ef438f 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -52,7 +52,7 @@ import org.apache.log4j.Logger;
public class VolumeUtil {
private static final Logger log = Logger.getLogger(VolumeUtil.class);
-
+
private static boolean isActiveVolume(Path dir) {
// consider relative path as active and take no action
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index 3f33a0e..f0dcd14 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@ -146,9 +146,10 @@ public class ChangeSecret {
private static void updateHdfs(VolumeManager fs, Instance inst, String newInstanceId) throws IOException {
// Need to recreate the instanceId on all of them to keep consistency
for (Volume v : fs.getVolumes()) {
- v.getFileSystem().delete(ServerConstants.getInstanceIdLocation(v), true);
- v.getFileSystem().mkdirs(ServerConstants.getInstanceIdLocation(v));
- v.getFileSystem().create(new Path(ServerConstants.getInstanceIdLocation(v), newInstanceId)).close();
+ final Path instanceId = ServerConstants.getInstanceIdLocation(v);
+ v.getFileSystem().delete(instanceId, true);
+ v.getFileSystem().mkdirs(instanceId);
+ v.getFileSystem().create(new Path(instanceId, newInstanceId)).close();
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
index 3b905c9..0013d04 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
@@ -202,7 +202,7 @@ public class VolumeUtilTest {
List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
replacements.add(new Pair<Path,Path>(new Path("file:/foo/v1"), new Path("file:/foo/v8")));
replacements.add(new Pair<Path,Path>(new Path("file:/foo/v2"), new Path("file:/foo/v9")));
-
+
FileType ft = FileType.TABLE;
Assert.assertEquals("file:/foo/v8/tables/+r/root_tablet",
[15/25] git commit: ACCUMULO-2061 Add a utility method to pull a Path
with the instance_id
Posted by el...@apache.org.
ACCUMULO-2061 Add a utility method to pull a Path with the instance_id
Consolidates a little bit of code in one place.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c25a41fe
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c25a41fe
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c25a41fe
Branch: refs/heads/master
Commit: c25a41fe905cda4aa914f232db5cc26b3254f4a0
Parents: 03baf91
Author: Josh Elser <el...@apache.org>
Authored: Thu Mar 20 17:39:13 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:59:30 2014 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/server/Accumulo.java | 7 +++++++
.../java/org/apache/accumulo/server/ServerConstants.java | 2 +-
.../apache/accumulo/server/client/HdfsZooInstance.java | 10 +++++-----
.../org/apache/accumulo/server/conf/ZooConfiguration.java | 10 ++++------
4 files changed, 17 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c25a41fe/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index f7f2298..925c0d0 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.util.Version;
import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
@@ -91,6 +92,12 @@ public class Accumulo {
return getAccumuloPersistentVersion(v.getFileSystem(), path);
}
+ public static synchronized Path getAccumuloInstanceIdPath(VolumeManager fs) {
+ // It doesn't matter which Volume is used as they should all have the instance ID stored
+ Volume v = fs.getVolumes().iterator().next();
+ return ServerConstants.getInstanceIdLocation(v);
+ }
+
public static void enableTracing(String address, String application) {
try {
DistributedTrace.enable(HdfsZooInstance.getInstance(), ZooReaderWriter.getInstance(), application, address);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c25a41fe/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index 7dd0a08..b577abb 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -76,7 +76,7 @@ public class ServerConstants {
String currentIid;
Integer currentVersion;
try {
- currentIid = ZooUtil.getInstanceIDFromHdfs(new Path(baseDir, INSTANCE_ID_DIR), ServerConfiguration.getSiteConfiguration());
+ currentIid = ZooUtil.getInstanceIDFromHdfs(path, ServerConfiguration.getSiteConfiguration());
Path vpath = new Path(baseDir, VERSION_DIR);
currentVersion = Accumulo.getAccumuloPersistentVersion(vpath.getFileSystem(CachedConfiguration.getInstance()), vpath);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c25a41fe/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index ee928f3..620188c 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@ -39,14 +39,14 @@ import org.apache.accumulo.core.util.ByteBufferUtil;
import org.apache.accumulo.core.util.OpTimer;
import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.core.util.TextUtil;
-import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
-import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.Accumulo;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -134,9 +134,9 @@ public class HdfsZooInstance implements Instance {
} catch (IOException e) {
throw new RuntimeException(e);
}
- Volume randVolume = fs.getVolumes().iterator().next();
- log.trace("Looking for instanceId from " + randVolume);
- String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), acuConf);
+ Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
+ log.trace("Looking for instanceId from " + instanceIdPath);
+ String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, acuConf);
instanceId = instanceIdFromFile;
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c25a41fe/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
index 94e468b..0c03aac 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
@@ -26,15 +26,13 @@ import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.accumulo.core.volume.Volume;
-import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
-import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.Accumulo;
import org.apache.accumulo.server.client.HdfsZooInstance.AccumuloNotInitializedException;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
public class ZooConfiguration extends AccumuloConfiguration {
@@ -70,8 +68,8 @@ public class ZooConfiguration extends AccumuloConfiguration {
} catch (IOException e) {
throw new RuntimeException(e);
}
- Volume randVolume = fs.getVolumes().iterator().next();
- String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), parent);
+ Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
+ String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, parent);
instanceId = deprecatedInstanceIdFromHdfs;
}
return instance;
[16/25] git commit: ACCUMULO-2061 Use URI instead of FileSystem as
the key to find correct Volumes and ensure that absolute URIs are still valid
even after they are not configured.
Posted by el...@apache.org.
ACCUMULO-2061 Use URI instead of FileSystem as the key to find correct Volumes and ensure that absolute URIs
are still valid even after they are not configured.
This will help ensure that FileSystem implementations' hashCode and equals don't
have the potential to collide but still provide unique access back to the Volumes
contained in the FileSystem. Added tests for the NonConfiguredVolume and also
for the no-longer-configured volumes.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/42f6b58e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/42f6b58e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/42f6b58e
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 42f6b58e58a8fb1a8f05c371844f86c536fe143a
Parents: 492768d
Author: Josh Elser <el...@apache.org>
Authored: Fri Mar 14 17:06:32 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:59:30 2014 -0400
----------------------------------------------------------------------
.../core/volume/NonConfiguredVolume.java | 92 ++++++++++++++++++++
.../core/volume/NonConfiguredVolumeTest.java | 71 +++++++++++++++
.../accumulo/server/fs/VolumeManagerImpl.java | 25 +++---
.../java/org/apache/accumulo/test/VolumeIT.java | 55 +++++++++++-
4 files changed, 231 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/42f6b58e/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
new file mode 100644
index 0000000..7dcbd88
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.volume;
+
+import java.io.IOException;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Volume implementation which represents a Volume for which we have a FileSystem
+ * but no base path because it is not configured via {@link Property#INSTANCE_VOLUMES}
+ *
+ * This is useful to handle volumes that have been removed from accumulo-site.xml but references
+ * to these volumes have not been updated. This Volume should never be used to create new files,
+ * only to read existing files.
+ */
+public class NonConfiguredVolume implements Volume {
+ private static final Logger log = Logger.getLogger(NonConfiguredVolume.class);
+
+ private FileSystem fs;
+
+ public NonConfiguredVolume(FileSystem fs) {
+ this.fs = fs;
+ }
+
+ @Override
+ public FileSystem getFileSystem() {
+ return fs;
+ }
+
+ @Override
+ public String getBasePath() {
+ throw new UnsupportedOperationException("No base path known because this volume isn't configured in accumulo-site.xml");
+ }
+
+ @Override
+ public Path prefixChild(Path p) {
+ throw new UnsupportedOperationException("Cannot prefix path because this volume isn't configured in accumulo-site.xml");
+ }
+
+ @Override
+ public Path prefixChild(String p) {
+ throw new UnsupportedOperationException("Cannot prefix path because this volume isn't configured in accumulo-site.xml");
+ }
+
+ @Override
+ public boolean isValidPath(Path p) {
+ try {
+ return fs.equals(p.getFileSystem(CachedConfiguration.getInstance()));
+ } catch (IOException e) {
+ log.debug("Cannot determine FileSystem from path: " + p, e);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof NonConfiguredVolume) {
+ NonConfiguredVolume other = (NonConfiguredVolume) o;
+ return this.fs.equals(other.getFileSystem());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "NonConfiguredVolume: " + this.fs.toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return NonConfiguredVolume.class.hashCode() ^ this.fs.hashCode();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo/blob/42f6b58e/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java b/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
new file mode 100644
index 0000000..937baf8
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.volume;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class NonConfiguredVolumeTest {
+
+ private NonConfiguredVolume volume;
+
+ @Before
+ public void create() throws IOException {
+ volume = new NonConfiguredVolume(FileSystem.getLocal(new Configuration()));
+ }
+
+ @Test
+ public void testSameFileSystem() throws IOException {
+ Assert.assertEquals(FileSystem.getLocal(new Configuration()), volume.getFileSystem());
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testGetBasePathFails() {
+ volume.getBasePath();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testPrefixChildPath() {
+ volume.prefixChild(new Path("/foo"));
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testPrefixChildString() {
+ volume.prefixChild("/foo");
+ }
+
+ @Test
+ public void testEquality() throws IOException {
+ Volume newVolume = new NonConfiguredVolume(FileSystem.getLocal(new Configuration()));
+ Assert.assertEquals(volume, newVolume);
+ }
+
+ @Test
+ public void testHashCode() throws IOException {
+ Volume newVolume = new NonConfiguredVolume(FileSystem.getLocal(new Configuration()));
+ Assert.assertEquals(volume.hashCode(), newVolume.hashCode());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/42f6b58e/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index b860f53..64a6390 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -38,6 +38,7 @@ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.NonConfiguredVolume;
import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.server.client.HdfsZooInstance;
@@ -65,7 +66,7 @@ public class VolumeManagerImpl implements VolumeManager {
private static final Logger log = Logger.getLogger(VolumeManagerImpl.class);
Map<String,Volume> volumesByName;
- Multimap<FileSystem,Volume> volumesByFileSystem;
+ Multimap<URI,Volume> volumesByFileSystemUri;
Volume defaultVolume;
AccumuloConfiguration conf;
VolumeChooser chooser;
@@ -74,16 +75,16 @@ public class VolumeManagerImpl implements VolumeManager {
this.volumesByName = volumes;
this.defaultVolume = defaultVolume;
// We may have multiple directories used in a single FileSystem (e.g. testing)
- this.volumesByFileSystem = HashMultimap.create();
- invertVolumesByFileSystem(volumesByName, volumesByFileSystem);
+ this.volumesByFileSystemUri = HashMultimap.create();
+ invertVolumesByFileSystem(volumesByName, volumesByFileSystemUri);
this.conf = conf;
ensureSyncIsEnabled();
chooser = Property.createInstanceFromPropertyName(conf, Property.GENERAL_VOLUME_CHOOSER, VolumeChooser.class, new RandomVolumeChooser());
}
- private void invertVolumesByFileSystem(Map<String,Volume> forward, Multimap<FileSystem,Volume> inverted) {
+ private void invertVolumesByFileSystem(Map<String,Volume> forward, Multimap<URI,Volume> inverted) {
for (Volume volume : forward.values()) {
- inverted.put(volume.getFileSystem(), volume);
+ inverted.put(volume.getFileSystem().getUri(), volume);
}
}
@@ -299,8 +300,9 @@ public class VolumeManagerImpl implements VolumeManager {
public Volume getVolumeByPath(Path path) {
if (path.toString().contains(":")) {
try {
- FileSystem pathFs = path.getFileSystem(CachedConfiguration.getInstance());
- Collection<Volume> candidateVolumes = volumesByFileSystem.get(pathFs);
+ FileSystem desiredFs = path.getFileSystem(CachedConfiguration.getInstance());
+ URI desiredFsUri = desiredFs.getUri();
+ Collection<Volume> candidateVolumes = volumesByFileSystemUri.get(desiredFsUri);
if (null != candidateVolumes) {
for (Volume candidateVolume : candidateVolumes) {
if (candidateVolume.isValidPath(path)) {
@@ -309,11 +311,12 @@ public class VolumeManagerImpl implements VolumeManager {
}
// For the same reason as we can have multiple Volumes within a single filesystem
- // we could also not find a matching one. We should defer back to the defaultVolume
- // e.g. volume rename with old path references
- log.debug("Defaulting to " + defaultVolume + " as a valid volume could not be determined for " + path);
+ // we could also not find a matching one. We should still provide a Volume with the
+ // correct FileSystem even though we don't know what the proper base dir is
+ // e.g. Files on volumes that are now removed
+ log.debug("Found no configured Volume for the given path: " + path);
- return defaultVolume;
+ return new NonConfiguredVolume(desiredFs);
}
log.debug("Could not determine volume for Path '" + path + "' from defined volumes");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/42f6b58e/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index a7f7556..6c1da2c 100644
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@ -307,6 +307,59 @@ public class VolumeIT extends ConfigurableMacIT {
}
+ @Test
+ public void testNonConfiguredVolumes() throws Exception {
+
+ String[] tableNames = getTableNames(2);
+
+ // grab this before shutting down cluster
+ String uuid = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getInstanceID();
+
+ verifyVolumesUsed(tableNames[0], false, v1, v2);
+
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ cluster.stop();
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ File v3f = new File(volDirBase, "v3");
+ v3f.mkdir();
+ Path v3 = new Path("file://" + v3f.getAbsolutePath());
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v2.toString() + "," + v3.toString());
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // initialize volume
+ Assert.assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor());
+
+ // check that all volumes are initialized
+ for (Path volumePath : Arrays.asList(v1, v2, v3)) {
+ FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
+ Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
+ FileStatus[] iids = fs.listStatus(vp);
+ Assert.assertEquals(1, iids.length);
+ Assert.assertEquals(uuid, iids[0].getPath().getName());
+ }
+
+ // start cluster and verify that new volume is used
+ cluster.start();
+
+ // Make sure we can still read the tables (tableNames[0] is very likely to have a file still on v1)
+ List<String> expected = new ArrayList<String>();
+ for (int i = 0; i < 100; i++) {
+ String row = String.format("%06d", i * 100 + 3);
+ expected.add(row + ":cf1:cq1:1");
+ }
+
+ verifyData(expected, getConnector().createScanner(tableNames[0], Authorizations.EMPTY));
+
+ // v1 should not have any data for tableNames[1]
+ verifyVolumesUsed(tableNames[1], false, v2, v3);
+ }
+
private void writeData(String tableName, Connector conn) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
MutationsRejectedException {
TreeSet<Text> splits = new TreeSet<Text>();
@@ -331,7 +384,7 @@ public class VolumeIT extends ConfigurableMacIT {
private void verifyVolumesUsed(String tableName, boolean shouldExist, Path... paths) throws AccumuloException, AccumuloSecurityException,
TableExistsException, TableNotFoundException, MutationsRejectedException {
- Connector conn = cluster.getConnector("root", ROOT_PASSWORD);
+ Connector conn = getConnector();
List<String> expected = new ArrayList<String>();
for (int i = 0; i < 100; i++) {
[22/25] git commit: ACCUMULO-2061 A few more minor fixes from
reviewboard.
Posted by el...@apache.org.
ACCUMULO-2061 A few more minor fixes from reviewboard.
Fixed a PrintInfo comment. Expand the VolumeImpl.isValidPath javadoc
and implementation to be more encompassing. Suppress warnings in VolumeIT.
Remove implementation of isValidPath from NonConfiguredVolume as it
could be misleading.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9172f52b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9172f52b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9172f52b
Branch: refs/heads/master
Commit: 9172f52bc74de2590603c5282c19264447e33da6
Parents: 42f6b58
Author: Josh Elser <el...@apache.org>
Authored: Thu Mar 20 11:22:38 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:59:30 2014 -0400
----------------------------------------------------------------------
.../accumulo/core/file/rfile/PrintInfo.java | 7 +++--
.../core/volume/NonConfiguredVolume.java | 30 ++++++--------------
.../org/apache/accumulo/core/volume/Volume.java | 4 +--
.../apache/accumulo/core/volume/VolumeImpl.java | 16 ++++++++++-
.../accumulo/server/fs/VolumeManagerImpl.java | 8 +++---
.../java/org/apache/accumulo/test/VolumeIT.java | 1 +
6 files changed, 36 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9172f52b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 7ed8f34..dc54b49 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -52,8 +52,11 @@ public class PrintInfo {
@SuppressWarnings("deprecation")
AccumuloConfiguration aconf = AccumuloConfiguration.getSiteConfiguration();
- // TODO ACCUMULO-2462 This will only work for RFiles in HDFS when the filesystem is defined in the core-site.xml
- // on the classpath if a path, and not a URI, is given
+ // TODO ACCUMULO-2462 This will only work for RFiles (path only, not URI) in HDFS when the correct filesystem for the given file
+ // is on Property.INSTANCE_DFS_DIR or, when INSTANCE_DFS_DIR is not defined, is on the default filesystem
+ // defined in the Hadoop's core-site.xml
+ //
+ // A workaround is to always provide a URI to this class
FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem();
FileSystem localFs = FileSystem.getLocal(conf);
Opts opts = new Opts();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9172f52b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
index 7dcbd88..3cfd1c2 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
@@ -16,25 +16,18 @@
*/
package org.apache.accumulo.core.volume;
-import java.io.IOException;
-
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.log4j.Logger;
/**
- * Volume implementation which represents a Volume for which we have a FileSystem
- * but no base path because it is not configured via {@link Property#INSTANCE_VOLUMES}
+ * Volume implementation which represents a Volume for which we have a FileSystem but no base path because it is not configured via
+ * {@link Property#INSTANCE_VOLUMES}
*
- * This is useful to handle volumes that have been removed from accumulo-site.xml but references
- * to these volumes have not been updated. This Volume should never be used to create new files,
- * only to read existing files.
+ * This is useful to handle volumes that have been removed from accumulo-site.xml but references to these volumes have not been updated. This Volume should
+ * never be used to create new files, only to read existing files.
*/
public class NonConfiguredVolume implements Volume {
- private static final Logger log = Logger.getLogger(NonConfiguredVolume.class);
-
private FileSystem fs;
public NonConfiguredVolume(FileSystem fs) {
@@ -48,27 +41,22 @@ public class NonConfiguredVolume implements Volume {
@Override
public String getBasePath() {
- throw new UnsupportedOperationException("No base path known because this volume isn't configured in accumulo-site.xml");
+ throw new UnsupportedOperationException("No base path known because this Volume isn't configured in accumulo-site.xml");
}
@Override
public Path prefixChild(Path p) {
- throw new UnsupportedOperationException("Cannot prefix path because this volume isn't configured in accumulo-site.xml");
+ throw new UnsupportedOperationException("Cannot prefix path because this Volume isn't configured in accumulo-site.xml");
}
@Override
public Path prefixChild(String p) {
- throw new UnsupportedOperationException("Cannot prefix path because this volume isn't configured in accumulo-site.xml");
+ throw new UnsupportedOperationException("Cannot prefix path because this Volume isn't configured in accumulo-site.xml");
}
@Override
public boolean isValidPath(Path p) {
- try {
- return fs.equals(p.getFileSystem(CachedConfiguration.getInstance()));
- } catch (IOException e) {
- log.debug("Cannot determine FileSystem from path: " + p, e);
- }
- return false;
+ throw new UnsupportedOperationException("Cannot determine if path is valid because this Volume isn't configured in accumulo-site.xml");
}
@Override
@@ -89,4 +77,4 @@ public class NonConfiguredVolume implements Volume {
public int hashCode() {
return NonConfiguredVolume.class.hashCode() ^ this.fs.hashCode();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9172f52b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
index 17b2bf3..58b0ada 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
@@ -50,8 +50,8 @@ public interface Volume {
public Path prefixChild(String p);
/**
- * Determine if the Path is valid on this Volume (contained by the basePath)
- * @return True if path is contained within the basePath, false otherwise
+ * Determine if the Path is valid on this Volume. A Path is valid if it is contained
+ * in the Volume's FileSystem and is rooted beneath the basePath
*/
public boolean isValidPath(Path p);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9172f52b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
index 55ccfbc..f902c35 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
@@ -20,15 +20,21 @@ import static com.google.common.base.Preconditions.checkNotNull;
import java.io.IOException;
+import jline.internal.Log;
+
+import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
/**
* Basic Volume implementation that contains a FileSystem and a base path
* that should be used within that filesystem.
*/
public class VolumeImpl implements Volume {
+ private static final Logger log = Logger.getLogger(VolumeImpl.class);
+
protected final FileSystem fs;
protected final String basePath;
@@ -67,7 +73,15 @@ public class VolumeImpl implements Volume {
public boolean isValidPath(Path p) {
checkNotNull(p);
- return p.toUri().getPath().startsWith(basePath);
+ try {
+ if (fs.equals(p.getFileSystem(CachedConfiguration.getInstance()))) {
+ return p.toUri().getPath().startsWith(basePath);
+ }
+ } catch (IOException e) {
+ log.warn("Could not determine filesystem from path: " + p);
+ }
+
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9172f52b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index 64a6390..6a3140b 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -314,12 +314,12 @@ public class VolumeManagerImpl implements VolumeManager {
// we could also not find a matching one. We should still provide a Volume with the
// correct FileSystem even though we don't know what the proper base dir is
// e.g. Files on volumes that are now removed
- log.debug("Found no configured Volume for the given path: " + path);
-
- return new NonConfiguredVolume(desiredFs);
+ log.debug("Found candidate Volumes for Path but none of the Paths are valid on the candidates: " + path);
}
- log.debug("Could not determine volume for Path '" + path + "' from defined volumes");
+ log.debug("Could not determine volume for Path: " + path);
+
+ return new NonConfiguredVolume(desiredFs);
} catch (IOException ex) {
throw new RuntimeException(ex);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9172f52b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index 6c1da2c..0a7ef3f 100644
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@ -102,6 +102,7 @@ public class VolumeIT extends ConfigurableMacIT {
FileUtils.deleteQuietly(new File(v1.getParent().toUri()));
}
+ @SuppressWarnings("deprecation")
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
// Run MAC on two locations in the local file system
[02/25] git commit: ACCUMULO-2061 Initial implementation to remove
instance.dfs.dir from usage of instance.volumes
Posted by el...@apache.org.
ACCUMULO-2061 Initial implementation to remove instance.dfs.dir from usage of instance.volumes
The value of instance.volumes, when provided, appended the value of
instance.dfs.dir to each configured volume which was unintuitive. It is more
straightforward to use each volume provided as-is instead of appending
on some more cruft.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7c94c086
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7c94c086
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7c94c086
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 7c94c086de2945d698d0907c9dc5ec6959042286
Parents: ef5dc4a
Author: Josh Elser <el...@apache.org>
Authored: Thu Feb 27 00:51:29 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:55:47 2014 -0400
----------------------------------------------------------------------
.../core/client/admin/TableOperationsImpl.java | 4 +-
.../core/client/impl/OfflineScanner.java | 4 +-
.../org/apache/accumulo/core/conf/Property.java | 2 +
.../accumulo/core/file/VolumeConfiguration.java | 112 ------------
.../accumulo/core/file/rfile/PrintInfo.java | 4 +-
.../core/file/rfile/bcfile/PrintInfo.java | 4 +-
.../apache/accumulo/core/util/shell/Shell.java | 4 +-
.../org/apache/accumulo/core/volume/Volume.java | 60 +++++++
.../core/volume/VolumeConfiguration.java | 151 +++++++++++++++++
.../apache/accumulo/core/volume/VolumeImpl.java | 93 ++++++++++
.../apache/accumulo/core/zookeeper/ZooUtil.java | 4 +-
.../org/apache/accumulo/server/Accumulo.java | 13 +-
.../apache/accumulo/server/ServerConstants.java | 48 +++---
.../accumulo/server/client/BulkImporter.java | 4 +-
.../accumulo/server/client/HdfsZooInstance.java | 16 +-
.../accumulo/server/conf/ZooConfiguration.java | 16 +-
.../accumulo/server/fs/VolumeManager.java | 21 ++-
.../accumulo/server/fs/VolumeManagerImpl.java | 169 +++++++++++++------
.../apache/accumulo/server/fs/VolumeUtil.java | 27 ++-
.../apache/accumulo/server/init/Initialize.java | 18 +-
.../server/master/recovery/HadoopLogCloser.java | 2 +-
.../server/master/recovery/MapRLogCloser.java | 2 +-
.../accumulo/server/util/ChangeSecret.java | 18 +-
.../apache/accumulo/server/util/FileUtil.java | 12 +-
.../accumulo/server/util/LocalityCheck.java | 2 +-
.../accumulo/server/util/TabletOperations.java | 4 +-
.../accumulo/server/util/ZooKeeperMain.java | 2 +-
.../accumulo/server/ServerConstantsTest.java | 12 +-
.../apache/accumulo/server/fs/FileTypeTest.java | 48 ++++--
.../accumulo/server/fs/VolumeUtilTest.java | 89 +++++++++-
.../accumulo/gc/GarbageCollectionTest.java | 65 +++++++
.../accumulo/master/tableOps/ExportTable.java | 2 +-
.../accumulo/master/tableOps/ImportTable.java | 2 +-
.../monitor/servlets/DefaultServlet.java | 6 +-
.../tserver/BulkFailedCopyProcessor.java | 27 +--
.../org/apache/accumulo/tserver/Compactor.java | 4 +-
.../apache/accumulo/tserver/FileManager.java | 2 +-
.../org/apache/accumulo/tserver/Tablet.java | 4 +-
.../apache/accumulo/tserver/TabletServer.java | 2 +-
.../compaction/MajorCompactionRequest.java | 2 +-
.../apache/accumulo/tserver/log/LogSorter.java | 2 +-
.../accumulo/tserver/log/MultiReader.java | 2 +-
.../apache/accumulo/tserver/RootFilesTest.java | 1 +
.../tserver/TabletServerSyncCheckTest.java | 14 +-
.../accumulo/tserver/log/MultiReaderTest.java | 8 +-
.../tserver/log/SortedLogRecoveryTest.java | 9 +-
.../tserver/log/TestUpgradePathForWALogs.java | 6 +-
.../performance/scan/CollectTabletStats.java | 6 +-
.../java/org/apache/accumulo/test/VolumeIT.java | 25 +--
.../accumulo/test/functional/BulkFileIT.java | 4 +-
50 files changed, 822 insertions(+), 336 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
index 0b2f10e..9d033e2 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
@@ -82,7 +82,6 @@ import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
@@ -109,6 +108,7 @@ import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.core.util.TextUtil;
import org.apache.accumulo.core.util.ThriftUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.trace.instrument.Tracer;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -1139,7 +1139,7 @@ public class TableOperationsImpl extends TableOperationsHelper {
@SuppressWarnings("deprecation")
private Path checkPath(String dir, String kind, String type) throws IOException, AccumuloException {
Path ret;
- FileSystem fs = VolumeConfiguration.getFileSystem(dir, CachedConfiguration.getInstance(), ServerConfigurationUtil.getConfiguration(instance));
+ FileSystem fs = VolumeConfiguration.getVolume(dir, CachedConfiguration.getInstance(), ServerConfigurationUtil.getConfiguration(instance)).getFileSystem();
if (dir.contains(":")) {
ret = new Path(dir);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
index c90d380..20228d2 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
@@ -43,7 +43,6 @@ import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVIterator;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
@@ -65,6 +64,7 @@ import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.LocalityGroupUtil;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -306,7 +306,7 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
// TODO need to close files - ACCUMULO-1303
for (String file : absFiles) {
- FileSystem fs = VolumeConfiguration.getFileSystem(file, conf, config);
+ FileSystem fs = VolumeConfiguration.getVolume(file, conf, config).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().openReader(file, false, fs, conf, acuTableConf, null, null);
readers.add(reader);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index fc4d012..795a250 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -103,6 +103,7 @@ public enum Property {
INSTANCE_ZK_HOST("instance.zookeeper.host", "localhost:2181", PropertyType.HOSTLIST, "Comma separated list of zookeeper servers"),
INSTANCE_ZK_TIMEOUT("instance.zookeeper.timeout", "30s", PropertyType.TIMEDURATION,
"Zookeeper session timeout; max value when represented as milliseconds should be no larger than " + Integer.MAX_VALUE),
+ @Deprecated
INSTANCE_DFS_URI(
"instance.dfs.uri",
"",
@@ -111,6 +112,7 @@ public enum Property {
+ "will only be used when creating new files if instance.volumes is empty. After an upgrade to 1.6.0 Accumulo will start using absolute paths to "
+ "reference files. Files created before a 1.6.0 upgrade are referenced via relative paths. Relative paths will always be resolved using this config "
+ "(if empty using the hadoop config)."),
+ @Deprecated
INSTANCE_DFS_DIR("instance.dfs.dir", "/accumulo", PropertyType.ABSOLUTEPATH,
"HDFS directory in which accumulo instance will run. Do not change after accumulo is initialized."),
@Sensitive
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/file/VolumeConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/file/VolumeConfiguration.java
deleted file mode 100644
index fb8c6c8..0000000
--- a/core/src/main/java/org/apache/accumulo/core/file/VolumeConfiguration.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.core.file;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-public class VolumeConfiguration {
-
- public static FileSystem getFileSystem(String path, Configuration conf, AccumuloConfiguration acuconf) throws IOException {
- if (path.contains(":"))
- return new Path(path).getFileSystem(conf);
- else
- return getDefaultFilesystem(conf, acuconf);
- }
-
- public static FileSystem getDefaultFilesystem(Configuration conf, AccumuloConfiguration acuconf) throws IOException {
- String uri = acuconf.get(Property.INSTANCE_DFS_URI);
- if ("".equals(uri))
- return FileSystem.get(conf);
- else
- try {
- return FileSystem.get(new URI(uri), conf);
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- public static String getConfiguredBaseDir(AccumuloConfiguration conf) {
- String singleNamespace = conf.get(Property.INSTANCE_DFS_DIR);
- String dfsUri = conf.get(Property.INSTANCE_DFS_URI);
- String baseDir;
-
- if (dfsUri == null || dfsUri.isEmpty()) {
- Configuration hadoopConfig = CachedConfiguration.getInstance();
- try {
- baseDir = FileSystem.get(hadoopConfig).getUri().toString() + singleNamespace;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- } else {
- if (!dfsUri.contains(":"))
- throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_DFS_URI.getKey() + " got " + dfsUri);
- baseDir = dfsUri + singleNamespace;
- }
- return baseDir;
- }
-
- public static String[] getConfiguredBaseDirs(AccumuloConfiguration conf) {
- String singleNamespace = conf.get(Property.INSTANCE_DFS_DIR);
- String ns = conf.get(Property.INSTANCE_VOLUMES);
-
- String configuredBaseDirs[];
-
- if (ns == null || ns.isEmpty()) {
- configuredBaseDirs = new String[] {getConfiguredBaseDir(conf)};
- } else {
- String namespaces[] = ns.split(",");
- String unescapedNamespaces[] = new String[namespaces.length];
- int i = 0;
- for (String namespace : namespaces) {
- if (!namespace.contains(":")) {
- throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_VOLUMES.getKey() + " got " + namespace);
- }
-
- try {
- // pass through URI to unescape hex encoded chars (e.g. convert %2C to "," char)
- unescapedNamespaces[i++] = new Path(new URI(namespace)).toString();
- } catch (URISyntaxException e) {
- throw new IllegalArgumentException(Property.INSTANCE_VOLUMES.getKey() + " contains " + namespace + " which has a syntax error", e);
- }
- }
-
- configuredBaseDirs = prefix(unescapedNamespaces, singleNamespace);
- }
-
- return configuredBaseDirs;
- }
-
- public static String[] prefix(String bases[], String suffix) {
- if (suffix.startsWith("/"))
- suffix = suffix.substring(1);
- String result[] = new String[bases.length];
- for (int i = 0; i < bases.length; i++) {
- result[i] = bases[i] + "/" + suffix;
- }
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 4cfefad..4e39fc7 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -25,9 +25,9 @@ import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
import org.apache.accumulo.core.file.rfile.RFile.Reader;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -50,7 +50,7 @@ public class PrintInfo {
@SuppressWarnings("deprecation")
AccumuloConfiguration aconf = AccumuloConfiguration.getSiteConfiguration();
- FileSystem hadoopFs = VolumeConfiguration.getDefaultFilesystem(conf, aconf);
+ FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem();
FileSystem localFs = FileSystem.getLocal(conf);
Opts opts = new Opts();
opts.parseArgs(PrintInfo.class.getName(), args);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
index f21190e..e2c8a6d 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
@@ -22,8 +22,8 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.file.rfile.bcfile.BCFile.MetaIndexEntry;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -57,7 +57,7 @@ public class PrintInfo {
Configuration conf = new Configuration();
@SuppressWarnings("deprecation")
AccumuloConfiguration siteConf = AccumuloConfiguration.getSiteConfiguration();
- FileSystem hadoopFs = VolumeConfiguration.getDefaultFilesystem(conf, siteConf);
+ FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, siteConf).getFileSystem();
FileSystem localFs = FileSystem.getLocal(conf);
Path path = new Path(args[0]);
FileSystem fs;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
index 0c2cf3c..ff6ba09 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
@@ -63,7 +63,6 @@ import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.data.thrift.TConstraintViolationSummary;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
import org.apache.accumulo.core.trace.DistributedTrace;
import org.apache.accumulo.core.util.BadArgumentException;
@@ -156,6 +155,7 @@ import org.apache.accumulo.core.util.shell.commands.UserCommand;
import org.apache.accumulo.core.util.shell.commands.UserPermissionsCommand;
import org.apache.accumulo.core.util.shell.commands.UsersCommand;
import org.apache.accumulo.core.util.shell.commands.WhoAmICommand;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooReader;
import org.apache.commons.cli.BasicParser;
@@ -432,7 +432,7 @@ public class Shell extends ShellOptions {
if (instanceName == null || keepers == null) {
AccumuloConfiguration conf = SiteConfiguration.getInstance(ServerConfigurationUtil.convertClientConfig(DefaultConfiguration.getInstance(), clientConfig));
if (instanceName == null) {
- Path instanceDir = new Path(VolumeConfiguration.getConfiguredBaseDirs(conf)[0], "instance_id");
+ Path instanceDir = new Path(VolumeConfiguration.getVolumeUris(conf)[0], "instance_id");
instanceId = UUID.fromString(ZooUtil.getInstanceIDFromHdfs(instanceDir, conf));
}
if (keepers == null) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
new file mode 100644
index 0000000..08f61d4
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.volume;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Encapsulates a {@link FileSystem} and a base {@link Path} within that filesystem. This
+ * also avoid the necessity to pass around a Configuration.
+ */
+public interface Volume {
+
+ /**
+ * A {@link FileSystem} that Accumulo will use
+ * @return
+ */
+ public FileSystem getFileSystem();
+
+ /**
+ * The base path which Accumulo will use within the given {@link FileSystem}
+ * @return
+ */
+ public String getBasePath();
+
+ /**
+ * Convert the given Path into a Path that is relative to the base path for this Volume
+ * @param p
+ * @return
+ */
+ public Path prefixChild(Path p);
+
+ /**
+ * Convert the given child path into a Path that is relative to the base path for this Volume
+ * @param p
+ * @return
+ */
+ public Path prefixChild(String p);
+
+ /**
+ * Determine if the Path is valid on this Volume (contained by the basePath)
+ * @param p
+ * @return True if path is contained within the basePath, false otherwise
+ */
+ public boolean isValidPath(Path p);
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
new file mode 100644
index 0000000..3005174
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.volume;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import com.google.common.base.Preconditions;
+
+public class VolumeConfiguration {
+
+ public static Volume getVolume(String path, Configuration conf, AccumuloConfiguration acuconf) throws IOException {
+ Preconditions.checkNotNull(path);
+
+ if (path.contains(":")) {
+ // An absolute path
+ return create(new Path(path), conf);
+ } else {
+ // A relative path
+ return getDefaultVolume(conf, acuconf);
+ }
+ }
+
+ public static Volume getDefaultVolume(Configuration conf, AccumuloConfiguration acuconf) throws IOException {
+ @SuppressWarnings("deprecation")
+ String uri = acuconf.get(Property.INSTANCE_DFS_URI);
+
+ // By default pull from INSTANCE_DFS_URI, falling back to the Hadoop defined
+ // default filesystem (fs.defaultFS or the deprecated fs.default.name)
+ if ("".equals(uri))
+ return create(FileSystem.get(conf), acuconf);
+ else
+ try {
+ return create(FileSystem.get(new URI(uri), conf), acuconf);
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Deprecated
+ public static String getConfiguredBaseDir(AccumuloConfiguration conf) {
+ String singleNamespace = conf.get(Property.INSTANCE_DFS_DIR);
+ String dfsUri = conf.get(Property.INSTANCE_DFS_URI);
+ String baseDir;
+
+ if (dfsUri == null || dfsUri.isEmpty()) {
+ Configuration hadoopConfig = CachedConfiguration.getInstance();
+ try {
+ baseDir = FileSystem.get(hadoopConfig).getUri().toString() + singleNamespace;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ if (!dfsUri.contains(":"))
+ throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_DFS_URI.getKey() + " got " + dfsUri);
+ baseDir = dfsUri + singleNamespace;
+ }
+ return baseDir;
+ }
+
+ /**
+ * Compute the URIs to be used by Accumulo
+ * @param conf
+ * @return
+ */
+ public static String[] getVolumeUris(AccumuloConfiguration conf) {
+ String ns = conf.get(Property.INSTANCE_VOLUMES);
+
+ String configuredBaseDirs[];
+
+ if (ns == null || ns.isEmpty()) {
+ // Fall back to using the old config values
+ configuredBaseDirs = new String[] {getConfiguredBaseDir(conf)};
+ } else {
+ String namespaces[] = ns.split(",");
+ configuredBaseDirs = new String[namespaces.length];
+ int i = 0;
+ for (String namespace : namespaces) {
+ if (!namespace.contains(":")) {
+ throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_VOLUMES.getKey() + " got " + namespace);
+ }
+
+ try {
+ // pass through URI to unescape hex encoded chars (e.g. convert %2C to "," char)
+ configuredBaseDirs[i++] = new Path(new URI(namespace)).toString();
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(Property.INSTANCE_VOLUMES.getKey() + " contains " + namespace + " which has a syntax error", e);
+ }
+ }
+ }
+
+ return configuredBaseDirs;
+ }
+
+ public static String[] prefix(String bases[], String suffix) {
+ if (suffix.startsWith("/"))
+ suffix = suffix.substring(1);
+ String result[] = new String[bases.length];
+ for (int i = 0; i < bases.length; i++) {
+ result[i] = bases[i] + "/" + suffix;
+ }
+ return result;
+ }
+
+ /**
+ * Create a Volume with the given FileSystem that writes to the default path
+ * @param fs A FileSystem to write to
+ * @return A Volume instance writing to the given FileSystem in the default path
+ */
+ @SuppressWarnings("deprecation")
+ public static <T extends FileSystem> Volume create(T fs, AccumuloConfiguration acuconf) {
+ String dfsDir = acuconf.get(Property.INSTANCE_DFS_DIR);
+ return new VolumeImpl(fs, null == dfsDir ? Property.INSTANCE_DFS_DIR.getDefaultValue() : dfsDir);
+ }
+
+ public static <T extends FileSystem> Volume create(T fs, String basePath) {
+ return new VolumeImpl(fs, basePath);
+ }
+
+ public static Volume create(String path, Configuration conf) throws IOException {
+ Preconditions.checkNotNull(path);
+ return create(new Path(path), conf);
+ }
+
+ public static Volume create(Path path, Configuration conf) throws IOException {
+ return new VolumeImpl(path, conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
new file mode 100644
index 0000000..0aaf482
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.volume;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+
+/**
+ *
+ */
+public class VolumeImpl implements Volume {
+ protected final FileSystem fs;
+ protected final String basePath;
+
+ public VolumeImpl(Path path, Configuration conf) throws IOException {
+ checkNotNull(path);
+ checkNotNull(conf);
+
+ this.fs = path.getFileSystem(conf);
+ this.basePath = path.toUri().getPath();
+ }
+
+ public VolumeImpl(FileSystem fs, String basePath) {
+ checkNotNull(fs);
+ checkNotNull(basePath);
+
+ this.fs = fs;
+ this.basePath = basePath;
+ }
+
+ @Override
+ public FileSystem getFileSystem() {
+ return fs;
+ }
+
+ @Override
+ public String getBasePath() {
+ return basePath;
+ }
+
+ @Override
+ public Path prefixChild(Path p) {
+ return new Path(basePath, p);
+ }
+
+ @Override
+ public boolean isValidPath(Path p) {
+ checkNotNull(p);
+
+ return p.toUri().getPath().startsWith(basePath);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof VolumeImpl) {
+ VolumeImpl other = (VolumeImpl) o;
+ return getFileSystem().equals(other.getFileSystem()) && getBasePath().equals(other.getBasePath());
+ }
+
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return getFileSystem() + " " + basePath;
+ }
+
+ @Override
+ public Path prefixChild(String p) {
+ return new Path(basePath, p);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooUtil.java b/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooUtil.java
index de1b432..d536f42 100644
--- a/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooUtil.java
@@ -23,8 +23,8 @@ import java.net.UnknownHostException;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -49,7 +49,7 @@ public class ZooUtil extends org.apache.accumulo.fate.zookeeper.ZooUtil {
public static String getInstanceIDFromHdfs(Path instanceDirectory, AccumuloConfiguration conf) {
try {
- FileSystem fs = VolumeConfiguration.getFileSystem(instanceDirectory.toString(), CachedConfiguration.getInstance(), conf);
+ FileSystem fs = VolumeConfiguration.getVolume(instanceDirectory.toString(), CachedConfiguration.getInstance(), conf).getFileSystem();
FileStatus[] files = null;
try {
files = fs.listStatus(instanceDirectory);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index 2fa9051..48534f0 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -31,6 +31,7 @@ import org.apache.accumulo.core.trace.DistributedTrace;
import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.util.Version;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
@@ -50,10 +51,12 @@ public class Accumulo {
private static final Logger log = Logger.getLogger(Accumulo.class);
public static synchronized void updateAccumuloVersion(VolumeManager fs) {
+ // TODO ACCUMULO-2451 Should update all volumes, not one
+ Volume volume = fs.getVolumes().iterator().next();
try {
if (getAccumuloPersistentVersion(fs) == ServerConstants.PREV_DATA_VERSION) {
- fs.create(new Path(ServerConstants.getDataVersionLocation() + "/" + ServerConstants.DATA_VERSION));
- fs.delete(new Path(ServerConstants.getDataVersionLocation() + "/" + ServerConstants.PREV_DATA_VERSION));
+ fs.create(new Path(ServerConstants.getDataVersionLocation(volume), Integer.toString(ServerConstants.DATA_VERSION)));
+ fs.delete(new Path(ServerConstants.getDataVersionLocation(volume), Integer.toString(ServerConstants.PREV_DATA_VERSION)));
}
} catch (IOException e) {
throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
@@ -76,8 +79,10 @@ public class Accumulo {
}
public static synchronized int getAccumuloPersistentVersion(VolumeManager fs) {
- Path path = ServerConstants.getDataVersionLocation();
- return getAccumuloPersistentVersion(fs.getFileSystemByPath(path), path);
+ // It doesn't matter which Volume is used as they should all have the data version stored
+ Volume v = fs.getVolumes().iterator().next();
+ Path path = ServerConstants.getDataVersionLocation(v);
+ return getAccumuloPersistentVersion(v.getFileSystem(), path);
}
public static void enableTracing(String address, String application) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index 8983d08..7dd0a08 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -24,10 +24,11 @@ import java.util.HashSet;
import java.util.List;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.VolumeUtil;
@@ -51,29 +52,20 @@ public class ServerConstants {
public static final int DATA_VERSION = 6;
public static final int PREV_DATA_VERSION = 5;
- private static String[] baseDirs = null;
- private static String defaultBaseDir = null;
+ private static String[] baseUris = null;
private static List<Pair<Path,Path>> replacementsList = null;
- public static synchronized String getDefaultBaseDir() {
- if (defaultBaseDir == null) {
- defaultBaseDir = new Path(VolumeConfiguration.getConfiguredBaseDir(ServerConfiguration.getSiteConfiguration())).toString();
- }
-
- return defaultBaseDir;
- }
-
// these are functions to delay loading the Accumulo configuration unless we must
- public static synchronized String[] getBaseDirs() {
- if (baseDirs == null) {
- baseDirs = checkBaseDirs(VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration()), false);
+ public static synchronized String[] getBaseUris() {
+ if (baseUris == null) {
+ baseUris = checkBaseUris(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()), false);
}
- return baseDirs;
+ return baseUris;
}
- public static String[] checkBaseDirs(String[] configuredBaseDirs, boolean ignore) {
+ public static String[] checkBaseUris(String[] configuredBaseDirs, boolean ignore) {
// all base dirs must have same instance id and data version, any dirs that have neither should be ignored
String firstDir = null;
String firstIid = null;
@@ -121,29 +113,29 @@ public class ServerConstants {
public static final String WAL_DIR = "wal";
public static String[] getTablesDirs() {
- return VolumeConfiguration.prefix(getBaseDirs(), TABLE_DIR);
+ return VolumeConfiguration.prefix(getBaseUris(), TABLE_DIR);
}
public static String[] getRecoveryDirs() {
- return VolumeConfiguration.prefix(getBaseDirs(), RECOVERY_DIR);
+ return VolumeConfiguration.prefix(getBaseUris(), RECOVERY_DIR);
}
public static String[] getWalDirs() {
- return VolumeConfiguration.prefix(getBaseDirs(), WAL_DIR);
+ return VolumeConfiguration.prefix(getBaseUris(), WAL_DIR);
}
public static String[] getWalogArchives() {
- return VolumeConfiguration.prefix(getBaseDirs(), "walogArchive");
+ return VolumeConfiguration.prefix(getBaseUris(), "walogArchive");
}
- public static Path getInstanceIdLocation() {
+ public static Path getInstanceIdLocation(Volume v) {
// all base dirs should have the same instance id, so can choose any one
- return new Path(getBaseDirs()[0], INSTANCE_ID_DIR);
+ return v.prefixChild(INSTANCE_ID_DIR);
}
- public static Path getDataVersionLocation() {
+ public static Path getDataVersionLocation(Volume v) {
// all base dirs should have the same version, so can choose any one
- return new Path(getBaseDirs()[0], VERSION_DIR);
+ return v.prefixChild(VERSION_DIR);
}
public static String[] getMetadataTableDirs() {
@@ -151,7 +143,7 @@ public class ServerConstants {
}
public static String[] getTemporaryDirs() {
- return VolumeConfiguration.prefix(getBaseDirs(), "tmp");
+ return VolumeConfiguration.prefix(getBaseUris(), "tmp");
}
public static synchronized List<Pair<Path,Path>> getVolumeReplacements() {
@@ -194,9 +186,9 @@ public class ServerConstants {
}
HashSet<Path> baseDirs = new HashSet<Path>();
- for (String baseDir : getBaseDirs()) {
- // normalize using path and remove accumulo dir
- baseDirs.add(new Path(baseDir).getParent());
+ for (String baseDir : getBaseUris()) {
+ // normalize using path
+ baseDirs.add(new Path(baseDir));
}
for (Pair<Path,Path> pair : ret)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
index dc9acf8..27ab078 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
@@ -339,7 +339,7 @@ public class BulkImporter {
try {
for (Path path : paths) {
- FileSystem fs = vm.getFileSystemByPath(path);
+ FileSystem fs = vm.getVolumeByPath(path).getFileSystem();
mapFileSizes.put(path, fs.getContentSummary(path).getLength());
}
} catch (IOException e) {
@@ -639,7 +639,7 @@ public class BulkImporter {
Collection<ByteSequence> columnFamilies = Collections.emptyList();
String filename = file.toString();
// log.debug(filename + " finding overlapping tablets " + startRow + " -> " + endRow);
- FileSystem fs = vm.getFileSystemByPath(file);
+ FileSystem fs = vm.getVolumeByPath(file).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().openReader(filename, true, fs, fs.getConf(), acuConf);
try {
Text row = startRow;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index 6993a0a..ee928f3 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.client;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
@@ -38,10 +39,13 @@ import org.apache.accumulo.core.util.ByteBufferUtil;
import org.apache.accumulo.core.util.OpTimer;
import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.zookeeper.ZooLock;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Level;
@@ -122,7 +126,17 @@ public class HdfsZooInstance implements Instance {
private static synchronized void _getInstanceID() {
if (instanceId == null) {
- String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(), ServerConfiguration.getSiteConfiguration());
+ AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
+ // InstanceID should be the same across all volumes, so just choose one
+ VolumeManager fs;
+ try {
+ fs = VolumeManagerImpl.get();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Volume randVolume = fs.getVolumes().iterator().next();
+ log.trace("Looking for instanceId from " + randVolume);
+ String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), acuConf);
instanceId = instanceIdFromFile;
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
index 32f6126..94e468b 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.conf;
+import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -25,10 +26,15 @@ import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.client.HdfsZooInstance.AccumuloNotInitializedException;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.log4j.Logger;
public class ZooConfiguration extends AccumuloConfiguration {
@@ -57,7 +63,15 @@ public class ZooConfiguration extends AccumuloConfiguration {
if (instance == null) {
propCache = new ZooCache(parent.get(Property.INSTANCE_ZK_HOST), (int) parent.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
instance = new ZooConfiguration(parent);
- String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(), parent);
+ // InstanceID should be the same across all volumes, so just choose one
+ VolumeManager fs;
+ try {
+ fs = VolumeManagerImpl.get();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Volume randVolume = fs.getVolumes().iterator().next();
+ String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), parent);
instanceId = deprecatedInstanceIdFromHdfs;
}
return instance;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
index f0c7083..9b8fb98 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
@@ -17,14 +17,15 @@
package org.apache.accumulo.server.fs;
import java.io.IOException;
+import java.util.Collection;
import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.server.ServerConstants;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
/**
@@ -47,9 +48,10 @@ public interface VolumeManager {
}
private static int endOfVolumeIndex(String path, String dir) {
+ // Strip off the suffix that starts with the FileType (e.g. tables, wal, etc)
int dirIndex = path.indexOf('/' + dir);
if (dirIndex != -1) {
- return path.lastIndexOf('/', dirIndex - 1);
+ return dirIndex;
}
if (path.contains(":"))
@@ -110,7 +112,7 @@ public interface VolumeManager {
FileStatus getFileStatus(Path path) throws IOException;
// find the appropriate FileSystem object given a path
- FileSystem getFileSystemByPath(Path path);
+ Volume getVolumeByPath(Path path);
// return the item in options that is in the same volume as source
Path matchingFileSystem(Path source, String[] options);
@@ -155,4 +157,17 @@ public interface VolumeManager {
// decide on which of the given locations to create a new file
String choose(String[] options);
+
+ /**
+ * Fetch the default Volume
+ * @return
+ */
+ public Volume getDefaultVolume();
+
+ /**
+ * Fetch the configured Volumes, excluding the default Volume
+ * @return
+ */
+ public Collection<Volume> getVolumes();
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index 80301ef..ca5167d 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -16,12 +16,15 @@
*/
package org.apache.accumulo.server.fs;
+import static com.google.common.base.Preconditions.checkNotNull;
+
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URI;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
@@ -34,9 +37,9 @@ import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.commons.lang.NotImplementedException;
@@ -54,34 +57,49 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.util.Progressable;
import org.apache.log4j.Logger;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
public class VolumeManagerImpl implements VolumeManager {
private static final Logger log = Logger.getLogger(VolumeManagerImpl.class);
- Map<String,? extends FileSystem> volumes;
- FileSystem defaultVolume;
+ Map<String,Volume> volumesByName;
+ Multimap<FileSystem,Volume> volumesByFileSystem;
+ Volume defaultVolume;
AccumuloConfiguration conf;
VolumeChooser chooser;
- protected VolumeManagerImpl(Map<String,? extends FileSystem> volumes, String defaultVolume, AccumuloConfiguration conf) {
- this.volumes = volumes;
- this.defaultVolume = volumes.get(defaultVolume);
+ protected VolumeManagerImpl(Map<String,Volume> volumes, Volume defaultVolume, AccumuloConfiguration conf) {
+ this.volumesByName = volumes;
+ this.defaultVolume = defaultVolume;
+ // We may have multiple directories used in a single FileSystem (e.g. testing)
+ this.volumesByFileSystem = HashMultimap.create();
+ invertVolumesByFileSystem(volumesByName, volumesByFileSystem);
this.conf = conf;
ensureSyncIsEnabled();
chooser = Property.createInstanceFromPropertyName(conf, Property.GENERAL_VOLUME_CHOOSER, VolumeChooser.class, new RandomVolumeChooser());
}
- public static org.apache.accumulo.server.fs.VolumeManager getLocal() throws IOException {
- return new VolumeManagerImpl(Collections.singletonMap("", FileSystem.getLocal(CachedConfiguration.getInstance())), "",
- DefaultConfiguration.getDefaultConfiguration());
+ private void invertVolumesByFileSystem(Map<String,Volume> forward, Multimap<FileSystem,Volume> inverted) {
+ for (Volume volume : forward.values()) {
+ inverted.put(volume.getFileSystem(), volume);
+ }
+ }
+
+ public static org.apache.accumulo.server.fs.VolumeManager getLocal(String localBasePath) throws IOException {
+ AccumuloConfiguration accConf = DefaultConfiguration.getDefaultConfiguration();
+ Volume defaultLocalVolume = VolumeConfiguration.create(FileSystem.getLocal(CachedConfiguration.getInstance()), localBasePath);
+
+ return new VolumeManagerImpl(Collections.singletonMap(DEFAULT, defaultLocalVolume), defaultLocalVolume, accConf);
}
@Override
public void close() throws IOException {
IOException ex = null;
- for (FileSystem fs : volumes.values()) {
+ for (Volume volume : volumesByName.values()) {
try {
- fs.close();
+ volume.getFileSystem().close();
} catch (IOException e) {
ex = e;
}
@@ -93,14 +111,20 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public FSDataOutputStream create(Path path) throws IOException {
- FileSystem fs = getFileSystemByPath(path);
- return fs.create(path);
+ checkNotNull(path);
+
+ Volume v = getVolumeByPath(path);
+
+ return v.getFileSystem().create(path);
}
@Override
public FSDataOutputStream create(Path path, boolean overwrite) throws IOException {
- FileSystem fs = getFileSystemByPath(path);
- return fs.create(path, overwrite);
+ checkNotNull(path);
+
+ Volume v = getVolumeByPath(path);
+
+ return v.getFileSystem().create(path, overwrite);
}
private static long correctBlockSize(Configuration conf, long blockSize) {
@@ -121,7 +145,11 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public FSDataOutputStream create(Path path, boolean overwrite, int bufferSize, short replication, long blockSize) throws IOException {
- FileSystem fs = getFileSystemByPath(path);
+ checkNotNull(path);
+
+ Volume v = getVolumeByPath(path);
+ FileSystem fs = v.getFileSystem();
+
if (bufferSize == 0) {
fs.getConf().getInt("io.file.buffer.size", 4096);
}
@@ -130,13 +158,16 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public boolean createNewFile(Path path) throws IOException {
- FileSystem fs = getFileSystemByPath(path);
- return fs.createNewFile(path);
+ checkNotNull(path);
+
+ Volume v = getVolumeByPath(path);
+ return v.getFileSystem().createNewFile(path);
}
@Override
public FSDataOutputStream createSyncable(Path logPath, int bufferSize, short replication, long blockSize) throws IOException {
- FileSystem fs = getFileSystemByPath(logPath);
+ Volume v = getVolumeByPath(logPath);
+ FileSystem fs = v.getFileSystem();
blockSize = correctBlockSize(fs.getConf(), blockSize);
bufferSize = correctBufferSize(fs.getConf(), bufferSize);
try {
@@ -174,18 +205,18 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public boolean delete(Path path) throws IOException {
- return getFileSystemByPath(path).delete(path, false);
+ return getVolumeByPath(path).getFileSystem().delete(path, false);
}
@Override
public boolean deleteRecursively(Path path) throws IOException {
- return getFileSystemByPath(path).delete(path, true);
+ return getVolumeByPath(path).getFileSystem().delete(path, true);
}
protected void ensureSyncIsEnabled() {
- for (Entry<String,? extends FileSystem> entry : getFileSystems().entrySet()) {
+ for (Entry<String,Volume> entry : getFileSystems().entrySet()) {
final String volumeName = entry.getKey();
- FileSystem fs = entry.getValue();
+ FileSystem fs = entry.getValue().getFileSystem();
if (ViewFSUtils.isViewFS(fs)) {
try {
@@ -255,19 +286,36 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public boolean exists(Path path) throws IOException {
- return getFileSystemByPath(path).exists(path);
+ return getVolumeByPath(path).getFileSystem().exists(path);
}
@Override
public FileStatus getFileStatus(Path path) throws IOException {
- return getFileSystemByPath(path).getFileStatus(path);
+ return getVolumeByPath(path).getFileSystem().getFileStatus(path);
}
@Override
- public FileSystem getFileSystemByPath(Path path) {
+ public Volume getVolumeByPath(Path path) {
if (path.toString().contains(":")) {
try {
- return path.getFileSystem(CachedConfiguration.getInstance());
+ FileSystem pathFs = path.getFileSystem(CachedConfiguration.getInstance());
+ Collection<Volume> candidateVolumes = volumesByFileSystem.get(pathFs);
+ if (null != candidateVolumes) {
+ for (Volume candidateVolume : candidateVolumes) {
+ if (candidateVolume.isValidPath(path)) {
+ return candidateVolume;
+ }
+ }
+
+ // For the same reason as we can have multiple Volumes within a single filesystem
+ // we could also not find a matching one. We should defer back to the defaultVolume
+ // e.g. volume rename with old path references
+ log.debug("Defaulting to " + defaultVolume + " as a valid volume could not be determined for " + path);
+
+ return defaultVolume;
+ }
+
+ log.debug("Could not determine volume for Path '" + path + "' from defined volumes");
} catch (IOException ex) {
throw new RuntimeException(ex);
}
@@ -276,29 +324,31 @@ public class VolumeManagerImpl implements VolumeManager {
return defaultVolume;
}
- private Map<String,? extends FileSystem> getFileSystems() {
- return volumes;
+ private Map<String,Volume> getFileSystems() {
+ return volumesByName;
}
@Override
public FileStatus[] listStatus(Path path) throws IOException {
- return getFileSystemByPath(path).listStatus(path);
+ return getVolumeByPath(path).getFileSystem().listStatus(path);
}
@Override
public boolean mkdirs(Path path) throws IOException {
- return getFileSystemByPath(path).mkdirs(path);
+ return getVolumeByPath(path).getFileSystem().mkdirs(path);
}
@Override
public FSDataInputStream open(Path path) throws IOException {
- return getFileSystemByPath(path).open(path);
+ return getVolumeByPath(path).getFileSystem().open(path);
}
@Override
public boolean rename(Path path, Path newPath) throws IOException {
- FileSystem source = getFileSystemByPath(path);
- FileSystem dest = getFileSystemByPath(newPath);
+ Volume srcVolume = getVolumeByPath(path);
+ Volume destVolume = getVolumeByPath(newPath);
+ FileSystem source = srcVolume.getFileSystem();
+ FileSystem dest = destVolume.getFileSystem();
if (source != dest) {
throw new NotImplementedException("Cannot rename files across volumes: " + path + " -> " + newPath);
}
@@ -307,14 +357,15 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public boolean moveToTrash(Path path) throws IOException {
- FileSystem fs = getFileSystemByPath(path);
+ FileSystem fs = getVolumeByPath(path).getFileSystem();
Trash trash = new Trash(fs, fs.getConf());
return trash.moveToTrash(path);
}
@Override
public short getDefaultReplication(Path path) {
- FileSystem fs = getFileSystemByPath(path);
+ Volume v = getVolumeByPath(path);
+ FileSystem fs = v.getFileSystem();
try {
// try calling hadoop 2 method
Method method = fs.getClass().getMethod("getDefaultReplication", Path.class);
@@ -336,7 +387,7 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public boolean isFile(Path path) throws IOException {
- return getFileSystemByPath(path).isFile(path);
+ return getVolumeByPath(path).getFileSystem().isFile(path);
}
public static VolumeManager get() throws IOException {
@@ -347,26 +398,30 @@ public class VolumeManagerImpl implements VolumeManager {
static private final String DEFAULT = "";
public static VolumeManager get(AccumuloConfiguration conf) throws IOException {
- Map<String,FileSystem> fileSystems = new HashMap<String,FileSystem>();
- Configuration hadoopConf = CachedConfiguration.getInstance();
- fileSystems.put(DEFAULT, VolumeConfiguration.getDefaultFilesystem(hadoopConf, conf));
- for (String space : VolumeConfiguration.getConfiguredBaseDirs(conf)) {
- if (space.equals(DEFAULT))
+ final Map<String,Volume> volumes = new HashMap<String,Volume>();
+ final Configuration hadoopConf = CachedConfiguration.getInstance();
+
+ // The "default" Volume for Accumulo (in case no volumes are specified)
+ for (String volumeUriOrDir : VolumeConfiguration.getVolumeUris(conf)) {
+ if (volumeUriOrDir.equals(DEFAULT))
+ // Cannot re-define the default volume
throw new IllegalArgumentException();
- if (space.contains(":")) {
- fileSystems.put(space, new Path(space).getFileSystem(hadoopConf));
+ // We require a URI here, fail if it doesn't look like one
+ if (volumeUriOrDir.contains(":")) {
+ volumes.put(volumeUriOrDir, VolumeConfiguration.create(new Path(volumeUriOrDir), hadoopConf));
} else {
- throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_VOLUMES.getKey() + " got " + space);
+ throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_VOLUMES.getKey() + " got " + volumeUriOrDir);
}
}
- return new VolumeManagerImpl(fileSystems, DEFAULT, conf);
+ return new VolumeManagerImpl(volumes, VolumeConfiguration.getDefaultVolume(hadoopConf, conf), conf);
}
@Override
public boolean isReady() throws IOException {
- for (FileSystem fs : getFileSystems().values()) {
+ for (Volume volume : getFileSystems().values()) {
+ FileSystem fs = volume.getFileSystem();
if (ViewFSUtils.isViewFS(fs)) {
try {
@@ -421,7 +476,7 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public FileStatus[] globStatus(Path pathPattern) throws IOException {
- return getFileSystemByPath(pathPattern).globStatus(pathPattern);
+ return getVolumeByPath(pathPattern).getFileSystem().globStatus(pathPattern);
}
@Override
@@ -476,18 +531,18 @@ public class VolumeManagerImpl implements VolumeManager {
return new Path(path);
// normalize the path
- Path fullPath = new Path(ServerConstants.getDefaultBaseDir(), fileType.getDirectory());
+ Path fullPath = new Path(defaultVolume.getBasePath(), fileType.getDirectory());
if (path.startsWith("/"))
path = path.substring(1);
fullPath = new Path(fullPath, path);
- FileSystem fs = getFileSystemByPath(fullPath);
+ FileSystem fs = getVolumeByPath(fullPath).getFileSystem();
return fs.makeQualified(fullPath);
}
@Override
public ContentSummary getContentSummary(Path dir) throws IOException {
- return getFileSystemByPath(dir).getContentSummary(dir);
+ return getVolumeByPath(dir).getFileSystem().getContentSummary(dir);
}
@Override
@@ -495,4 +550,14 @@ public class VolumeManagerImpl implements VolumeManager {
return chooser.choose(options);
}
+ @Override
+ public Volume getDefaultVolume() {
+ return defaultVolume;
+ }
+
+ @Override
+ public Collection<Volume> getVolumes() {
+ return volumesByName.values();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index da3baa6..bcfb008 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -52,7 +52,7 @@ import org.apache.log4j.Logger;
public class VolumeUtil {
private static final Logger log = Logger.getLogger(VolumeUtil.class);
-
+
private static boolean isActiveVolume(Path dir) {
// consider relative path as active and take no action
@@ -81,8 +81,10 @@ public class VolumeUtil {
}
public static String switchVolume(String path, FileType ft, List<Pair<Path,Path>> replacements) {
- if (replacements.size() == 0)
+ if (replacements.size() == 0) {
+ log.trace("Not switching volume because there are no replacements");
return null;
+ }
Path p = new Path(path);
@@ -92,10 +94,15 @@ public class VolumeUtil {
for (Pair<Path,Path> pair : replacements) {
Path key = removeTrailingSlash(pair.getFirst());
- if (key.equals(volume))
- return new Path(pair.getSecond(), ft.removeVolume(p)).toString();
+ if (key.equals(volume)) {
+ String replacement = new Path(pair.getSecond(), ft.removeVolume(p)).toString();
+ log.trace("Replacing " + path + " with " + replacement);
+ return replacement;
+ }
}
+ log.trace("Could not find replacement for " + ft + " at " + path);
+
return null;
}
@@ -119,13 +126,17 @@ public class VolumeUtil {
}
- if (numSwitched == 0)
+ if (numSwitched == 0) {
+ log.trace("Did not switch " + le);
return null;
+ }
LogEntry newLogEntry = new LogEntry(le);
newLogEntry.filename = switchedPath;
newLogEntry.logSet = switchedLogs;
+ log.trace("Switched " + le + " to " + newLogEntry);
+
return newLogEntry;
}
@@ -165,6 +176,7 @@ public class VolumeUtil {
*/
public static TabletFiles updateTabletVolumes(ZooLock zooLock, VolumeManager vm, KeyExtent extent, TabletFiles tabletFiles) throws IOException {
List<Pair<Path,Path>> replacements = ServerConstants.getVolumeReplacements();
+ log.trace("Using volume replacements: " + replacements);
List<LogEntry> logsToRemove = new ArrayList<LogEntry>();
List<LogEntry> logsToAdd = new ArrayList<LogEntry>();
@@ -239,8 +251,8 @@ public class VolumeUtil {
// this code needs to be idempotent
- FileSystem fs1 = vm.getFileSystemByPath(dir);
- FileSystem fs2 = vm.getFileSystemByPath(newDir);
+ FileSystem fs1 = vm.getVolumeByPath(dir).getFileSystem();
+ FileSystem fs2 = vm.getVolumeByPath(newDir).getFileSystem();
if (!same(fs1, dir, fs2, newDir)) {
if (fs2.exists(newDir)) {
@@ -335,4 +347,5 @@ public class VolumeUtil {
SecureRandom rand = new SecureRandom();
return new Path(path.getParent(), path.getName() + "_" + System.currentTimeMillis() + "_" + Math.abs(rand.nextInt()) + ".bak");
}
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index 925f602..206a721 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@ -40,7 +40,6 @@ import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVWriter;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.iterators.user.VersioningIterator;
import org.apache.accumulo.core.master.state.tables.TableState;
import org.apache.accumulo.core.master.thrift.MasterGoalState;
@@ -52,6 +51,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Da
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
import org.apache.accumulo.core.security.SecurityUtil;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
@@ -151,7 +151,7 @@ public class Initialize {
else
fsUri = FileSystem.getDefaultUri(conf).toString();
log.info("Hadoop Filesystem is " + fsUri);
- log.info("Accumulo data dirs are " + Arrays.asList(VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration())));
+ log.info("Accumulo data dirs are " + Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())));
log.info("Zookeeper server is " + sconf.get(Property.INSTANCE_ZK_HOST));
log.info("Checking if Zookeeper is available. If this hangs, then you need to make sure zookeeper is running");
if (!zookeeperAvailable()) {
@@ -173,7 +173,7 @@ public class Initialize {
try {
if (isInitialized(fs)) {
String instanceDfsDir = sconf.get(Property.INSTANCE_DFS_DIR);
- log.fatal("It appears the directories " + Arrays.asList(VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration()))
+ log.fatal("It appears the directories " + Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()))
+ " were previously initialized.");
String instanceVolumes = sconf.get(Property.INSTANCE_VOLUMES);
String instanceDfsUri = sconf.get(Property.INSTANCE_DFS_URI);
@@ -220,7 +220,7 @@ public class Initialize {
UUID uuid = UUID.randomUUID();
// the actual disk locations of the root table and tablets
- String[] configuredTableDirs = VolumeConfiguration.prefix(VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration()),
+ String[] configuredTableDirs = VolumeConfiguration.prefix(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()),
ServerConstants.TABLE_DIR);
final Path rootTablet = new Path(fs.choose(configuredTableDirs) + "/" + RootTable.ID + RootTable.ROOT_TABLET_LOCATION);
try {
@@ -296,7 +296,7 @@ public class Initialize {
private static void initFileSystem(Opts opts, VolumeManager fs, UUID uuid, Path rootTablet) throws IOException {
FileStatus fstat;
- initDirs(fs, uuid, VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration()), false);
+ initDirs(fs, uuid, VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()), false);
// the actual disk locations of the metadata table and tablets
final Path[] metadataTableDirs = paths(ServerConstants.getMetadataTableDirs());
@@ -341,7 +341,7 @@ public class Initialize {
// the root tablet contains the key extent and locations of all the
// metadata tablets
String initRootTabFile = rootTablet + "/00000_00000." + FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration());
- FileSystem ns = fs.getFileSystemByPath(new Path(initRootTabFile));
+ FileSystem ns = fs.getVolumeByPath(new Path(initRootTabFile)).getFileSystem();
FileSKVWriter mfw = FileOperations.getInstance().openWriter(initRootTabFile, ns, ns.getConf(), AccumuloConfiguration.getDefaultConfiguration());
mfw.startDefaultLocalityGroup();
@@ -553,7 +553,7 @@ public class Initialize {
}
public static boolean isInitialized(VolumeManager fs) throws IOException {
- for (String baseDir : VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration())) {
+ for (String baseDir : VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())) {
if (fs.exists(new Path(baseDir, ServerConstants.INSTANCE_ID_DIR)) || fs.exists(new Path(baseDir, ServerConstants.VERSION_DIR)))
return true;
}
@@ -564,10 +564,10 @@ public class Initialize {
private static void addVolumes(VolumeManager fs) throws IOException {
HashSet<String> initializedDirs = new HashSet<String>();
initializedDirs
- .addAll(Arrays.asList(ServerConstants.checkBaseDirs(VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration()), true)));
+ .addAll(Arrays.asList(ServerConstants.checkBaseUris(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()), true)));
HashSet<String> uinitializedDirs = new HashSet<String>();
- uinitializedDirs.addAll(Arrays.asList(VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration())));
+ uinitializedDirs.addAll(Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())));
uinitializedDirs.removeAll(initializedDirs);
Path aBasePath = new Path(initializedDirs.iterator().next());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
index 7edc0cf..0e7f83a 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
@@ -37,7 +37,7 @@ public class HadoopLogCloser implements LogCloser {
@Override
public long close(AccumuloConfiguration conf, VolumeManager fs, Path source) throws IOException {
- FileSystem ns = fs.getFileSystemByPath(source);
+ FileSystem ns = fs.getVolumeByPath(source).getFileSystem();
// if path points to a viewfs path, then resolve to underlying filesystem
if (ViewFSUtils.isViewFS(ns)) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
index bba7ac5..d737109 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
@@ -32,7 +32,7 @@ public class MapRLogCloser implements LogCloser {
@Override
public long close(AccumuloConfiguration conf, VolumeManager fs, Path path) throws IOException {
log.info("Recovering file " + path.toString() + " by changing permission to readonly");
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
FsPermission roPerm = new FsPermission((short) 0444);
try {
ns.setPermission(path, roPerm);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index ac13034..3f33a0e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@ -24,7 +24,7 @@ import java.util.UUID;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.ZooReader;
@@ -32,8 +32,9 @@ import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.ACL;
@@ -57,7 +58,7 @@ public class ChangeSecret {
argsList.add("--new");
argsList.addAll(Arrays.asList(args));
opts.parseArgs(ChangeSecret.class.getName(), argsList.toArray(new String[0]));
- FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+ VolumeManager fs = VolumeManagerImpl.get();
Instance inst = opts.getInstance();
if (!verifyAccumuloIsDown(inst, opts.oldPass))
System.exit(-1);
@@ -142,10 +143,13 @@ public class ChangeSecret {
return newInstanceId;
}
- private static void updateHdfs(FileSystem fs, Instance inst, String newInstanceId) throws IOException {
- fs.delete(ServerConstants.getInstanceIdLocation(), true);
- fs.mkdirs(ServerConstants.getInstanceIdLocation());
- fs.create(new Path(ServerConstants.getInstanceIdLocation(), newInstanceId)).close();
+ private static void updateHdfs(VolumeManager fs, Instance inst, String newInstanceId) throws IOException {
+ // Need to recreate the instanceId on all of them to keep consistency
+ for (Volume v : fs.getVolumes()) {
+ v.getFileSystem().delete(ServerConstants.getInstanceIdLocation(v), true);
+ v.getFileSystem().mkdirs(ServerConstants.getInstanceIdLocation(v));
+ v.getFileSystem().create(new Path(ServerConstants.getInstanceIdLocation(v), newInstanceId)).close();
+ }
}
private static void deleteInstance(Instance origInstance, String oldPass) throws Exception {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
index 8e38cbd..277a0c2 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
@@ -125,7 +125,7 @@ public class FileUtil {
String newMapFile = String.format("%s/%04d.%s", newDir, count++, RFile.EXTENSION);
outFiles.add(newMapFile);
- FileSystem ns = fs.getFileSystemByPath(new Path(newMapFile));
+ FileSystem ns = fs.getVolumeByPath(new Path(newMapFile)).getFileSystem();
FileSKVWriter writer = new RFileOperations().openWriter(newMapFile.toString(), ns, ns.getConf(), acuConf);
writer.startDefaultLocalityGroup();
List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(inFiles.size());
@@ -133,7 +133,7 @@ public class FileUtil {
FileSKVIterator reader = null;
try {
for (String s : inFiles) {
- ns = fs.getFileSystemByPath(new Path(s));
+ ns = fs.getVolumeByPath(new Path(s)).getFileSystem();
reader = FileOperations.getInstance().openIndex(s, ns, ns.getConf(), acuConf);
iters.add(reader);
}
@@ -390,7 +390,7 @@ public class FileUtil {
for (String ref : mapFiles) {
FileSKVIterator reader = null;
Path path = new Path(ref);
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
try {
if (useIndex)
reader = FileOperations.getInstance().openIndex(path.toString(), ns, ns.getConf(), acuConf);
@@ -435,7 +435,7 @@ public class FileUtil {
for (FileRef mapfile : mapfiles) {
FileSKVIterator reader = null;
- FileSystem ns = fs.getFileSystemByPath(mapfile.path());
+ FileSystem ns = fs.getVolumeByPath(mapfile.path()).getFileSystem();
try {
reader = FileOperations.getInstance().openReader(mapfile.toString(), false, ns, ns.getConf(), acuConf);
@@ -470,7 +470,7 @@ public class FileUtil {
for (FileRef ref : mapFiles) {
Path path = ref.path();
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), acuConf);
try {
@@ -513,7 +513,7 @@ public class FileUtil {
counts.put(keyExtent, new MLong(0));
Text row = new Text();
- FileSystem ns = fs.getFileSystemByPath(mapFile);
+ FileSystem ns = fs.getVolumeByPath(mapFile).getFileSystem();
FileSKVIterator index = FileOperations.getInstance().openIndex(mapFile.toString(), ns, ns.getConf(), acuConf);
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
index a96e791..35bb8f5 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
@@ -88,7 +88,7 @@ public class LocalityCheck {
}
for (String file : files) {
Path filePath = new Path(file);
- FileSystem ns = fs.getFileSystemByPath(filePath);
+ FileSystem ns = fs.getVolumeByPath(filePath).getFileSystem();
FileStatus fileStatus = ns.getFileStatus(filePath);
BlockLocation[] fileBlockLocations = ns.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
for (BlockLocation blockLocation : fileBlockLocations) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java b/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
index b237cd0..b8e7113 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
@@ -46,7 +46,7 @@ public class TabletOperations {
lowDirectory = Constants.DEFAULT_TABLET_LOCATION;
Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" + lowDirectory);
if (fs.exists(lowDirectoryPath) || fs.mkdirs(lowDirectoryPath)) {
- FileSystem pathFs = fs.getFileSystemByPath(lowDirectoryPath);
+ FileSystem pathFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem();
return lowDirectoryPath.makeQualified(pathFs.getUri(), pathFs.getWorkingDirectory()).toString();
}
log.warn("Failed to create " + lowDirectoryPath + " for unknown reason");
@@ -56,7 +56,7 @@ public class TabletOperations {
if (fs.exists(lowDirectoryPath))
throw new IllegalStateException("Dir exist when it should not " + lowDirectoryPath);
if (fs.mkdirs(lowDirectoryPath)) {
- FileSystem lowDirectoryFs = fs.getFileSystemByPath(lowDirectoryPath);
+ FileSystem lowDirectoryFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem();
return lowDirectoryPath.makeQualified(lowDirectoryFs.getUri(), lowDirectoryFs.getWorkingDirectory()).toString();
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java b/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java
index 37edb1a..7cac046 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java
@@ -41,7 +41,7 @@ public class ZooKeeperMain {
Opts opts = new Opts();
opts.parseArgs(ZooKeeperMain.class.getName(), args);
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
- String baseDir = ServerConstants.getBaseDirs()[0];
+ String baseDir = ServerConstants.getBaseUris()[0];
System.out.println("Using " + fs.makeQualified(new Path(baseDir + "/instance_id")) + " to lookup accumulo instance");
Instance instance = HdfsZooInstance.getInstance();
if (opts.servers == null) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/test/java/org/apache/accumulo/server/ServerConstantsTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/ServerConstantsTest.java b/server/base/src/test/java/org/apache/accumulo/server/ServerConstantsTest.java
index a316155..8c8b2f2 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/ServerConstantsTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/ServerConstantsTest.java
@@ -59,26 +59,26 @@ public class ServerConstantsTest {
}
private void verifyAllPass(ArrayList<String> paths) {
- Assert.assertEquals(paths, Arrays.asList(ServerConstants.checkBaseDirs(paths.toArray(new String[paths.size()]), true)));
- Assert.assertEquals(paths, Arrays.asList(ServerConstants.checkBaseDirs(paths.toArray(new String[paths.size()]), false)));
+ Assert.assertEquals(paths, Arrays.asList(ServerConstants.checkBaseUris(paths.toArray(new String[paths.size()]), true)));
+ Assert.assertEquals(paths, Arrays.asList(ServerConstants.checkBaseUris(paths.toArray(new String[paths.size()]), false)));
}
private void verifySomePass(ArrayList<String> paths, int numExpected) {
- Assert.assertEquals(paths.subList(0, 2), Arrays.asList(ServerConstants.checkBaseDirs(paths.toArray(new String[paths.size()]), true)));
+ Assert.assertEquals(paths.subList(0, 2), Arrays.asList(ServerConstants.checkBaseUris(paths.toArray(new String[paths.size()]), true)));
try {
- ServerConstants.checkBaseDirs(paths.toArray(new String[paths.size()]), false);
+ ServerConstants.checkBaseUris(paths.toArray(new String[paths.size()]), false);
Assert.fail();
} catch (Exception e) {}
}
private void verifyError(ArrayList<String> paths) {
try {
- ServerConstants.checkBaseDirs(paths.toArray(new String[paths.size()]), true);
+ ServerConstants.checkBaseUris(paths.toArray(new String[paths.size()]), true);
Assert.fail();
} catch (Exception e) {}
try {
- ServerConstants.checkBaseDirs(paths.toArray(new String[paths.size()]), false);
+ ServerConstants.checkBaseUris(paths.toArray(new String[paths.size()]), false);
Assert.fail();
} catch (Exception e) {}
}
[09/25] git commit: ACCUMULO-2061 Only append a trailing slash when
one doesn't exist on the base dir
Posted by el...@apache.org.
ACCUMULO-2061 Only append a trailing slash when one doesn't exist on the base dir
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/492768d3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/492768d3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/492768d3
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 492768d38221af64b71ba37bb8979d7bb841e738
Parents: 7d48b1a
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 16:25:19 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:59:19 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/core/file/rfile/PrintInfo.java | 2 +-
.../org/apache/accumulo/core/volume/VolumeConfiguration.java | 6 +++++-
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/492768d3/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 7c0f067..7ed8f34 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -52,7 +52,7 @@ public class PrintInfo {
@SuppressWarnings("deprecation")
AccumuloConfiguration aconf = AccumuloConfiguration.getSiteConfiguration();
- // TODO This will only work for RFiles in HDFS when the filesystem is defined in the core-site.xml
+ // TODO ACCUMULO-2462 This will only work for RFiles in HDFS when the filesystem is defined in the core-site.xml
// on the classpath if a path, and not a URI, is given
FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem();
FileSystem localFs = FileSystem.getLocal(conf);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/492768d3/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
index 5db5bb2..71ad611 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
@@ -123,7 +123,11 @@ public class VolumeConfiguration {
suffix = suffix.substring(1);
String result[] = new String[bases.length];
for (int i = 0; i < bases.length; i++) {
- result[i] = bases[i] + "/" + suffix;
+ if (bases[i].endsWith("/")) {
+ result[i] = bases[i] + suffix;
+ } else {
+ result[i] = bases[i] + "/" + suffix;
+ }
}
return result;
}
[06/25] git commit: ACCUMULO-2451 Update the data version on all
volumes instead of just one
Posted by el...@apache.org.
ACCUMULO-2451 Update the data version on all volumes instead of just one
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d45b723f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d45b723f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d45b723f
Branch: refs/heads/master
Commit: d45b723f0df73fa3f24d9911da44481615c7298b
Parents: 7c94c08
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 12:10:44 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:58:41 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/server/Accumulo.java | 22 +++++++++++++-------
1 file changed, 14 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d45b723f/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index 48534f0..f7f2298 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -51,15 +51,21 @@ public class Accumulo {
private static final Logger log = Logger.getLogger(Accumulo.class);
public static synchronized void updateAccumuloVersion(VolumeManager fs) {
- // TODO ACCUMULO-2451 Should update all volumes, not one
- Volume volume = fs.getVolumes().iterator().next();
- try {
- if (getAccumuloPersistentVersion(fs) == ServerConstants.PREV_DATA_VERSION) {
- fs.create(new Path(ServerConstants.getDataVersionLocation(volume), Integer.toString(ServerConstants.DATA_VERSION)));
- fs.delete(new Path(ServerConstants.getDataVersionLocation(volume), Integer.toString(ServerConstants.PREV_DATA_VERSION)));
+ for (Volume volume : fs.getVolumes()) {
+ try {
+ if (getAccumuloPersistentVersion(fs) == ServerConstants.PREV_DATA_VERSION) {
+ log.debug("Attempting to upgrade " + volume);
+ Path dataVersionLocation = ServerConstants.getDataVersionLocation(volume);
+ fs.create(new Path(dataVersionLocation, Integer.toString(ServerConstants.DATA_VERSION))).close();
+
+ Path prevDataVersionLoc = new Path(dataVersionLocation, Integer.toString(ServerConstants.PREV_DATA_VERSION));
+ if (!fs.delete(prevDataVersionLoc)) {
+ throw new RuntimeException("Could not delete previous data version location (" + prevDataVersionLoc + ") for " + volume);
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
}
- } catch (IOException e) {
- throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
}
}
[18/25] git commit: ACCUMULO-2061 Co-locate volumes with the MAC dir
Posted by el...@apache.org.
ACCUMULO-2061 Co-locate volumes with the MAC dir
Losing the volumes on test failure sucks because you can't validate
if we misplaced a file on the wrong volume in the first place.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/53b9f25d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/53b9f25d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/53b9f25d
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 53b9f25db94d812f78fd468e88e05da33fdd5190
Parents: 9172f52
Author: Josh Elser <el...@apache.org>
Authored: Thu Mar 20 12:18:42 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:59:30 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/test/VolumeIT.java | 26 ++++++--------------
1 file changed, 7 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/53b9f25d/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index 0a7ef3f..c0e41c1 100644
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertTrue;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
-import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
@@ -65,7 +64,6 @@ import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.init.Initialize;
import org.apache.accumulo.server.util.Admin;
import org.apache.accumulo.test.functional.ConfigurableMacIT;
-import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -73,38 +71,28 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.Text;
import org.apache.zookeeper.ZooKeeper;
-import org.junit.After;
import org.junit.Assert;
-import org.junit.BeforeClass;
import org.junit.Test;
public class VolumeIT extends ConfigurableMacIT {
private static final Text EMPTY = new Text();
private static final Value EMPTY_VALUE = new Value(new byte[] {});
- public static File volDirBase;
- public static Path v1;
- public static Path v2;
+ private File volDirBase;
+ private Path v1, v2;
- @BeforeClass
- public static void createVolumeDirs() throws IOException {
- volDirBase = createSharedTestDir(VolumeIT.class.getName() + "-volumes");
+ @SuppressWarnings("deprecation")
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ File baseDir = cfg.getDir();
+ volDirBase = new File(baseDir, "volumes");
File v1f = new File(volDirBase, "v1");
File v2f = new File(volDirBase, "v2");
v1f.mkdir();
v2f.mkdir();
v1 = new Path("file://" + v1f.getAbsolutePath());
v2 = new Path("file://" + v2f.getAbsolutePath());
- }
- @After
- public void clearDirs() throws IOException {
- FileUtils.deleteQuietly(new File(v1.getParent().toUri()));
- }
-
- @SuppressWarnings("deprecation")
- @Override
- public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
// Run MAC on two locations in the local file system
URI v1Uri = v1.toUri();
cfg.setProperty(Property.INSTANCE_DFS_DIR, v1Uri.getPath());
[03/25] ACCUMULO-2061 Initial implementation to remove
instance.dfs.dir from usage of instance.volumes
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java
index 205a793..ad29c19 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java
@@ -27,28 +27,50 @@ import org.junit.Test;
public class FileTypeTest {
@Test
public void testVolumeExtraction() {
- Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:/a/accumulo"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:///a/accumulo"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:///a/accumulo"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:/a/accumulo"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
+ // Having an 'accumulo' directory is not a requirement
+ Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:/a/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:/a/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:///a/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:///a/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:///accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:/a"), FileType.WAL.getVolume(new Path("file:/a/accumulo/wal/1.2.3.4/aaa-bbb-ccc-ddd")));
+ // Having an 'accumulo' directory is not a requirement
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:/a/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:///a/tables/2b/t-001/C00.rf")));
+
+ Assert.assertEquals(new Path("file:/accumulo"), FileType.TABLE.getVolume(new Path("file:/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:/accumulo"), FileType.TABLE.getVolume(new Path("file:///accumulo/tables/2b/t-001/C00.rf")));
+
+ // Having an 'accumulo' directory is not a requirement
+ Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:///tables/2b/t-001/C00.rf")));
+
+ Assert.assertEquals(new Path("file:/a"), FileType.WAL.getVolume(new Path("file:/a/wal/1.2.3.4/aaa-bbb-ccc-ddd")));
Assert.assertNull(FileType.WAL.getVolume(new Path("1.2.3.4/aaa-bbb-ccc-ddd")));
Assert.assertNull(FileType.TABLE.getVolume(new Path("../2b/t-001/C00.rf")));
Assert.assertNull(FileType.TABLE.getVolume(new Path("/t-001/C00.rf")));
- Assert.assertEquals(new Path("hdfs://nn1/"), FileType.TABLE.getVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("hdfs://nn1/a/"), FileType.TABLE.getVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("hdfs://nn1/accumulo"), FileType.TABLE.getVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("hdfs://nn1/a/accumulo"), FileType.TABLE.getVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf")));
+
+ // Having an 'accumulo' directory is not a requirement
+ Assert.assertEquals(new Path("hdfs://nn1/"), FileType.TABLE.getVolume(new Path("hdfs://nn1/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("hdfs://nn1/a"), FileType.TABLE.getVolume(new Path("hdfs://nn1/a/tables/2b/t-001/C00.rf")));
+
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf")));
+ // Having an 'accumulo' directory is not a requirement
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/a/tables/2b/t-001/C00.rf")));
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
index c85be45..3b905c9 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
@@ -43,9 +43,9 @@ public class VolumeUtilTest {
@Test
public void testSwitchVolume() {
List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/"), new Path("viewfs:/b")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/b/accumulo")));
Assert.assertEquals("viewfs:/a/accumulo/tables/t-00000/C000.rf",
VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
@@ -57,9 +57,9 @@ public class VolumeUtilTest {
Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
replacements.clear();
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/"), new Path("viewfs:/b")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/b/accumulo")));
Assert.assertEquals("viewfs:/a/accumulo/tables/t-00000/C000.rf",
VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
@@ -73,6 +73,70 @@ public class VolumeUtilTest {
}
@Test
+ public void testSwitchVolumesDifferentSourceDepths() {
+ List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/b")));
+
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/b/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/a/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+
+ replacements.clear();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/b")));
+
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/b/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/d2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/a/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ }
+
+ @Test
+ public void testSwitchVolumesDifferentTargetDepths() {
+ List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/path3")));
+
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path3/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/path1/path2/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+
+ replacements.clear();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/path3")));
+
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path3/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/d2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/path1/path2/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ }
+
+ @Test
public void testSame() throws Exception {
FileSystem fs = FileSystem.getLocal(new Configuration());
@@ -133,6 +197,19 @@ public class VolumeUtilTest {
}
+ @Test
+ public void testRootTableReplacement() throws IOException {
+ List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
+ replacements.add(new Pair<Path,Path>(new Path("file:/foo/v1"), new Path("file:/foo/v8")));
+ replacements.add(new Pair<Path,Path>(new Path("file:/foo/v2"), new Path("file:/foo/v9")));
+
+ FileType ft = FileType.TABLE;
+
+ Assert.assertEquals("file:/foo/v8/tables/+r/root_tablet",
+ VolumeUtil.switchVolume("file:/foo/v1/tables/+r/root_tablet",
+ ft, replacements));
+ }
+
private void writeFile(FileSystem fs, Path dir, String filename, String data) throws IOException {
FSDataOutputStream out = fs.create(new Path(dir, filename));
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
index 4e2a878..1e9c1dd 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
@@ -335,6 +335,71 @@ public class GarbageCollectionTest {
assertRemoved(gce);
}
+
+ @Test
+ public void testCustomDirectories() throws Exception {
+ TestGCE gce = new TestGCE();
+
+ gce.candidates.add("/4/t-0");
+ gce.candidates.add("/4/t-0/F002.rf");
+ gce.candidates.add("hdfs://foo.com:6000/user/foo/tables/5/t-0");
+ gce.candidates.add("/6/t-0");
+ gce.candidates.add("hdfs://foo:6000/user/foo/tables/7/t-0/");
+ gce.candidates.add("/8/t-0");
+ gce.candidates.add("hdfs://foo:6000/user/foo/tables/9/t-0");
+ gce.candidates.add("/a/t-0");
+ gce.candidates.add("hdfs://foo:6000/user/foo/tables/b/t-0");
+ gce.candidates.add("/c/t-0");
+ gce.candidates.add("hdfs://foo:6000/user/foo/tables/d/t-0");
+
+ gce.addDirReference("4", null, "/t-0");
+ gce.addDirReference("5", null, "/t-0");
+ gce.addDirReference("6", null, "hdfs://foo.com:6000/user/foo/tables/6/t-0");
+ gce.addDirReference("7", null, "hdfs://foo.com:6000/user/foo/tables/7/t-0");
+
+ gce.addFileReference("8", "m", "/t-0/F00.rf");
+ gce.addFileReference("9", "m", "/t-0/F00.rf");
+
+ gce.addFileReference("a", "m", "hdfs://foo.com:6000/user/foo/tables/a/t-0/F00.rf");
+ gce.addFileReference("b", "m", "hdfs://foo.com:6000/user/foo/tables/b/t-0/F00.rf");
+
+ gce.addFileReference("e", "m", "../c/t-0/F00.rf");
+ gce.addFileReference("f", "m", "../d/t-0/F00.rf");
+
+ GarbageCollectionAlgorithm gca = new GarbageCollectionAlgorithm();
+
+ // A directory reference does not preclude a candidate file beneath that directory from deletion
+ gca.collect(gce);
+ assertRemoved(gce, "/4/t-0/F002.rf");
+
+ // Removing the dir reference for a table will delete all tablet directories
+ gce.removeDirReference("5", null);
+ gca.collect(gce);
+ assertRemoved(gce, "hdfs://foo.com:6000/user/foo/tables/5/t-0");
+
+ gce.removeDirReference("4", null);
+ gca.collect(gce);
+ assertRemoved(gce, "/4/t-0");
+
+ gce.removeDirReference("6", null);
+ gce.removeDirReference("7", null);
+ gca.collect(gce);
+ assertRemoved(gce, "/6/t-0", "hdfs://foo:6000/user/foo/tables/7/t-0/");
+
+ gce.removeFileReference("8", "m", "/t-0/F00.rf");
+ gce.removeFileReference("9", "m", "/t-0/F00.rf");
+ gce.removeFileReference("a", "m", "hdfs://foo.com:6000/user/foo/tables/a/t-0/F00.rf");
+ gce.removeFileReference("b", "m", "hdfs://foo.com:6000/user/foo/tables/b/t-0/F00.rf");
+ gce.removeFileReference("e", "m", "../c/t-0/F00.rf");
+ gce.removeFileReference("f", "m", "../d/t-0/F00.rf");
+ gca.collect(gce);
+ assertRemoved(gce, "/8/t-0", "hdfs://foo:6000/user/foo/tables/9/t-0", "/a/t-0", "hdfs://foo:6000/user/foo/tables/b/t-0", "/c/t-0",
+ "hdfs://foo:6000/user/foo/tables/d/t-0");
+
+ gca.collect(gce);
+ assertRemoved(gce);
+ }
+
private void badRefTest(String ref) {
TestGCE gce = new TestGCE();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
index 36bbb53..eeb9b16 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
@@ -152,7 +152,7 @@ class WriteExportFiles extends MasterRepo {
public static void exportTable(VolumeManager fs, Connector conn, String tableName, String tableID, String exportDir) throws Exception {
fs.mkdirs(new Path(exportDir));
- Path exportMetaFilePath = fs.getFileSystemByPath(new Path(exportDir)).makeQualified(new Path(exportDir, Constants.EXPORT_FILE));
+ Path exportMetaFilePath = fs.getVolumeByPath(new Path(exportDir)).getFileSystem().makeQualified(new Path(exportDir, Constants.EXPORT_FILE));
FSDataOutputStream fileOut = fs.create(exportMetaFilePath, false);
ZipOutputStream zipOut = new ZipOutputStream(fileOut);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
index 7e84c55..cd59b78 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
@@ -427,7 +427,7 @@ class ImportPopulateZookeeper extends MasterRepo {
Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
try {
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
return TableOperationsImpl.getExportedProps(ns, path);
} catch (IOException ioe) {
throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
index 942f866..31e63ed 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
@@ -33,11 +33,11 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.conf.DefaultConfiguration;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
import org.apache.accumulo.core.util.Duration;
import org.apache.accumulo.core.util.NumUtil;
import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.monitor.Monitor;
import org.apache.accumulo.monitor.ZooKeeperStatus;
import org.apache.accumulo.monitor.ZooKeeperStatus.ZooKeeperState;
@@ -268,9 +268,9 @@ public class DefaultServlet extends BasicServlet {
long totalHdfsBytesUsed = 0l;
try {
- for (String baseDir : VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration())) {
+ for (String baseDir : VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())) {
final Path basePath = new Path(baseDir);
- final FileSystem fs = vm.getFileSystemByPath(basePath);
+ final FileSystem fs = vm.getVolumeByPath(basePath).getFileSystem();
try {
// Calculate the amount of space used by Accumulo on the FileSystem
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
index e9f1083..6b31af1 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.trace.TraceFileSystem;
import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor;
@@ -32,35 +33,35 @@ import org.apache.log4j.Logger;
* Copy failed bulk imports.
*/
public class BulkFailedCopyProcessor implements Processor {
-
+
private static final Logger log = Logger.getLogger(BulkFailedCopyProcessor.class);
-
+
@Override
public Processor newProcessor() {
return new BulkFailedCopyProcessor();
}
-
+
@Override
public void process(String workID, byte[] data) {
-
+
String paths[] = new String(data, Constants.UTF8).split(",");
-
+
Path orig = new Path(paths[0]);
Path dest = new Path(paths[1]);
Path tmp = new Path(dest.getParent(), dest.getName() + ".tmp");
-
+
try {
- FileSystem fs = TraceFileSystem.wrap(org.apache.accumulo.core.file.VolumeConfiguration.getDefaultFilesystem(CachedConfiguration.getInstance(),
- ServerConfiguration.getSiteConfiguration()));
-
+ FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(CachedConfiguration.getInstance(),
+ ServerConfiguration.getSiteConfiguration()).getFileSystem());
+
FileUtil.copy(fs, orig, fs, tmp, false, true, CachedConfiguration.getInstance());
fs.rename(tmp, dest);
log.debug("copied " + orig + " to " + dest);
} catch (IOException ex) {
try {
- FileSystem fs = TraceFileSystem.wrap(org.apache.accumulo.core.file.VolumeConfiguration.getDefaultFilesystem(CachedConfiguration.getInstance(),
- ServerConfiguration.getSiteConfiguration()));
-
+ FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(CachedConfiguration.getInstance(),
+ ServerConfiguration.getSiteConfiguration()).getFileSystem());
+
fs.create(dest).close();
log.warn(" marked " + dest + " failed", ex);
} catch (IOException e) {
@@ -69,5 +70,5 @@ public class BulkFailedCopyProcessor implements Processor {
}
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
index 151db6e..822171c 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
@@ -336,7 +336,7 @@ public class Compactor implements Callable<CompactionStats> {
thread = Thread.currentThread();
try {
FileOperations fileFactory = FileOperations.getInstance();
- FileSystem ns = this.fs.getFileSystemByPath(outputFile.path());
+ FileSystem ns = this.fs.getVolumeByPath(outputFile.path()).getFileSystem();
mfw = fileFactory.openWriter(outputFile.path().toString(), ns, ns.getConf(), acuTableConf);
Map<String,Set<ByteSequence>> lGroups;
@@ -421,7 +421,7 @@ public class Compactor implements Callable<CompactionStats> {
try {
FileOperations fileFactory = FileOperations.getInstance();
- FileSystem fs = this.fs.getFileSystemByPath(mapFile.path());
+ FileSystem fs = this.fs.getVolumeByPath(mapFile.path()).getFileSystem();
FileSKVIterator reader;
reader = fileFactory.openReader(mapFile.path().toString(), false, fs, conf, acuTableConf);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
index bb95532..8bf2517 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
@@ -309,7 +309,7 @@ public class FileManager {
if (!file.contains(":"))
throw new IllegalArgumentException("Expected uri, got : " + file);
Path path = new Path(file);
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
//log.debug("Opening "+file + " path " + path);
FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), false, ns, ns.getConf(), conf.getTableConfiguration(table.toString()),
dataCache, indexCache);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
index cc4b68d..3fe60b7 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
@@ -1260,7 +1260,7 @@ public class Tablet {
long rtime = Long.MIN_VALUE;
for (FileRef ref : datafiles.keySet()) {
Path path = ref.path();
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), tabletServer.getTableConfiguration(extent));
long maxTime = -1;
try {
@@ -2975,7 +2975,7 @@ public class Tablet {
FileOperations fileFactory = FileOperations.getInstance();
for (Entry<FileRef,DataFileValue> entry : allFiles.entrySet()) {
FileRef file = entry.getKey();
- FileSystem ns = fs.getFileSystemByPath(file.path());
+ FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
FileSKVIterator openReader = fileFactory.openReader(file.path().toString(), true, ns, ns.getConf(), this.getTableConfiguration());
try {
Key first = openReader.getFirstKey();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 475621b..6d73125 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -969,7 +969,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
Map<FileRef,MapFileInfo> fileRefMap = new HashMap<FileRef,MapFileInfo>();
for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
Path path = new Path(mapping.getKey());
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
path = ns.makeQualified(path);
fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
index 3bbb476..900600f 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
@@ -81,7 +81,7 @@ public class MajorCompactionRequest implements Cloneable {
// @TODO verify the file isn't some random file in HDFS
// @TODO ensure these files are always closed?
FileOperations fileFactory = FileOperations.getInstance();
- FileSystem ns = volumeManager.getFileSystemByPath(ref.path());
+ FileSystem ns = volumeManager.getVolumeByPath(ref.path()).getFileSystem();
FileSKVIterator openReader = fileFactory.openReader(ref.path().toString(), true, ns, ns.getConf(), tableConfig);
return openReader;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
index 8f783c3..bb8e3c7 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
@@ -160,7 +160,7 @@ public class LogSorter {
private void writeBuffer(String destPath, ArrayList<Pair<LogFileKey,LogFileValue>> buffer, int part) throws IOException {
Path path = new Path(destPath, String.format("part-r-%05d", part++));
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
@SuppressWarnings("deprecation")
MapFile.Writer output = new MapFile.Writer(ns.getConf(), ns, path.toString(), LogFileKey.class, LogFileValue.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java
index a28bac4..541f075 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java
@@ -97,7 +97,7 @@ public class MultiReader {
foundFinish = true;
continue;
}
- FileSystem ns = fs.getFileSystemByPath(child.getPath());
+ FileSystem ns = fs.getVolumeByPath(child.getPath()).getFileSystem();
heap.add(new Index(new Reader(ns, child.getPath().toString(), ns.getConf())));
}
if (!foundFinish)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java
index 1cd8f12..7a1c84b 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java
@@ -116,6 +116,7 @@ public class RootFilesTest {
ConfigurationCopy conf = new ConfigurationCopy();
conf.set(Property.INSTANCE_DFS_URI, "file:///");
+ conf.set(Property.INSTANCE_DFS_DIR, "/");
VolumeManager vm = VolumeManagerImpl.get(conf);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
index 50c8b31..dad9a75 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
@@ -22,6 +22,8 @@ import java.util.Map;
import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.volume.VolumeImpl;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
@@ -44,7 +46,7 @@ public class TabletServerSyncCheckTest {
conf.set(DFS_DURABLE_SYNC, "false");
FileSystem fs = new TestFileSystem(conf);
- TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.of("foo", fs));
+ TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.<String,Volume> of("foo", new VolumeImpl(fs, "/")));
vm.ensureSyncIsEnabled();
}
@@ -56,7 +58,7 @@ public class TabletServerSyncCheckTest {
FileSystem fs1 = new TestFileSystem(conf1);
FileSystem fs2 = new TestFileSystem(conf2);
- TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.of("bar", fs2, "foo", fs1));
+ TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.<String,Volume> of("bar", new VolumeImpl(fs2, "/"), "foo", new VolumeImpl(fs1, "/")));
vm.ensureSyncIsEnabled();
}
@@ -67,7 +69,7 @@ public class TabletServerSyncCheckTest {
conf.set(DFS_SUPPORT_APPEND, "false");
FileSystem fs = new TestFileSystem(conf);
- TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.of("foo", fs));
+ TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.<String,Volume> of("foo", new VolumeImpl(fs, "/")));
vm.ensureSyncIsEnabled();
}
@@ -89,8 +91,8 @@ public class TabletServerSyncCheckTest {
private class TestVolumeManagerImpl extends VolumeManagerImpl {
- public TestVolumeManagerImpl(Map<String,? extends FileSystem> volumes) {
- super(volumes, volumes.keySet().iterator().next(), new ConfigurationCopy(Collections.<String,String> emptyMap()));
+ public TestVolumeManagerImpl(Map<String,Volume> volumes) {
+ super(volumes, volumes.values().iterator().next(), new ConfigurationCopy(Collections.<String,String> emptyMap()));
}
@Override
@@ -149,7 +151,7 @@ public class TabletServerSyncCheckTest {
}
@Override
- public FileSystem getFileSystemByPath(Path path) {
+ public Volume getVolumeByPath(Path path) {
return null;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java
index c4d3dfb..a79e77e 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java
@@ -42,13 +42,13 @@ public class MultiReaderTest {
@Before
public void setUp() throws Exception {
- fs = VolumeManagerImpl.getLocal();
root.create();
- String path = root.getRoot().getAbsolutePath();
- Path root = new Path("file://" + path + "/manyMaps");
+ String path = root.getRoot().getAbsolutePath() + "/manyMaps";
+ fs = VolumeManagerImpl.getLocal(path);
+ Path root = new Path("file://" + path);
fs.mkdirs(root);
fs.create(new Path(root, "finished")).close();
- FileSystem ns = fs.getFileSystemByPath(root);
+ FileSystem ns = fs.getVolumeByPath(root).getFileSystem();
@SuppressWarnings("deprecation")
Writer oddWriter = new Writer(ns.getConf(), ns, new Path(root, "odd").toString(), IntWritable.class, BytesWritable.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
index 359bfa1..fffa15e 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
@@ -117,14 +117,15 @@ public class SortedLogRecoveryTest {
private static List<Mutation> recover(Map<String,KeyValue[]> logs, Set<String> files, KeyExtent extent) throws IOException {
TemporaryFolder root = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
root.create();
- final String workdir = "file://" + root.getRoot().getAbsolutePath() + "/workdir";
- VolumeManager fs = VolumeManagerImpl.getLocal();
- fs.deleteRecursively(new Path(workdir));
+ final String workdir = root.getRoot().getAbsolutePath() + "/workdir";
+ VolumeManager fs = VolumeManagerImpl.getLocal(workdir);
+ final Path workdirPath = new Path("file://" + workdir);
+ fs.deleteRecursively(workdirPath);
ArrayList<Path> dirs = new ArrayList<Path>();
try {
for (Entry<String,KeyValue[]> entry : logs.entrySet()) {
String path = workdir + "/" + entry.getKey();
- FileSystem ns = fs.getFileSystemByPath(new Path(path));
+ FileSystem ns = fs.getVolumeByPath(new Path(path)).getFileSystem();
@SuppressWarnings("deprecation")
Writer map = new MapFile.Writer(ns.getConf(), ns, path + "/log1", LogFileKey.class, LogFileValue.class);
for (KeyValue lfe : entry.getValue()) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
index af149fa..d6c23e3 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
@@ -61,10 +61,10 @@ public class TestUpgradePathForWALogs {
public void setUp() throws Exception {
// quiet log messages about compress.CodecPool
Logger.getRootLogger().setLevel(Level.ERROR);
- fs = VolumeManagerImpl.getLocal();
root.create();
- String path = root.getRoot().getAbsolutePath();
- Path manyMapsPath = new Path("file://" + path + "/manyMaps");
+ String path = root.getRoot().getAbsolutePath() + "/manyMaps";
+ fs = VolumeManagerImpl.getLocal(path);
+ Path manyMapsPath = new Path("file://" + path);
fs.mkdirs(manyMapsPath);
fs.create(new Path(manyMapsPath, "finished")).close();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
index 9a9cad7..d2c8d0f 100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
@@ -398,7 +398,7 @@ public class CollectTabletStats {
// assume it is a map file
status = fs.getFileStatus(new Path(file + "/data"));
}
- FileSystem ns = fs.getFileSystemByPath(file.path());
+ FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
BlockLocation[] locs = ns.getFileBlockLocations(status, 0, status.getLen());
System.out.println("\t\t\tBlocks for : " + file);
@@ -445,7 +445,7 @@ public class CollectTabletStats {
HashSet<ByteSequence> columnSet = createColumnBSS(columns);
for (FileRef file : files) {
- FileSystem ns = fs.getFileSystemByPath(file.path());
+ FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().openReader(file.path().toString(), false, ns, ns.getConf(), aconf);
Range range = new Range(ke.getPrevEndRow(), false, ke.getEndRow(), true);
reader.seek(range, columnSet, columnSet.size() == 0 ? false : true);
@@ -475,7 +475,7 @@ public class CollectTabletStats {
List<SortedKeyValueIterator<Key,Value>> readers = new ArrayList<SortedKeyValueIterator<Key,Value>>(files.size());
for (FileRef file : files) {
- FileSystem ns = fs.getFileSystemByPath(file.path());
+ FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
readers.add(FileOperations.getInstance().openReader(file.path().toString(), false, ns, ns.getConf(), aconf.getConfiguration()));
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index a0efe45..a7f7556 100644
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@ -23,6 +23,7 @@ import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -104,8 +105,9 @@ public class VolumeIT extends ConfigurableMacIT {
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
// Run MAC on two locations in the local file system
- cfg.setProperty(Property.INSTANCE_DFS_URI, v1.toString());
- cfg.setProperty(Property.INSTANCE_DFS_DIR, "/accumulo");
+ URI v1Uri = v1.toUri();
+ cfg.setProperty(Property.INSTANCE_DFS_DIR, v1Uri.getPath());
+ cfg.setProperty(Property.INSTANCE_DFS_URI, v1Uri.getScheme() + v1Uri.getHost());
cfg.setProperty(Property.INSTANCE_VOLUMES, v1.toString() + "," + v2.toString());
// use raw local file system so walogs sync and flush will work
@@ -114,7 +116,7 @@ public class VolumeIT extends ConfigurableMacIT {
super.configure(cfg, hadoopCoreSite);
}
- @Test
+ @Test(timeout = 2 * 60 * 1000)
public void test() throws Exception {
// create a table
Connector connector = getConnector();
@@ -176,7 +178,7 @@ public class VolumeIT extends ConfigurableMacIT {
Assert.assertEquals(expected, actual);
}
- @Test
+ @Test(timeout = 2 * 60 * 1000)
public void testRelativePaths() throws Exception {
List<String> expected = new ArrayList<String>();
@@ -292,9 +294,8 @@ public class VolumeIT extends ConfigurableMacIT {
// check that all volumes are initialized
for (Path volumePath : Arrays.asList(v1, v2, v3)) {
FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
- Path vp = new Path(volumePath, "accumulo");
- Path vpi = new Path(vp, ServerConstants.INSTANCE_ID_DIR);
- FileStatus[] iids = fs.listStatus(vpi);
+ Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
+ FileStatus[] iids = fs.listStatus(vp);
Assert.assertEquals(1, iids.length);
Assert.assertEquals(uuid, iids[0].getPath().getName());
}
@@ -390,7 +391,7 @@ public class VolumeIT extends ConfigurableMacIT {
Assert.assertEquals(200, sum);
}
- @Test
+ @Test(timeout = 5 * 60 * 1000)
public void testRemoveVolumes() throws Exception {
String[] tableNames = getTableNames(2);
@@ -447,12 +448,12 @@ public class VolumeIT extends ConfigurableMacIT {
File v1f = new File(v1.toUri());
File v8f = new File(new File(v1.getParent().toUri()), "v8");
- v1f.renameTo(v8f);
+ Assert.assertTrue("Failed to rename " + v1f + " to " + v8f, v1f.renameTo(v8f));
Path v8 = new Path(v8f.toURI());
File v2f = new File(v2.toUri());
File v9f = new File(new File(v2.getParent().toUri()), "v9");
- v2f.renameTo(v9f);
+ Assert.assertTrue("Failed to rename " + v2f + " to " + v9f, v2f.renameTo(v9f));
Path v9 = new Path(v9f.toURI());
Configuration conf = new Configuration(false);
@@ -494,12 +495,12 @@ public class VolumeIT extends ConfigurableMacIT {
verifyVolumesUsed(tableNames[2], true, v8, v9);
}
- @Test
+ @Test(timeout = 5 * 60 * 1000)
public void testCleanReplaceVolumes() throws Exception {
testReplaceVolume(true);
}
- @Test
+ @Test(timeout = 5 * 60 * 1000)
public void testDirtyReplaceVolumes() throws Exception {
testReplaceVolume(false);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
index c8023c0..eee093b 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
@@ -29,9 +29,9 @@ import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVWriter;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.file.rfile.RFile;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.trace.TraceFileSystem;
import org.apache.hadoop.conf.Configuration;
@@ -53,7 +53,7 @@ public class BulkFileIT extends SimpleMacIT {
c.tableOperations().addSplits(tableName, splits);
Configuration conf = new Configuration();
AccumuloConfiguration aconf = ServerConfiguration.getDefaultConfiguration();
- FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultFilesystem(conf, aconf));
+ FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem());
String dir = rootPath() + "/bulk_test_diff_files_89723987592_" + getTableNames(1)[0];
[14/25] git commit: ACCUMULO-2061 Fixing whitespace nits
Posted by el...@apache.org.
ACCUMULO-2061 Fixing whitespace nits
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/03baf916
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/03baf916
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/03baf916
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 03baf91611120dfdd36eaa5a824b99144b9eb36a
Parents: 53b9f25
Author: Josh Elser <el...@apache.org>
Authored: Thu Mar 20 13:23:04 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:59:30 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/volume/NonConfiguredVolume.java | 2 +-
.../org/apache/accumulo/core/volume/VolumeConfiguration.java | 4 ++--
.../main/java/org/apache/accumulo/core/volume/VolumeImpl.java | 2 +-
.../org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java | 3 ---
4 files changed, 4 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/03baf916/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
index 3cfd1c2..3d56fa9 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.fs.Path;
/**
* Volume implementation which represents a Volume for which we have a FileSystem but no base path because it is not configured via
* {@link Property#INSTANCE_VOLUMES}
- *
+ *
* This is useful to handle volumes that have been removed from accumulo-site.xml but references to these volumes have not been updated. This Volume should
* never be used to create new files, only to read existing files.
*/
http://git-wip-us.apache.org/repos/asf/accumulo/blob/03baf916/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
index 71ad611..e7a51d7 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
@@ -85,7 +85,7 @@ public class VolumeConfiguration {
/**
* Compute the URIs to be used by Accumulo
- *
+ *
* @param conf
* @return
*/
@@ -134,7 +134,7 @@ public class VolumeConfiguration {
/**
* Create a Volume with the given FileSystem that writes to the default path
- *
+ *
* @param fs
* A FileSystem to write to
* @return A Volume instance writing to the given FileSystem in the default path
http://git-wip-us.apache.org/repos/asf/accumulo/blob/03baf916/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
index f902c35..43ab96b 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
/**
- * Basic Volume implementation that contains a FileSystem and a base path
+ * Basic Volume implementation that contains a FileSystem and a base path
* that should be used within that filesystem.
*/
public class VolumeImpl implements Volume {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/03baf916/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java b/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
index 937baf8..8d4a800 100644
--- a/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
@@ -25,9 +25,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-/**
- *
- */
public class NonConfiguredVolumeTest {
private NonConfiguredVolume volume;
[05/25] git commit: ACCUMULO-2451 Update the data version on all
volumes instead of just one
Posted by el...@apache.org.
ACCUMULO-2451 Update the data version on all volumes instead of just one
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d45b723f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d45b723f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d45b723f
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: d45b723f0df73fa3f24d9911da44481615c7298b
Parents: 7c94c08
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 12:10:44 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:58:41 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/server/Accumulo.java | 22 +++++++++++++-------
1 file changed, 14 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d45b723f/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index 48534f0..f7f2298 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -51,15 +51,21 @@ public class Accumulo {
private static final Logger log = Logger.getLogger(Accumulo.class);
public static synchronized void updateAccumuloVersion(VolumeManager fs) {
- // TODO ACCUMULO-2451 Should update all volumes, not one
- Volume volume = fs.getVolumes().iterator().next();
- try {
- if (getAccumuloPersistentVersion(fs) == ServerConstants.PREV_DATA_VERSION) {
- fs.create(new Path(ServerConstants.getDataVersionLocation(volume), Integer.toString(ServerConstants.DATA_VERSION)));
- fs.delete(new Path(ServerConstants.getDataVersionLocation(volume), Integer.toString(ServerConstants.PREV_DATA_VERSION)));
+ for (Volume volume : fs.getVolumes()) {
+ try {
+ if (getAccumuloPersistentVersion(fs) == ServerConstants.PREV_DATA_VERSION) {
+ log.debug("Attempting to upgrade " + volume);
+ Path dataVersionLocation = ServerConstants.getDataVersionLocation(volume);
+ fs.create(new Path(dataVersionLocation, Integer.toString(ServerConstants.DATA_VERSION))).close();
+
+ Path prevDataVersionLoc = new Path(dataVersionLocation, Integer.toString(ServerConstants.PREV_DATA_VERSION));
+ if (!fs.delete(prevDataVersionLoc)) {
+ throw new RuntimeException("Could not delete previous data version location (" + prevDataVersionLoc + ") for " + volume);
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
}
- } catch (IOException e) {
- throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
}
}
[17/25] git commit: ACCUMULO-2061 Fixing whitespace nits
Posted by el...@apache.org.
ACCUMULO-2061 Fixing whitespace nits
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/03baf916
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/03baf916
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/03baf916
Branch: refs/heads/master
Commit: 03baf91611120dfdd36eaa5a824b99144b9eb36a
Parents: 53b9f25
Author: Josh Elser <el...@apache.org>
Authored: Thu Mar 20 13:23:04 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:59:30 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/volume/NonConfiguredVolume.java | 2 +-
.../org/apache/accumulo/core/volume/VolumeConfiguration.java | 4 ++--
.../main/java/org/apache/accumulo/core/volume/VolumeImpl.java | 2 +-
.../org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java | 3 ---
4 files changed, 4 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/03baf916/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
index 3cfd1c2..3d56fa9 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.fs.Path;
/**
* Volume implementation which represents a Volume for which we have a FileSystem but no base path because it is not configured via
* {@link Property#INSTANCE_VOLUMES}
- *
+ *
* This is useful to handle volumes that have been removed from accumulo-site.xml but references to these volumes have not been updated. This Volume should
* never be used to create new files, only to read existing files.
*/
http://git-wip-us.apache.org/repos/asf/accumulo/blob/03baf916/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
index 71ad611..e7a51d7 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
@@ -85,7 +85,7 @@ public class VolumeConfiguration {
/**
* Compute the URIs to be used by Accumulo
- *
+ *
* @param conf
* @return
*/
@@ -134,7 +134,7 @@ public class VolumeConfiguration {
/**
* Create a Volume with the given FileSystem that writes to the default path
- *
+ *
* @param fs
* A FileSystem to write to
* @return A Volume instance writing to the given FileSystem in the default path
http://git-wip-us.apache.org/repos/asf/accumulo/blob/03baf916/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
index f902c35..43ab96b 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
/**
- * Basic Volume implementation that contains a FileSystem and a base path
+ * Basic Volume implementation that contains a FileSystem and a base path
* that should be used within that filesystem.
*/
public class VolumeImpl implements Volume {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/03baf916/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java b/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
index 937baf8..8d4a800 100644
--- a/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
@@ -25,9 +25,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-/**
- *
- */
public class NonConfiguredVolumeTest {
private NonConfiguredVolume volume;
[19/25] git commit: ACCUMULO-2061 A few more minor fixes from
reviewboard.
Posted by el...@apache.org.
ACCUMULO-2061 A few more minor fixes from reviewboard.
Fixed a PrintInfo comment. Expand the VolumeImpl.isValidPath javadoc
and implementation to be more encompassing. Suppress warnings in VolumeIT.
Remove implementation of isValidPath from NonConfiguredVolume as it
could be misleading.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9172f52b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9172f52b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9172f52b
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 9172f52bc74de2590603c5282c19264447e33da6
Parents: 42f6b58
Author: Josh Elser <el...@apache.org>
Authored: Thu Mar 20 11:22:38 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:59:30 2014 -0400
----------------------------------------------------------------------
.../accumulo/core/file/rfile/PrintInfo.java | 7 +++--
.../core/volume/NonConfiguredVolume.java | 30 ++++++--------------
.../org/apache/accumulo/core/volume/Volume.java | 4 +--
.../apache/accumulo/core/volume/VolumeImpl.java | 16 ++++++++++-
.../accumulo/server/fs/VolumeManagerImpl.java | 8 +++---
.../java/org/apache/accumulo/test/VolumeIT.java | 1 +
6 files changed, 36 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9172f52b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 7ed8f34..dc54b49 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -52,8 +52,11 @@ public class PrintInfo {
@SuppressWarnings("deprecation")
AccumuloConfiguration aconf = AccumuloConfiguration.getSiteConfiguration();
- // TODO ACCUMULO-2462 This will only work for RFiles in HDFS when the filesystem is defined in the core-site.xml
- // on the classpath if a path, and not a URI, is given
+ // TODO ACCUMULO-2462 This will only work for RFiles (path only, not URI) in HDFS when the correct filesystem for the given file
+ // is on Property.INSTANCE_DFS_DIR or, when INSTANCE_DFS_DIR is not defined, is on the default filesystem
+ // defined in the Hadoop's core-site.xml
+ //
+ // A workaround is to always provide a URI to this class
FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem();
FileSystem localFs = FileSystem.getLocal(conf);
Opts opts = new Opts();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9172f52b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
index 7dcbd88..3cfd1c2 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
@@ -16,25 +16,18 @@
*/
package org.apache.accumulo.core.volume;
-import java.io.IOException;
-
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.log4j.Logger;
/**
- * Volume implementation which represents a Volume for which we have a FileSystem
- * but no base path because it is not configured via {@link Property#INSTANCE_VOLUMES}
+ * Volume implementation which represents a Volume for which we have a FileSystem but no base path because it is not configured via
+ * {@link Property#INSTANCE_VOLUMES}
*
- * This is useful to handle volumes that have been removed from accumulo-site.xml but references
- * to these volumes have not been updated. This Volume should never be used to create new files,
- * only to read existing files.
+ * This is useful to handle volumes that have been removed from accumulo-site.xml but references to these volumes have not been updated. This Volume should
+ * never be used to create new files, only to read existing files.
*/
public class NonConfiguredVolume implements Volume {
- private static final Logger log = Logger.getLogger(NonConfiguredVolume.class);
-
private FileSystem fs;
public NonConfiguredVolume(FileSystem fs) {
@@ -48,27 +41,22 @@ public class NonConfiguredVolume implements Volume {
@Override
public String getBasePath() {
- throw new UnsupportedOperationException("No base path known because this volume isn't configured in accumulo-site.xml");
+ throw new UnsupportedOperationException("No base path known because this Volume isn't configured in accumulo-site.xml");
}
@Override
public Path prefixChild(Path p) {
- throw new UnsupportedOperationException("Cannot prefix path because this volume isn't configured in accumulo-site.xml");
+ throw new UnsupportedOperationException("Cannot prefix path because this Volume isn't configured in accumulo-site.xml");
}
@Override
public Path prefixChild(String p) {
- throw new UnsupportedOperationException("Cannot prefix path because this volume isn't configured in accumulo-site.xml");
+ throw new UnsupportedOperationException("Cannot prefix path because this Volume isn't configured in accumulo-site.xml");
}
@Override
public boolean isValidPath(Path p) {
- try {
- return fs.equals(p.getFileSystem(CachedConfiguration.getInstance()));
- } catch (IOException e) {
- log.debug("Cannot determine FileSystem from path: " + p, e);
- }
- return false;
+ throw new UnsupportedOperationException("Cannot determine if path is valid because this Volume isn't configured in accumulo-site.xml");
}
@Override
@@ -89,4 +77,4 @@ public class NonConfiguredVolume implements Volume {
public int hashCode() {
return NonConfiguredVolume.class.hashCode() ^ this.fs.hashCode();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9172f52b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
index 17b2bf3..58b0ada 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
@@ -50,8 +50,8 @@ public interface Volume {
public Path prefixChild(String p);
/**
- * Determine if the Path is valid on this Volume (contained by the basePath)
- * @return True if path is contained within the basePath, false otherwise
+ * Determine if the Path is valid on this Volume. A Path is valid if it is contained
+ * in the Volume's FileSystem and is rooted beneath the basePath
*/
public boolean isValidPath(Path p);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9172f52b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
index 55ccfbc..f902c35 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
@@ -20,15 +20,21 @@ import static com.google.common.base.Preconditions.checkNotNull;
import java.io.IOException;
+import jline.internal.Log;
+
+import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
/**
* Basic Volume implementation that contains a FileSystem and a base path
* that should be used within that filesystem.
*/
public class VolumeImpl implements Volume {
+ private static final Logger log = Logger.getLogger(VolumeImpl.class);
+
protected final FileSystem fs;
protected final String basePath;
@@ -67,7 +73,15 @@ public class VolumeImpl implements Volume {
public boolean isValidPath(Path p) {
checkNotNull(p);
- return p.toUri().getPath().startsWith(basePath);
+ try {
+ if (fs.equals(p.getFileSystem(CachedConfiguration.getInstance()))) {
+ return p.toUri().getPath().startsWith(basePath);
+ }
+ } catch (IOException e) {
+ log.warn("Could not determine filesystem from path: " + p);
+ }
+
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9172f52b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index 64a6390..6a3140b 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -314,12 +314,12 @@ public class VolumeManagerImpl implements VolumeManager {
// we could also not find a matching one. We should still provide a Volume with the
// correct FileSystem even though we don't know what the proper base dir is
// e.g. Files on volumes that are now removed
- log.debug("Found no configured Volume for the given path: " + path);
-
- return new NonConfiguredVolume(desiredFs);
+ log.debug("Found candidate Volumes for Path but none of the Paths are valid on the candidates: " + path);
}
- log.debug("Could not determine volume for Path '" + path + "' from defined volumes");
+ log.debug("Could not determine volume for Path: " + path);
+
+ return new NonConfiguredVolume(desiredFs);
} catch (IOException ex) {
throw new RuntimeException(ex);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9172f52b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index 6c1da2c..0a7ef3f 100644
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@ -102,6 +102,7 @@ public class VolumeIT extends ConfigurableMacIT {
FileUtils.deleteQuietly(new File(v1.getParent().toUri()));
}
+ @SuppressWarnings("deprecation")
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
// Run MAC on two locations in the local file system
[25/25] git commit: Merge branch '1.6.0-SNAPSHOT'
Posted by el...@apache.org.
Merge branch '1.6.0-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2dbc14fa
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2dbc14fa
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2dbc14fa
Branch: refs/heads/master
Commit: 2dbc14fa9c2da05e97e270af9dfa1b3eaf3a6a50
Parents: 866422d cb2f4b5
Author: Josh Elser <el...@apache.org>
Authored: Thu Mar 20 22:12:39 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 22:12:39 2014 -0400
----------------------------------------------------------------------
.../core/client/admin/TableOperationsImpl.java | 4 +-
.../core/client/impl/OfflineScanner.java | 4 +-
.../org/apache/accumulo/core/conf/Property.java | 2 +
.../accumulo/core/file/VolumeConfiguration.java | 112 ------------
.../accumulo/core/file/rfile/PrintInfo.java | 16 +-
.../core/file/rfile/bcfile/PrintInfo.java | 4 +-
.../apache/accumulo/core/util/shell/Shell.java | 4 +-
.../core/volume/NonConfiguredVolume.java | 80 +++++++++
.../org/apache/accumulo/core/volume/Volume.java | 57 ++++++
.../core/volume/VolumeConfiguration.java | 161 +++++++++++++++++
.../apache/accumulo/core/volume/VolumeImpl.java | 107 ++++++++++++
.../apache/accumulo/core/zookeeper/ZooUtil.java | 4 +-
.../core/volume/NonConfiguredVolumeTest.java | 68 ++++++++
.../org/apache/accumulo/server/Accumulo.java | 34 +++-
.../apache/accumulo/server/ServerConstants.java | 50 +++---
.../accumulo/server/client/BulkImporter.java | 4 +-
.../accumulo/server/client/HdfsZooInstance.java | 18 +-
.../server/conf/NamespaceConfiguration.java | 24 ++-
.../accumulo/server/conf/ZooConfiguration.java | 16 +-
.../accumulo/server/fs/VolumeManager.java | 19 +-
.../accumulo/server/fs/VolumeManagerImpl.java | 173 +++++++++++++------
.../apache/accumulo/server/fs/VolumeUtil.java | 25 ++-
.../apache/accumulo/server/init/Initialize.java | 18 +-
.../server/master/recovery/HadoopLogCloser.java | 2 +-
.../server/master/recovery/MapRLogCloser.java | 2 +-
.../accumulo/server/util/ChangeSecret.java | 25 ++-
.../apache/accumulo/server/util/FileUtil.java | 12 +-
.../accumulo/server/util/LocalityCheck.java | 2 +-
.../accumulo/server/util/TabletOperations.java | 4 +-
.../accumulo/server/util/ZooKeeperMain.java | 2 +-
.../accumulo/server/ServerConstantsTest.java | 12 +-
.../apache/accumulo/server/fs/FileTypeTest.java | 48 +++--
.../accumulo/server/fs/VolumeUtilTest.java | 89 +++++++++-
.../accumulo/gc/GarbageCollectionTest.java | 65 +++++++
.../accumulo/master/tableOps/ExportTable.java | 2 +-
.../accumulo/master/tableOps/ImportTable.java | 2 +-
.../monitor/servlets/DefaultServlet.java | 6 +-
.../tserver/BulkFailedCopyProcessor.java | 27 +--
.../org/apache/accumulo/tserver/Compactor.java | 4 +-
.../apache/accumulo/tserver/FileManager.java | 2 +-
.../org/apache/accumulo/tserver/Tablet.java | 4 +-
.../apache/accumulo/tserver/TabletServer.java | 2 +-
.../compaction/MajorCompactionRequest.java | 2 +-
.../apache/accumulo/tserver/log/LogSorter.java | 2 +-
.../accumulo/tserver/log/MultiReader.java | 2 +-
.../apache/accumulo/tserver/RootFilesTest.java | 1 +
.../tserver/TabletServerSyncCheckTest.java | 14 +-
.../accumulo/tserver/log/MultiReaderTest.java | 8 +-
.../tserver/log/SortedLogRecoveryTest.java | 9 +-
.../tserver/log/TestUpgradePathForWALogs.java | 6 +-
.../performance/scan/CollectTabletStats.java | 6 +-
.../test/TableConfigurationUpdateIT.java | 34 +---
.../java/org/apache/accumulo/test/VolumeIT.java | 105 +++++++----
.../accumulo/test/functional/BulkFileIT.java | 4 +-
54 files changed, 1113 insertions(+), 396 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2dbc14fa/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2dbc14fa/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2dbc14fa/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2dbc14fa/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2dbc14fa/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2dbc14fa/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2dbc14fa/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2dbc14fa/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2dbc14fa/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
----------------------------------------------------------------------
[23/25] git commit: ACCUMULO-2515 ACCUMULO-2489 Apply same fixes from
TableConfiguration to NamespaceConfiguration
Posted by el...@apache.org.
ACCUMULO-2515 ACCUMULO-2489 Apply same fixes from TableConfiguration to
NamespaceConfiguration
Also switch over TableConfigurationUpdateIT to use SimpleMacIT instead
of doing it itself.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cb2f4b58
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cb2f4b58
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cb2f4b58
Branch: refs/heads/master
Commit: cb2f4b58041a4176d907e77a0853e2c3316df4ec
Parents: c25a41f
Author: Josh Elser <el...@apache.org>
Authored: Thu Mar 20 21:14:35 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 21:14:35 2014 -0400
----------------------------------------------------------------------
.../server/conf/NamespaceConfiguration.java | 24 ++++++++++----
.../test/TableConfigurationUpdateIT.java | 34 ++++----------------
2 files changed, 24 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cb2f4b58/server/base/src/main/java/org/apache/accumulo/server/conf/NamespaceConfiguration.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/NamespaceConfiguration.java b/server/base/src/main/java/org/apache/accumulo/server/conf/NamespaceConfiguration.java
index d08d45f..99532ca 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/NamespaceConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/NamespaceConfiguration.java
@@ -38,13 +38,18 @@ public class NamespaceConfiguration extends AccumuloConfiguration {
private static final Logger log = Logger.getLogger(NamespaceConfiguration.class);
private final AccumuloConfiguration parent;
- private static ZooCache propCache = null;
+ private static volatile ZooCache propCache = null;
+ private static final Object lock = new Object();
protected String namespaceId = null;
protected Instance inst = null;
private Set<ConfigurationObserver> observers;
public NamespaceConfiguration(String namespaceId, AccumuloConfiguration parent) {
- inst = HdfsZooInstance.getInstance();
+ this(namespaceId, HdfsZooInstance.getInstance(), parent);
+ }
+
+ public NamespaceConfiguration(String namespaceId, Instance inst, AccumuloConfiguration parent) {
+ this.inst = inst;
this.parent = parent;
this.namespaceId = namespaceId;
this.observers = Collections.synchronizedSet(new HashSet<ConfigurationObserver>());
@@ -75,10 +80,17 @@ public class NamespaceConfiguration extends AccumuloConfiguration {
return value;
}
- private synchronized static ZooCache getPropCache() {
- Instance inst = HdfsZooInstance.getInstance();
- if (propCache == null)
- propCache = new ZooCache(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), new NamespaceConfWatcher(inst));
+ private void initializePropCache() {
+ synchronized (lock) {
+ if (propCache == null)
+ propCache = new ZooCache(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), new NamespaceConfWatcher(inst));
+ }
+ }
+
+ private ZooCache getPropCache() {
+ if (null == propCache) {
+ initializePropCache();
+ }
return propCache;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cb2f4b58/test/src/test/java/org/apache/accumulo/test/TableConfigurationUpdateIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/TableConfigurationUpdateIT.java b/test/src/test/java/org/apache/accumulo/test/TableConfigurationUpdateIT.java
index c3e3342..5e2b2c9 100644
--- a/test/src/test/java/org/apache/accumulo/test/TableConfigurationUpdateIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/TableConfigurationUpdateIT.java
@@ -27,50 +27,28 @@ import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.client.impl.Namespaces;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.accumulo.server.conf.NamespaceConfiguration;
import org.apache.accumulo.server.conf.TableConfiguration;
-import org.apache.accumulo.server.conf.TableParentConfiguration;
+import org.apache.accumulo.test.functional.SimpleMacIT;
import org.apache.log4j.Logger;
-import org.junit.After;
import org.junit.Assert;
-import org.junit.Before;
import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-public class TableConfigurationUpdateIT {
+public class TableConfigurationUpdateIT extends SimpleMacIT {
private static final Logger log = Logger.getLogger(TableConfigurationUpdateIT.class);
- public static TemporaryFolder folder = new TemporaryFolder();
- private MiniAccumuloCluster accumulo;
- private String secret = "secret";
-
- @Before
- public void setUp() throws Exception {
- folder.create();
- accumulo = new MiniAccumuloCluster(folder.getRoot(), secret);
- accumulo.start();
- }
-
- @After
- public void tearDown() throws Exception {
- accumulo.stop();
- folder.delete();
- }
-
@Test
public void test() throws Exception {
- Instance inst = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
- Connector conn = inst.getConnector("root", new PasswordToken(secret));
+ Connector conn = getConnector();
+ Instance inst = conn.getInstance();
String table = "foo";
conn.tableOperations().create(table);
- final NamespaceConfiguration defaultConf = new TableParentConfiguration(conn.tableOperations().tableIdMap().get(table), AccumuloConfiguration.getDefaultConfiguration());
+ final NamespaceConfiguration defaultConf = new NamespaceConfiguration(Namespaces.DEFAULT_NAMESPACE_ID, inst, AccumuloConfiguration.getDefaultConfiguration());
// Cache invalidates 25% of the time
int randomMax = 4;
[24/25] git commit: ACCUMULO-2515 ACCUMULO-2489 Apply same fixes from
TableConfiguration to NamespaceConfiguration
Posted by el...@apache.org.
ACCUMULO-2515 ACCUMULO-2489 Apply same fixes from TableConfiguration to
NamespaceConfiguration
Also switch over TableConfigurationUpdateIT to use SimpleMacIT instead
of doing it itself.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cb2f4b58
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cb2f4b58
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cb2f4b58
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: cb2f4b58041a4176d907e77a0853e2c3316df4ec
Parents: c25a41f
Author: Josh Elser <el...@apache.org>
Authored: Thu Mar 20 21:14:35 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 21:14:35 2014 -0400
----------------------------------------------------------------------
.../server/conf/NamespaceConfiguration.java | 24 ++++++++++----
.../test/TableConfigurationUpdateIT.java | 34 ++++----------------
2 files changed, 24 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cb2f4b58/server/base/src/main/java/org/apache/accumulo/server/conf/NamespaceConfiguration.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/NamespaceConfiguration.java b/server/base/src/main/java/org/apache/accumulo/server/conf/NamespaceConfiguration.java
index d08d45f..99532ca 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/NamespaceConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/NamespaceConfiguration.java
@@ -38,13 +38,18 @@ public class NamespaceConfiguration extends AccumuloConfiguration {
private static final Logger log = Logger.getLogger(NamespaceConfiguration.class);
private final AccumuloConfiguration parent;
- private static ZooCache propCache = null;
+ private static volatile ZooCache propCache = null;
+ private static final Object lock = new Object();
protected String namespaceId = null;
protected Instance inst = null;
private Set<ConfigurationObserver> observers;
public NamespaceConfiguration(String namespaceId, AccumuloConfiguration parent) {
- inst = HdfsZooInstance.getInstance();
+ this(namespaceId, HdfsZooInstance.getInstance(), parent);
+ }
+
+ public NamespaceConfiguration(String namespaceId, Instance inst, AccumuloConfiguration parent) {
+ this.inst = inst;
this.parent = parent;
this.namespaceId = namespaceId;
this.observers = Collections.synchronizedSet(new HashSet<ConfigurationObserver>());
@@ -75,10 +80,17 @@ public class NamespaceConfiguration extends AccumuloConfiguration {
return value;
}
- private synchronized static ZooCache getPropCache() {
- Instance inst = HdfsZooInstance.getInstance();
- if (propCache == null)
- propCache = new ZooCache(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), new NamespaceConfWatcher(inst));
+ private void initializePropCache() {
+ synchronized (lock) {
+ if (propCache == null)
+ propCache = new ZooCache(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), new NamespaceConfWatcher(inst));
+ }
+ }
+
+ private ZooCache getPropCache() {
+ if (null == propCache) {
+ initializePropCache();
+ }
return propCache;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cb2f4b58/test/src/test/java/org/apache/accumulo/test/TableConfigurationUpdateIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/TableConfigurationUpdateIT.java b/test/src/test/java/org/apache/accumulo/test/TableConfigurationUpdateIT.java
index c3e3342..5e2b2c9 100644
--- a/test/src/test/java/org/apache/accumulo/test/TableConfigurationUpdateIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/TableConfigurationUpdateIT.java
@@ -27,50 +27,28 @@ import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.client.impl.Namespaces;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.accumulo.server.conf.NamespaceConfiguration;
import org.apache.accumulo.server.conf.TableConfiguration;
-import org.apache.accumulo.server.conf.TableParentConfiguration;
+import org.apache.accumulo.test.functional.SimpleMacIT;
import org.apache.log4j.Logger;
-import org.junit.After;
import org.junit.Assert;
-import org.junit.Before;
import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-public class TableConfigurationUpdateIT {
+public class TableConfigurationUpdateIT extends SimpleMacIT {
private static final Logger log = Logger.getLogger(TableConfigurationUpdateIT.class);
- public static TemporaryFolder folder = new TemporaryFolder();
- private MiniAccumuloCluster accumulo;
- private String secret = "secret";
-
- @Before
- public void setUp() throws Exception {
- folder.create();
- accumulo = new MiniAccumuloCluster(folder.getRoot(), secret);
- accumulo.start();
- }
-
- @After
- public void tearDown() throws Exception {
- accumulo.stop();
- folder.delete();
- }
-
@Test
public void test() throws Exception {
- Instance inst = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
- Connector conn = inst.getConnector("root", new PasswordToken(secret));
+ Connector conn = getConnector();
+ Instance inst = conn.getInstance();
String table = "foo";
conn.tableOperations().create(table);
- final NamespaceConfiguration defaultConf = new TableParentConfiguration(conn.tableOperations().tableIdMap().get(table), AccumuloConfiguration.getDefaultConfiguration());
+ final NamespaceConfiguration defaultConf = new NamespaceConfiguration(Namespaces.DEFAULT_NAMESPACE_ID, inst, AccumuloConfiguration.getDefaultConfiguration());
// Cache invalidates 25% of the time
int randomMax = 4;
[11/25] git commit: ACCUMULO-2061 Propagate errors on delete or
mkdirs failure.
Posted by el...@apache.org.
ACCUMULO-2061 Propagate errors on delete or mkdirs failure.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7d48b1ad
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7d48b1ad
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7d48b1ad
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 7d48b1adec6b124054f64ca0c87913ba5a4b6b7e
Parents: 550e5e6
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 16:19:19 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:59:19 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/server/util/ChangeSecret.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7d48b1ad/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index f0dcd14..2926a3f 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@ -147,8 +147,14 @@ public class ChangeSecret {
// Need to recreate the instanceId on all of them to keep consistency
for (Volume v : fs.getVolumes()) {
final Path instanceId = ServerConstants.getInstanceIdLocation(v);
- v.getFileSystem().delete(instanceId, true);
- v.getFileSystem().mkdirs(instanceId);
+ if (!v.getFileSystem().delete(instanceId, true)) {
+ throw new IOException("Could not recursively delete " + instanceId);
+ }
+
+ if (!v.getFileSystem().mkdirs(instanceId)) {
+ throw new IOException("Could not create directory " + instanceId);
+ }
+
v.getFileSystem().create(new Path(instanceId, newInstanceId)).close();
}
}
[12/25] git commit: ACCUMULO-2061 Only append a trailing slash when
one doesn't exist on the base dir
Posted by el...@apache.org.
ACCUMULO-2061 Only append a trailing slash when one doesn't exist on the base dir
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/492768d3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/492768d3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/492768d3
Branch: refs/heads/master
Commit: 492768d38221af64b71ba37bb8979d7bb841e738
Parents: 7d48b1a
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 16:25:19 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:59:19 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/core/file/rfile/PrintInfo.java | 2 +-
.../org/apache/accumulo/core/volume/VolumeConfiguration.java | 6 +++++-
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/492768d3/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 7c0f067..7ed8f34 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -52,7 +52,7 @@ public class PrintInfo {
@SuppressWarnings("deprecation")
AccumuloConfiguration aconf = AccumuloConfiguration.getSiteConfiguration();
- // TODO This will only work for RFiles in HDFS when the filesystem is defined in the core-site.xml
+ // TODO ACCUMULO-2462 This will only work for RFiles in HDFS when the filesystem is defined in the core-site.xml
// on the classpath if a path, and not a URI, is given
FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem();
FileSystem localFs = FileSystem.getLocal(conf);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/492768d3/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
index 5db5bb2..71ad611 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
@@ -123,7 +123,11 @@ public class VolumeConfiguration {
suffix = suffix.substring(1);
String result[] = new String[bases.length];
for (int i = 0; i < bases.length; i++) {
- result[i] = bases[i] + "/" + suffix;
+ if (bases[i].endsWith("/")) {
+ result[i] = bases[i] + suffix;
+ } else {
+ result[i] = bases[i] + "/" + suffix;
+ }
}
return result;
}
[13/25] git commit: ACCUMULO-2061 Add a utility method to pull a Path
with the instance_id
Posted by el...@apache.org.
ACCUMULO-2061 Add a utility method to pull a Path with the instance_id
Consolidates a little bit of code in one place.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c25a41fe
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c25a41fe
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c25a41fe
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: c25a41fe905cda4aa914f232db5cc26b3254f4a0
Parents: 03baf91
Author: Josh Elser <el...@apache.org>
Authored: Thu Mar 20 17:39:13 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:59:30 2014 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/server/Accumulo.java | 7 +++++++
.../java/org/apache/accumulo/server/ServerConstants.java | 2 +-
.../apache/accumulo/server/client/HdfsZooInstance.java | 10 +++++-----
.../org/apache/accumulo/server/conf/ZooConfiguration.java | 10 ++++------
4 files changed, 17 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c25a41fe/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index f7f2298..925c0d0 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.util.Version;
import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
@@ -91,6 +92,12 @@ public class Accumulo {
return getAccumuloPersistentVersion(v.getFileSystem(), path);
}
+ public static synchronized Path getAccumuloInstanceIdPath(VolumeManager fs) {
+ // It doesn't matter which Volume is used as they should all have the instance ID stored
+ Volume v = fs.getVolumes().iterator().next();
+ return ServerConstants.getInstanceIdLocation(v);
+ }
+
public static void enableTracing(String address, String application) {
try {
DistributedTrace.enable(HdfsZooInstance.getInstance(), ZooReaderWriter.getInstance(), application, address);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c25a41fe/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index 7dd0a08..b577abb 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -76,7 +76,7 @@ public class ServerConstants {
String currentIid;
Integer currentVersion;
try {
- currentIid = ZooUtil.getInstanceIDFromHdfs(new Path(baseDir, INSTANCE_ID_DIR), ServerConfiguration.getSiteConfiguration());
+ currentIid = ZooUtil.getInstanceIDFromHdfs(path, ServerConfiguration.getSiteConfiguration());
Path vpath = new Path(baseDir, VERSION_DIR);
currentVersion = Accumulo.getAccumuloPersistentVersion(vpath.getFileSystem(CachedConfiguration.getInstance()), vpath);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c25a41fe/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index ee928f3..620188c 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@ -39,14 +39,14 @@ import org.apache.accumulo.core.util.ByteBufferUtil;
import org.apache.accumulo.core.util.OpTimer;
import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.core.util.TextUtil;
-import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
-import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.Accumulo;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -134,9 +134,9 @@ public class HdfsZooInstance implements Instance {
} catch (IOException e) {
throw new RuntimeException(e);
}
- Volume randVolume = fs.getVolumes().iterator().next();
- log.trace("Looking for instanceId from " + randVolume);
- String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), acuConf);
+ Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
+ log.trace("Looking for instanceId from " + instanceIdPath);
+ String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, acuConf);
instanceId = instanceIdFromFile;
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c25a41fe/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
index 94e468b..0c03aac 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
@@ -26,15 +26,13 @@ import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.accumulo.core.volume.Volume;
-import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
-import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.Accumulo;
import org.apache.accumulo.server.client.HdfsZooInstance.AccumuloNotInitializedException;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
public class ZooConfiguration extends AccumuloConfiguration {
@@ -70,8 +68,8 @@ public class ZooConfiguration extends AccumuloConfiguration {
} catch (IOException e) {
throw new RuntimeException(e);
}
- Volume randVolume = fs.getVolumes().iterator().next();
- String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), parent);
+ Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
+ String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, parent);
instanceId = deprecatedInstanceIdFromHdfs;
}
return instance;
[21/25] git commit: ACCUMULO-2061 Co-locate volumes with the MAC dir
Posted by el...@apache.org.
ACCUMULO-2061 Co-locate volumes with the MAC dir
Losing the volumes on test failure sucks because you can't validate
if we misplaced a file on the wrong volume in the first place.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/53b9f25d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/53b9f25d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/53b9f25d
Branch: refs/heads/master
Commit: 53b9f25db94d812f78fd468e88e05da33fdd5190
Parents: 9172f52
Author: Josh Elser <el...@apache.org>
Authored: Thu Mar 20 12:18:42 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:59:30 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/test/VolumeIT.java | 26 ++++++--------------
1 file changed, 7 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/53b9f25d/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index 0a7ef3f..c0e41c1 100644
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertTrue;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
-import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
@@ -65,7 +64,6 @@ import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.init.Initialize;
import org.apache.accumulo.server.util.Admin;
import org.apache.accumulo.test.functional.ConfigurableMacIT;
-import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -73,38 +71,28 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.Text;
import org.apache.zookeeper.ZooKeeper;
-import org.junit.After;
import org.junit.Assert;
-import org.junit.BeforeClass;
import org.junit.Test;
public class VolumeIT extends ConfigurableMacIT {
private static final Text EMPTY = new Text();
private static final Value EMPTY_VALUE = new Value(new byte[] {});
- public static File volDirBase;
- public static Path v1;
- public static Path v2;
+ private File volDirBase;
+ private Path v1, v2;
- @BeforeClass
- public static void createVolumeDirs() throws IOException {
- volDirBase = createSharedTestDir(VolumeIT.class.getName() + "-volumes");
+ @SuppressWarnings("deprecation")
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ File baseDir = cfg.getDir();
+ volDirBase = new File(baseDir, "volumes");
File v1f = new File(volDirBase, "v1");
File v2f = new File(volDirBase, "v2");
v1f.mkdir();
v2f.mkdir();
v1 = new Path("file://" + v1f.getAbsolutePath());
v2 = new Path("file://" + v2f.getAbsolutePath());
- }
- @After
- public void clearDirs() throws IOException {
- FileUtils.deleteQuietly(new File(v1.getParent().toUri()));
- }
-
- @SuppressWarnings("deprecation")
- @Override
- public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
// Run MAC on two locations in the local file system
URI v1Uri = v1.toUri();
cfg.setProperty(Property.INSTANCE_DFS_DIR, v1Uri.getPath());
[10/25] git commit: ACCUMULO-2061 Propagate errors on delete or
mkdirs failure.
Posted by el...@apache.org.
ACCUMULO-2061 Propagate errors on delete or mkdirs failure.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7d48b1ad
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7d48b1ad
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7d48b1ad
Branch: refs/heads/master
Commit: 7d48b1adec6b124054f64ca0c87913ba5a4b6b7e
Parents: 550e5e6
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 16:19:19 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:59:19 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/server/util/ChangeSecret.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7d48b1ad/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index f0dcd14..2926a3f 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@ -147,8 +147,14 @@ public class ChangeSecret {
// Need to recreate the instanceId on all of them to keep consistency
for (Volume v : fs.getVolumes()) {
final Path instanceId = ServerConstants.getInstanceIdLocation(v);
- v.getFileSystem().delete(instanceId, true);
- v.getFileSystem().mkdirs(instanceId);
+ if (!v.getFileSystem().delete(instanceId, true)) {
+ throw new IOException("Could not recursively delete " + instanceId);
+ }
+
+ if (!v.getFileSystem().mkdirs(instanceId)) {
+ throw new IOException("Could not create directory " + instanceId);
+ }
+
v.getFileSystem().create(new Path(instanceId, newInstanceId)).close();
}
}
[20/25] git commit: ACCUMULO-2061 Use URI instead of FileSystem as
the key to find correct Volumes and ensure that absolute URIs are still valid
even after they are not configured.
Posted by el...@apache.org.
ACCUMULO-2061 Use URI instead of FileSystem as the key to find correct Volumes and ensure that absolute URIs
are still valid even after they are not configured.
This will help ensure that FileSystem implementations' hashCode and equals don't
have the potential to collide but still provide unique access back to the Volumes
contained in the FileSystem. Added tests for the NonConfiguredVolume and also
for the no-longer-configured volumes.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/42f6b58e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/42f6b58e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/42f6b58e
Branch: refs/heads/master
Commit: 42f6b58e58a8fb1a8f05c371844f86c536fe143a
Parents: 492768d
Author: Josh Elser <el...@apache.org>
Authored: Fri Mar 14 17:06:32 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:59:30 2014 -0400
----------------------------------------------------------------------
.../core/volume/NonConfiguredVolume.java | 92 ++++++++++++++++++++
.../core/volume/NonConfiguredVolumeTest.java | 71 +++++++++++++++
.../accumulo/server/fs/VolumeManagerImpl.java | 25 +++---
.../java/org/apache/accumulo/test/VolumeIT.java | 55 +++++++++++-
4 files changed, 231 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/42f6b58e/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
new file mode 100644
index 0000000..7dcbd88
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.volume;
+
+import java.io.IOException;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Volume implementation which represents a Volume for which we have a FileSystem
+ * but no base path because it is not configured via {@link Property#INSTANCE_VOLUMES}
+ *
+ * This is useful to handle volumes that have been removed from accumulo-site.xml but references
+ * to these volumes have not been updated. This Volume should never be used to create new files,
+ * only to read existing files.
+ */
+public class NonConfiguredVolume implements Volume {
+ private static final Logger log = Logger.getLogger(NonConfiguredVolume.class);
+
+ private FileSystem fs;
+
+ public NonConfiguredVolume(FileSystem fs) {
+ this.fs = fs;
+ }
+
+ @Override
+ public FileSystem getFileSystem() {
+ return fs;
+ }
+
+ @Override
+ public String getBasePath() {
+ throw new UnsupportedOperationException("No base path known because this volume isn't configured in accumulo-site.xml");
+ }
+
+ @Override
+ public Path prefixChild(Path p) {
+ throw new UnsupportedOperationException("Cannot prefix path because this volume isn't configured in accumulo-site.xml");
+ }
+
+ @Override
+ public Path prefixChild(String p) {
+ throw new UnsupportedOperationException("Cannot prefix path because this volume isn't configured in accumulo-site.xml");
+ }
+
+ @Override
+ public boolean isValidPath(Path p) {
+ try {
+ return fs.equals(p.getFileSystem(CachedConfiguration.getInstance()));
+ } catch (IOException e) {
+ log.debug("Cannot determine FileSystem from path: " + p, e);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof NonConfiguredVolume) {
+ NonConfiguredVolume other = (NonConfiguredVolume) o;
+ return this.fs.equals(other.getFileSystem());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "NonConfiguredVolume: " + this.fs.toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return NonConfiguredVolume.class.hashCode() ^ this.fs.hashCode();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo/blob/42f6b58e/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java b/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
new file mode 100644
index 0000000..937baf8
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.volume;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class NonConfiguredVolumeTest {
+
+ private NonConfiguredVolume volume;
+
+ @Before
+ public void create() throws IOException {
+ volume = new NonConfiguredVolume(FileSystem.getLocal(new Configuration()));
+ }
+
+ @Test
+ public void testSameFileSystem() throws IOException {
+ Assert.assertEquals(FileSystem.getLocal(new Configuration()), volume.getFileSystem());
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testGetBasePathFails() {
+ volume.getBasePath();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testPrefixChildPath() {
+ volume.prefixChild(new Path("/foo"));
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testPrefixChildString() {
+ volume.prefixChild("/foo");
+ }
+
+ @Test
+ public void testEquality() throws IOException {
+ Volume newVolume = new NonConfiguredVolume(FileSystem.getLocal(new Configuration()));
+ Assert.assertEquals(volume, newVolume);
+ }
+
+ @Test
+ public void testHashCode() throws IOException {
+ Volume newVolume = new NonConfiguredVolume(FileSystem.getLocal(new Configuration()));
+ Assert.assertEquals(volume.hashCode(), newVolume.hashCode());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/42f6b58e/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index b860f53..64a6390 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -38,6 +38,7 @@ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.NonConfiguredVolume;
import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.server.client.HdfsZooInstance;
@@ -65,7 +66,7 @@ public class VolumeManagerImpl implements VolumeManager {
private static final Logger log = Logger.getLogger(VolumeManagerImpl.class);
Map<String,Volume> volumesByName;
- Multimap<FileSystem,Volume> volumesByFileSystem;
+ Multimap<URI,Volume> volumesByFileSystemUri;
Volume defaultVolume;
AccumuloConfiguration conf;
VolumeChooser chooser;
@@ -74,16 +75,16 @@ public class VolumeManagerImpl implements VolumeManager {
this.volumesByName = volumes;
this.defaultVolume = defaultVolume;
// We may have multiple directories used in a single FileSystem (e.g. testing)
- this.volumesByFileSystem = HashMultimap.create();
- invertVolumesByFileSystem(volumesByName, volumesByFileSystem);
+ this.volumesByFileSystemUri = HashMultimap.create();
+ invertVolumesByFileSystem(volumesByName, volumesByFileSystemUri);
this.conf = conf;
ensureSyncIsEnabled();
chooser = Property.createInstanceFromPropertyName(conf, Property.GENERAL_VOLUME_CHOOSER, VolumeChooser.class, new RandomVolumeChooser());
}
- private void invertVolumesByFileSystem(Map<String,Volume> forward, Multimap<FileSystem,Volume> inverted) {
+ private void invertVolumesByFileSystem(Map<String,Volume> forward, Multimap<URI,Volume> inverted) {
for (Volume volume : forward.values()) {
- inverted.put(volume.getFileSystem(), volume);
+ inverted.put(volume.getFileSystem().getUri(), volume);
}
}
@@ -299,8 +300,9 @@ public class VolumeManagerImpl implements VolumeManager {
public Volume getVolumeByPath(Path path) {
if (path.toString().contains(":")) {
try {
- FileSystem pathFs = path.getFileSystem(CachedConfiguration.getInstance());
- Collection<Volume> candidateVolumes = volumesByFileSystem.get(pathFs);
+ FileSystem desiredFs = path.getFileSystem(CachedConfiguration.getInstance());
+ URI desiredFsUri = desiredFs.getUri();
+ Collection<Volume> candidateVolumes = volumesByFileSystemUri.get(desiredFsUri);
if (null != candidateVolumes) {
for (Volume candidateVolume : candidateVolumes) {
if (candidateVolume.isValidPath(path)) {
@@ -309,11 +311,12 @@ public class VolumeManagerImpl implements VolumeManager {
}
// For the same reason as we can have multiple Volumes within a single filesystem
- // we could also not find a matching one. We should defer back to the defaultVolume
- // e.g. volume rename with old path references
- log.debug("Defaulting to " + defaultVolume + " as a valid volume could not be determined for " + path);
+ // we could also not find a matching one. We should still provide a Volume with the
+ // correct FileSystem even though we don't know what the proper base dir is
+ // e.g. Files on volumes that are now removed
+ log.debug("Found no configured Volume for the given path: " + path);
- return defaultVolume;
+ return new NonConfiguredVolume(desiredFs);
}
log.debug("Could not determine volume for Path '" + path + "' from defined volumes");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/42f6b58e/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index a7f7556..6c1da2c 100644
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@ -307,6 +307,59 @@ public class VolumeIT extends ConfigurableMacIT {
}
+ @Test
+ public void testNonConfiguredVolumes() throws Exception {
+
+ String[] tableNames = getTableNames(2);
+
+ // grab this before shutting down cluster
+ String uuid = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getInstanceID();
+
+ verifyVolumesUsed(tableNames[0], false, v1, v2);
+
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ cluster.stop();
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ File v3f = new File(volDirBase, "v3");
+ v3f.mkdir();
+ Path v3 = new Path("file://" + v3f.getAbsolutePath());
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v2.toString() + "," + v3.toString());
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // initialize volume
+ Assert.assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor());
+
+ // check that all volumes are initialized
+ for (Path volumePath : Arrays.asList(v1, v2, v3)) {
+ FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
+ Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
+ FileStatus[] iids = fs.listStatus(vp);
+ Assert.assertEquals(1, iids.length);
+ Assert.assertEquals(uuid, iids[0].getPath().getName());
+ }
+
+ // start cluster and verify that new volume is used
+ cluster.start();
+
+ // Make sure we can still read the tables (tableNames[0] is very likely to have a file still on v1)
+ List<String> expected = new ArrayList<String>();
+ for (int i = 0; i < 100; i++) {
+ String row = String.format("%06d", i * 100 + 3);
+ expected.add(row + ":cf1:cq1:1");
+ }
+
+ verifyData(expected, getConnector().createScanner(tableNames[0], Authorizations.EMPTY));
+
+ // v1 should not have any data for tableNames[1]
+ verifyVolumesUsed(tableNames[1], false, v2, v3);
+ }
+
private void writeData(String tableName, Connector conn) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
MutationsRejectedException {
TreeSet<Text> splits = new TreeSet<Text>();
@@ -331,7 +384,7 @@ public class VolumeIT extends ConfigurableMacIT {
private void verifyVolumesUsed(String tableName, boolean shouldExist, Path... paths) throws AccumuloException, AccumuloSecurityException,
TableExistsException, TableNotFoundException, MutationsRejectedException {
- Connector conn = cluster.getConnector("root", ROOT_PASSWORD);
+ Connector conn = getConnector();
List<String> expected = new ArrayList<String>();
for (int i = 0; i < 100; i++) {
[04/25] git commit: ACCUMULO-2061 Initial implementation to remove
instance.dfs.dir from usage of instance.volumes
Posted by el...@apache.org.
ACCUMULO-2061 Initial implementation to remove instance.dfs.dir from usage of instance.volumes
The value of instance.volumes, when provided, appended the value of
instance.dfs.dir to each configured volume which was unintuitive. It is more
straightforward to use each volume provided as-is instead of appending
on some more cruft.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7c94c086
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7c94c086
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7c94c086
Branch: refs/heads/master
Commit: 7c94c086de2945d698d0907c9dc5ec6959042286
Parents: ef5dc4a
Author: Josh Elser <el...@apache.org>
Authored: Thu Feb 27 00:51:29 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:55:47 2014 -0400
----------------------------------------------------------------------
.../core/client/admin/TableOperationsImpl.java | 4 +-
.../core/client/impl/OfflineScanner.java | 4 +-
.../org/apache/accumulo/core/conf/Property.java | 2 +
.../accumulo/core/file/VolumeConfiguration.java | 112 ------------
.../accumulo/core/file/rfile/PrintInfo.java | 4 +-
.../core/file/rfile/bcfile/PrintInfo.java | 4 +-
.../apache/accumulo/core/util/shell/Shell.java | 4 +-
.../org/apache/accumulo/core/volume/Volume.java | 60 +++++++
.../core/volume/VolumeConfiguration.java | 151 +++++++++++++++++
.../apache/accumulo/core/volume/VolumeImpl.java | 93 ++++++++++
.../apache/accumulo/core/zookeeper/ZooUtil.java | 4 +-
.../org/apache/accumulo/server/Accumulo.java | 13 +-
.../apache/accumulo/server/ServerConstants.java | 48 +++---
.../accumulo/server/client/BulkImporter.java | 4 +-
.../accumulo/server/client/HdfsZooInstance.java | 16 +-
.../accumulo/server/conf/ZooConfiguration.java | 16 +-
.../accumulo/server/fs/VolumeManager.java | 21 ++-
.../accumulo/server/fs/VolumeManagerImpl.java | 169 +++++++++++++------
.../apache/accumulo/server/fs/VolumeUtil.java | 27 ++-
.../apache/accumulo/server/init/Initialize.java | 18 +-
.../server/master/recovery/HadoopLogCloser.java | 2 +-
.../server/master/recovery/MapRLogCloser.java | 2 +-
.../accumulo/server/util/ChangeSecret.java | 18 +-
.../apache/accumulo/server/util/FileUtil.java | 12 +-
.../accumulo/server/util/LocalityCheck.java | 2 +-
.../accumulo/server/util/TabletOperations.java | 4 +-
.../accumulo/server/util/ZooKeeperMain.java | 2 +-
.../accumulo/server/ServerConstantsTest.java | 12 +-
.../apache/accumulo/server/fs/FileTypeTest.java | 48 ++++--
.../accumulo/server/fs/VolumeUtilTest.java | 89 +++++++++-
.../accumulo/gc/GarbageCollectionTest.java | 65 +++++++
.../accumulo/master/tableOps/ExportTable.java | 2 +-
.../accumulo/master/tableOps/ImportTable.java | 2 +-
.../monitor/servlets/DefaultServlet.java | 6 +-
.../tserver/BulkFailedCopyProcessor.java | 27 +--
.../org/apache/accumulo/tserver/Compactor.java | 4 +-
.../apache/accumulo/tserver/FileManager.java | 2 +-
.../org/apache/accumulo/tserver/Tablet.java | 4 +-
.../apache/accumulo/tserver/TabletServer.java | 2 +-
.../compaction/MajorCompactionRequest.java | 2 +-
.../apache/accumulo/tserver/log/LogSorter.java | 2 +-
.../accumulo/tserver/log/MultiReader.java | 2 +-
.../apache/accumulo/tserver/RootFilesTest.java | 1 +
.../tserver/TabletServerSyncCheckTest.java | 14 +-
.../accumulo/tserver/log/MultiReaderTest.java | 8 +-
.../tserver/log/SortedLogRecoveryTest.java | 9 +-
.../tserver/log/TestUpgradePathForWALogs.java | 6 +-
.../performance/scan/CollectTabletStats.java | 6 +-
.../java/org/apache/accumulo/test/VolumeIT.java | 25 +--
.../accumulo/test/functional/BulkFileIT.java | 4 +-
50 files changed, 822 insertions(+), 336 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
index 0b2f10e..9d033e2 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
@@ -82,7 +82,6 @@ import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
@@ -109,6 +108,7 @@ import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.core.util.TextUtil;
import org.apache.accumulo.core.util.ThriftUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.trace.instrument.Tracer;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -1139,7 +1139,7 @@ public class TableOperationsImpl extends TableOperationsHelper {
@SuppressWarnings("deprecation")
private Path checkPath(String dir, String kind, String type) throws IOException, AccumuloException {
Path ret;
- FileSystem fs = VolumeConfiguration.getFileSystem(dir, CachedConfiguration.getInstance(), ServerConfigurationUtil.getConfiguration(instance));
+ FileSystem fs = VolumeConfiguration.getVolume(dir, CachedConfiguration.getInstance(), ServerConfigurationUtil.getConfiguration(instance)).getFileSystem();
if (dir.contains(":")) {
ret = new Path(dir);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
index c90d380..20228d2 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
@@ -43,7 +43,6 @@ import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVIterator;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
@@ -65,6 +64,7 @@ import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.LocalityGroupUtil;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -306,7 +306,7 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
// TODO need to close files - ACCUMULO-1303
for (String file : absFiles) {
- FileSystem fs = VolumeConfiguration.getFileSystem(file, conf, config);
+ FileSystem fs = VolumeConfiguration.getVolume(file, conf, config).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().openReader(file, false, fs, conf, acuTableConf, null, null);
readers.add(reader);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index fc4d012..795a250 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -103,6 +103,7 @@ public enum Property {
INSTANCE_ZK_HOST("instance.zookeeper.host", "localhost:2181", PropertyType.HOSTLIST, "Comma separated list of zookeeper servers"),
INSTANCE_ZK_TIMEOUT("instance.zookeeper.timeout", "30s", PropertyType.TIMEDURATION,
"Zookeeper session timeout; max value when represented as milliseconds should be no larger than " + Integer.MAX_VALUE),
+ @Deprecated
INSTANCE_DFS_URI(
"instance.dfs.uri",
"",
@@ -111,6 +112,7 @@ public enum Property {
+ "will only be used when creating new files if instance.volumes is empty. After an upgrade to 1.6.0 Accumulo will start using absolute paths to "
+ "reference files. Files created before a 1.6.0 upgrade are referenced via relative paths. Relative paths will always be resolved using this config "
+ "(if empty using the hadoop config)."),
+ @Deprecated
INSTANCE_DFS_DIR("instance.dfs.dir", "/accumulo", PropertyType.ABSOLUTEPATH,
"HDFS directory in which accumulo instance will run. Do not change after accumulo is initialized."),
@Sensitive
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/file/VolumeConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/file/VolumeConfiguration.java
deleted file mode 100644
index fb8c6c8..0000000
--- a/core/src/main/java/org/apache/accumulo/core/file/VolumeConfiguration.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.core.file;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-public class VolumeConfiguration {
-
- public static FileSystem getFileSystem(String path, Configuration conf, AccumuloConfiguration acuconf) throws IOException {
- if (path.contains(":"))
- return new Path(path).getFileSystem(conf);
- else
- return getDefaultFilesystem(conf, acuconf);
- }
-
- public static FileSystem getDefaultFilesystem(Configuration conf, AccumuloConfiguration acuconf) throws IOException {
- String uri = acuconf.get(Property.INSTANCE_DFS_URI);
- if ("".equals(uri))
- return FileSystem.get(conf);
- else
- try {
- return FileSystem.get(new URI(uri), conf);
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- public static String getConfiguredBaseDir(AccumuloConfiguration conf) {
- String singleNamespace = conf.get(Property.INSTANCE_DFS_DIR);
- String dfsUri = conf.get(Property.INSTANCE_DFS_URI);
- String baseDir;
-
- if (dfsUri == null || dfsUri.isEmpty()) {
- Configuration hadoopConfig = CachedConfiguration.getInstance();
- try {
- baseDir = FileSystem.get(hadoopConfig).getUri().toString() + singleNamespace;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- } else {
- if (!dfsUri.contains(":"))
- throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_DFS_URI.getKey() + " got " + dfsUri);
- baseDir = dfsUri + singleNamespace;
- }
- return baseDir;
- }
-
- public static String[] getConfiguredBaseDirs(AccumuloConfiguration conf) {
- String singleNamespace = conf.get(Property.INSTANCE_DFS_DIR);
- String ns = conf.get(Property.INSTANCE_VOLUMES);
-
- String configuredBaseDirs[];
-
- if (ns == null || ns.isEmpty()) {
- configuredBaseDirs = new String[] {getConfiguredBaseDir(conf)};
- } else {
- String namespaces[] = ns.split(",");
- String unescapedNamespaces[] = new String[namespaces.length];
- int i = 0;
- for (String namespace : namespaces) {
- if (!namespace.contains(":")) {
- throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_VOLUMES.getKey() + " got " + namespace);
- }
-
- try {
- // pass through URI to unescape hex encoded chars (e.g. convert %2C to "," char)
- unescapedNamespaces[i++] = new Path(new URI(namespace)).toString();
- } catch (URISyntaxException e) {
- throw new IllegalArgumentException(Property.INSTANCE_VOLUMES.getKey() + " contains " + namespace + " which has a syntax error", e);
- }
- }
-
- configuredBaseDirs = prefix(unescapedNamespaces, singleNamespace);
- }
-
- return configuredBaseDirs;
- }
-
- public static String[] prefix(String bases[], String suffix) {
- if (suffix.startsWith("/"))
- suffix = suffix.substring(1);
- String result[] = new String[bases.length];
- for (int i = 0; i < bases.length; i++) {
- result[i] = bases[i] + "/" + suffix;
- }
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 4cfefad..4e39fc7 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -25,9 +25,9 @@ import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
import org.apache.accumulo.core.file.rfile.RFile.Reader;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -50,7 +50,7 @@ public class PrintInfo {
@SuppressWarnings("deprecation")
AccumuloConfiguration aconf = AccumuloConfiguration.getSiteConfiguration();
- FileSystem hadoopFs = VolumeConfiguration.getDefaultFilesystem(conf, aconf);
+ FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem();
FileSystem localFs = FileSystem.getLocal(conf);
Opts opts = new Opts();
opts.parseArgs(PrintInfo.class.getName(), args);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
index f21190e..e2c8a6d 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
@@ -22,8 +22,8 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.file.rfile.bcfile.BCFile.MetaIndexEntry;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -57,7 +57,7 @@ public class PrintInfo {
Configuration conf = new Configuration();
@SuppressWarnings("deprecation")
AccumuloConfiguration siteConf = AccumuloConfiguration.getSiteConfiguration();
- FileSystem hadoopFs = VolumeConfiguration.getDefaultFilesystem(conf, siteConf);
+ FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, siteConf).getFileSystem();
FileSystem localFs = FileSystem.getLocal(conf);
Path path = new Path(args[0]);
FileSystem fs;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
index 0c2cf3c..ff6ba09 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
@@ -63,7 +63,6 @@ import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.data.thrift.TConstraintViolationSummary;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
import org.apache.accumulo.core.trace.DistributedTrace;
import org.apache.accumulo.core.util.BadArgumentException;
@@ -156,6 +155,7 @@ import org.apache.accumulo.core.util.shell.commands.UserCommand;
import org.apache.accumulo.core.util.shell.commands.UserPermissionsCommand;
import org.apache.accumulo.core.util.shell.commands.UsersCommand;
import org.apache.accumulo.core.util.shell.commands.WhoAmICommand;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooReader;
import org.apache.commons.cli.BasicParser;
@@ -432,7 +432,7 @@ public class Shell extends ShellOptions {
if (instanceName == null || keepers == null) {
AccumuloConfiguration conf = SiteConfiguration.getInstance(ServerConfigurationUtil.convertClientConfig(DefaultConfiguration.getInstance(), clientConfig));
if (instanceName == null) {
- Path instanceDir = new Path(VolumeConfiguration.getConfiguredBaseDirs(conf)[0], "instance_id");
+ Path instanceDir = new Path(VolumeConfiguration.getVolumeUris(conf)[0], "instance_id");
instanceId = UUID.fromString(ZooUtil.getInstanceIDFromHdfs(instanceDir, conf));
}
if (keepers == null) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
new file mode 100644
index 0000000..08f61d4
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.volume;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Encapsulates a {@link FileSystem} and a base {@link Path} within that filesystem. This
+ * also avoid the necessity to pass around a Configuration.
+ */
+public interface Volume {
+
+ /**
+ * A {@link FileSystem} that Accumulo will use
+ * @return
+ */
+ public FileSystem getFileSystem();
+
+ /**
+ * The base path which Accumulo will use within the given {@link FileSystem}
+ * @return
+ */
+ public String getBasePath();
+
+ /**
+ * Convert the given Path into a Path that is relative to the base path for this Volume
+ * @param p
+ * @return
+ */
+ public Path prefixChild(Path p);
+
+ /**
+ * Convert the given child path into a Path that is relative to the base path for this Volume
+ * @param p
+ * @return
+ */
+ public Path prefixChild(String p);
+
+ /**
+ * Determine if the Path is valid on this Volume (contained by the basePath)
+ * @param p
+ * @return True if path is contained within the basePath, false otherwise
+ */
+ public boolean isValidPath(Path p);
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
new file mode 100644
index 0000000..3005174
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.volume;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import com.google.common.base.Preconditions;
+
+public class VolumeConfiguration {
+
+ public static Volume getVolume(String path, Configuration conf, AccumuloConfiguration acuconf) throws IOException {
+ Preconditions.checkNotNull(path);
+
+ if (path.contains(":")) {
+ // An absolute path
+ return create(new Path(path), conf);
+ } else {
+ // A relative path
+ return getDefaultVolume(conf, acuconf);
+ }
+ }
+
+ public static Volume getDefaultVolume(Configuration conf, AccumuloConfiguration acuconf) throws IOException {
+ @SuppressWarnings("deprecation")
+ String uri = acuconf.get(Property.INSTANCE_DFS_URI);
+
+ // By default pull from INSTANCE_DFS_URI, falling back to the Hadoop defined
+ // default filesystem (fs.defaultFS or the deprecated fs.default.name)
+ if ("".equals(uri))
+ return create(FileSystem.get(conf), acuconf);
+ else
+ try {
+ return create(FileSystem.get(new URI(uri), conf), acuconf);
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Deprecated
+ public static String getConfiguredBaseDir(AccumuloConfiguration conf) {
+ String singleNamespace = conf.get(Property.INSTANCE_DFS_DIR);
+ String dfsUri = conf.get(Property.INSTANCE_DFS_URI);
+ String baseDir;
+
+ if (dfsUri == null || dfsUri.isEmpty()) {
+ Configuration hadoopConfig = CachedConfiguration.getInstance();
+ try {
+ baseDir = FileSystem.get(hadoopConfig).getUri().toString() + singleNamespace;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ if (!dfsUri.contains(":"))
+ throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_DFS_URI.getKey() + " got " + dfsUri);
+ baseDir = dfsUri + singleNamespace;
+ }
+ return baseDir;
+ }
+
+ /**
+ * Compute the URIs to be used by Accumulo
+ * @param conf
+ * @return
+ */
+ public static String[] getVolumeUris(AccumuloConfiguration conf) {
+ String ns = conf.get(Property.INSTANCE_VOLUMES);
+
+ String configuredBaseDirs[];
+
+ if (ns == null || ns.isEmpty()) {
+ // Fall back to using the old config values
+ configuredBaseDirs = new String[] {getConfiguredBaseDir(conf)};
+ } else {
+ String namespaces[] = ns.split(",");
+ configuredBaseDirs = new String[namespaces.length];
+ int i = 0;
+ for (String namespace : namespaces) {
+ if (!namespace.contains(":")) {
+ throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_VOLUMES.getKey() + " got " + namespace);
+ }
+
+ try {
+ // pass through URI to unescape hex encoded chars (e.g. convert %2C to "," char)
+ configuredBaseDirs[i++] = new Path(new URI(namespace)).toString();
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(Property.INSTANCE_VOLUMES.getKey() + " contains " + namespace + " which has a syntax error", e);
+ }
+ }
+ }
+
+ return configuredBaseDirs;
+ }
+
+ public static String[] prefix(String bases[], String suffix) {
+ if (suffix.startsWith("/"))
+ suffix = suffix.substring(1);
+ String result[] = new String[bases.length];
+ for (int i = 0; i < bases.length; i++) {
+ result[i] = bases[i] + "/" + suffix;
+ }
+ return result;
+ }
+
+ /**
+ * Create a Volume with the given FileSystem that writes to the default path
+ * @param fs A FileSystem to write to
+ * @return A Volume instance writing to the given FileSystem in the default path
+ */
+ @SuppressWarnings("deprecation")
+ public static <T extends FileSystem> Volume create(T fs, AccumuloConfiguration acuconf) {
+ String dfsDir = acuconf.get(Property.INSTANCE_DFS_DIR);
+ return new VolumeImpl(fs, null == dfsDir ? Property.INSTANCE_DFS_DIR.getDefaultValue() : dfsDir);
+ }
+
+ public static <T extends FileSystem> Volume create(T fs, String basePath) {
+ return new VolumeImpl(fs, basePath);
+ }
+
+ public static Volume create(String path, Configuration conf) throws IOException {
+ Preconditions.checkNotNull(path);
+ return create(new Path(path), conf);
+ }
+
+ public static Volume create(Path path, Configuration conf) throws IOException {
+ return new VolumeImpl(path, conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
new file mode 100644
index 0000000..0aaf482
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.volume;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+
+/**
+ *
+ */
+public class VolumeImpl implements Volume {
+ protected final FileSystem fs;
+ protected final String basePath;
+
+ public VolumeImpl(Path path, Configuration conf) throws IOException {
+ checkNotNull(path);
+ checkNotNull(conf);
+
+ this.fs = path.getFileSystem(conf);
+ this.basePath = path.toUri().getPath();
+ }
+
+ public VolumeImpl(FileSystem fs, String basePath) {
+ checkNotNull(fs);
+ checkNotNull(basePath);
+
+ this.fs = fs;
+ this.basePath = basePath;
+ }
+
+ @Override
+ public FileSystem getFileSystem() {
+ return fs;
+ }
+
+ @Override
+ public String getBasePath() {
+ return basePath;
+ }
+
+ @Override
+ public Path prefixChild(Path p) {
+ return new Path(basePath, p);
+ }
+
+ @Override
+ public boolean isValidPath(Path p) {
+ checkNotNull(p);
+
+ return p.toUri().getPath().startsWith(basePath);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof VolumeImpl) {
+ VolumeImpl other = (VolumeImpl) o;
+ return getFileSystem().equals(other.getFileSystem()) && getBasePath().equals(other.getBasePath());
+ }
+
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return getFileSystem() + " " + basePath;
+ }
+
+ @Override
+ public Path prefixChild(String p) {
+ return new Path(basePath, p);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooUtil.java b/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooUtil.java
index de1b432..d536f42 100644
--- a/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooUtil.java
@@ -23,8 +23,8 @@ import java.net.UnknownHostException;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -49,7 +49,7 @@ public class ZooUtil extends org.apache.accumulo.fate.zookeeper.ZooUtil {
public static String getInstanceIDFromHdfs(Path instanceDirectory, AccumuloConfiguration conf) {
try {
- FileSystem fs = VolumeConfiguration.getFileSystem(instanceDirectory.toString(), CachedConfiguration.getInstance(), conf);
+ FileSystem fs = VolumeConfiguration.getVolume(instanceDirectory.toString(), CachedConfiguration.getInstance(), conf).getFileSystem();
FileStatus[] files = null;
try {
files = fs.listStatus(instanceDirectory);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index 2fa9051..48534f0 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -31,6 +31,7 @@ import org.apache.accumulo.core.trace.DistributedTrace;
import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.util.Version;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
@@ -50,10 +51,12 @@ public class Accumulo {
private static final Logger log = Logger.getLogger(Accumulo.class);
public static synchronized void updateAccumuloVersion(VolumeManager fs) {
+ // TODO ACCUMULO-2451 Should update all volumes, not one
+ Volume volume = fs.getVolumes().iterator().next();
try {
if (getAccumuloPersistentVersion(fs) == ServerConstants.PREV_DATA_VERSION) {
- fs.create(new Path(ServerConstants.getDataVersionLocation() + "/" + ServerConstants.DATA_VERSION));
- fs.delete(new Path(ServerConstants.getDataVersionLocation() + "/" + ServerConstants.PREV_DATA_VERSION));
+ fs.create(new Path(ServerConstants.getDataVersionLocation(volume), Integer.toString(ServerConstants.DATA_VERSION)));
+ fs.delete(new Path(ServerConstants.getDataVersionLocation(volume), Integer.toString(ServerConstants.PREV_DATA_VERSION)));
}
} catch (IOException e) {
throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
@@ -76,8 +79,10 @@ public class Accumulo {
}
public static synchronized int getAccumuloPersistentVersion(VolumeManager fs) {
- Path path = ServerConstants.getDataVersionLocation();
- return getAccumuloPersistentVersion(fs.getFileSystemByPath(path), path);
+ // It doesn't matter which Volume is used as they should all have the data version stored
+ Volume v = fs.getVolumes().iterator().next();
+ Path path = ServerConstants.getDataVersionLocation(v);
+ return getAccumuloPersistentVersion(v.getFileSystem(), path);
}
public static void enableTracing(String address, String application) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index 8983d08..7dd0a08 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -24,10 +24,11 @@ import java.util.HashSet;
import java.util.List;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.VolumeUtil;
@@ -51,29 +52,20 @@ public class ServerConstants {
public static final int DATA_VERSION = 6;
public static final int PREV_DATA_VERSION = 5;
- private static String[] baseDirs = null;
- private static String defaultBaseDir = null;
+ private static String[] baseUris = null;
private static List<Pair<Path,Path>> replacementsList = null;
- public static synchronized String getDefaultBaseDir() {
- if (defaultBaseDir == null) {
- defaultBaseDir = new Path(VolumeConfiguration.getConfiguredBaseDir(ServerConfiguration.getSiteConfiguration())).toString();
- }
-
- return defaultBaseDir;
- }
-
// these are functions to delay loading the Accumulo configuration unless we must
- public static synchronized String[] getBaseDirs() {
- if (baseDirs == null) {
- baseDirs = checkBaseDirs(VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration()), false);
+ public static synchronized String[] getBaseUris() {
+ if (baseUris == null) {
+ baseUris = checkBaseUris(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()), false);
}
- return baseDirs;
+ return baseUris;
}
- public static String[] checkBaseDirs(String[] configuredBaseDirs, boolean ignore) {
+ public static String[] checkBaseUris(String[] configuredBaseDirs, boolean ignore) {
// all base dirs must have same instance id and data version, any dirs that have neither should be ignored
String firstDir = null;
String firstIid = null;
@@ -121,29 +113,29 @@ public class ServerConstants {
public static final String WAL_DIR = "wal";
public static String[] getTablesDirs() {
- return VolumeConfiguration.prefix(getBaseDirs(), TABLE_DIR);
+ return VolumeConfiguration.prefix(getBaseUris(), TABLE_DIR);
}
public static String[] getRecoveryDirs() {
- return VolumeConfiguration.prefix(getBaseDirs(), RECOVERY_DIR);
+ return VolumeConfiguration.prefix(getBaseUris(), RECOVERY_DIR);
}
public static String[] getWalDirs() {
- return VolumeConfiguration.prefix(getBaseDirs(), WAL_DIR);
+ return VolumeConfiguration.prefix(getBaseUris(), WAL_DIR);
}
public static String[] getWalogArchives() {
- return VolumeConfiguration.prefix(getBaseDirs(), "walogArchive");
+ return VolumeConfiguration.prefix(getBaseUris(), "walogArchive");
}
- public static Path getInstanceIdLocation() {
+ public static Path getInstanceIdLocation(Volume v) {
// all base dirs should have the same instance id, so can choose any one
- return new Path(getBaseDirs()[0], INSTANCE_ID_DIR);
+ return v.prefixChild(INSTANCE_ID_DIR);
}
- public static Path getDataVersionLocation() {
+ public static Path getDataVersionLocation(Volume v) {
// all base dirs should have the same version, so can choose any one
- return new Path(getBaseDirs()[0], VERSION_DIR);
+ return v.prefixChild(VERSION_DIR);
}
public static String[] getMetadataTableDirs() {
@@ -151,7 +143,7 @@ public class ServerConstants {
}
public static String[] getTemporaryDirs() {
- return VolumeConfiguration.prefix(getBaseDirs(), "tmp");
+ return VolumeConfiguration.prefix(getBaseUris(), "tmp");
}
public static synchronized List<Pair<Path,Path>> getVolumeReplacements() {
@@ -194,9 +186,9 @@ public class ServerConstants {
}
HashSet<Path> baseDirs = new HashSet<Path>();
- for (String baseDir : getBaseDirs()) {
- // normalize using path and remove accumulo dir
- baseDirs.add(new Path(baseDir).getParent());
+ for (String baseDir : getBaseUris()) {
+ // normalize using path
+ baseDirs.add(new Path(baseDir));
}
for (Pair<Path,Path> pair : ret)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
index dc9acf8..27ab078 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
@@ -339,7 +339,7 @@ public class BulkImporter {
try {
for (Path path : paths) {
- FileSystem fs = vm.getFileSystemByPath(path);
+ FileSystem fs = vm.getVolumeByPath(path).getFileSystem();
mapFileSizes.put(path, fs.getContentSummary(path).getLength());
}
} catch (IOException e) {
@@ -639,7 +639,7 @@ public class BulkImporter {
Collection<ByteSequence> columnFamilies = Collections.emptyList();
String filename = file.toString();
// log.debug(filename + " finding overlapping tablets " + startRow + " -> " + endRow);
- FileSystem fs = vm.getFileSystemByPath(file);
+ FileSystem fs = vm.getVolumeByPath(file).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().openReader(filename, true, fs, fs.getConf(), acuConf);
try {
Text row = startRow;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index 6993a0a..ee928f3 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.client;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
@@ -38,10 +39,13 @@ import org.apache.accumulo.core.util.ByteBufferUtil;
import org.apache.accumulo.core.util.OpTimer;
import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.zookeeper.ZooLock;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Level;
@@ -122,7 +126,17 @@ public class HdfsZooInstance implements Instance {
private static synchronized void _getInstanceID() {
if (instanceId == null) {
- String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(), ServerConfiguration.getSiteConfiguration());
+ AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
+ // InstanceID should be the same across all volumes, so just choose one
+ VolumeManager fs;
+ try {
+ fs = VolumeManagerImpl.get();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Volume randVolume = fs.getVolumes().iterator().next();
+ log.trace("Looking for instanceId from " + randVolume);
+ String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), acuConf);
instanceId = instanceIdFromFile;
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
index 32f6126..94e468b 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.conf;
+import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -25,10 +26,15 @@ import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.client.HdfsZooInstance.AccumuloNotInitializedException;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.log4j.Logger;
public class ZooConfiguration extends AccumuloConfiguration {
@@ -57,7 +63,15 @@ public class ZooConfiguration extends AccumuloConfiguration {
if (instance == null) {
propCache = new ZooCache(parent.get(Property.INSTANCE_ZK_HOST), (int) parent.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
instance = new ZooConfiguration(parent);
- String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(), parent);
+ // InstanceID should be the same across all volumes, so just choose one
+ VolumeManager fs;
+ try {
+ fs = VolumeManagerImpl.get();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Volume randVolume = fs.getVolumes().iterator().next();
+ String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), parent);
instanceId = deprecatedInstanceIdFromHdfs;
}
return instance;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
index f0c7083..9b8fb98 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
@@ -17,14 +17,15 @@
package org.apache.accumulo.server.fs;
import java.io.IOException;
+import java.util.Collection;
import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.server.ServerConstants;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
/**
@@ -47,9 +48,10 @@ public interface VolumeManager {
}
private static int endOfVolumeIndex(String path, String dir) {
+ // Strip off the suffix that starts with the FileType (e.g. tables, wal, etc)
int dirIndex = path.indexOf('/' + dir);
if (dirIndex != -1) {
- return path.lastIndexOf('/', dirIndex - 1);
+ return dirIndex;
}
if (path.contains(":"))
@@ -110,7 +112,7 @@ public interface VolumeManager {
FileStatus getFileStatus(Path path) throws IOException;
// find the appropriate FileSystem object given a path
- FileSystem getFileSystemByPath(Path path);
+ Volume getVolumeByPath(Path path);
// return the item in options that is in the same volume as source
Path matchingFileSystem(Path source, String[] options);
@@ -155,4 +157,17 @@ public interface VolumeManager {
// decide on which of the given locations to create a new file
String choose(String[] options);
+
+ /**
+ * Fetch the default Volume
+ * @return
+ */
+ public Volume getDefaultVolume();
+
+ /**
+ * Fetch the configured Volumes, excluding the default Volume
+ * @return
+ */
+ public Collection<Volume> getVolumes();
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index 80301ef..ca5167d 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -16,12 +16,15 @@
*/
package org.apache.accumulo.server.fs;
+import static com.google.common.base.Preconditions.checkNotNull;
+
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URI;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
@@ -34,9 +37,9 @@ import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.commons.lang.NotImplementedException;
@@ -54,34 +57,49 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.util.Progressable;
import org.apache.log4j.Logger;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
public class VolumeManagerImpl implements VolumeManager {
private static final Logger log = Logger.getLogger(VolumeManagerImpl.class);
- Map<String,? extends FileSystem> volumes;
- FileSystem defaultVolume;
+ Map<String,Volume> volumesByName;
+ Multimap<FileSystem,Volume> volumesByFileSystem;
+ Volume defaultVolume;
AccumuloConfiguration conf;
VolumeChooser chooser;
- protected VolumeManagerImpl(Map<String,? extends FileSystem> volumes, String defaultVolume, AccumuloConfiguration conf) {
- this.volumes = volumes;
- this.defaultVolume = volumes.get(defaultVolume);
+ protected VolumeManagerImpl(Map<String,Volume> volumes, Volume defaultVolume, AccumuloConfiguration conf) {
+ this.volumesByName = volumes;
+ this.defaultVolume = defaultVolume;
+ // We may have multiple directories used in a single FileSystem (e.g. testing)
+ this.volumesByFileSystem = HashMultimap.create();
+ invertVolumesByFileSystem(volumesByName, volumesByFileSystem);
this.conf = conf;
ensureSyncIsEnabled();
chooser = Property.createInstanceFromPropertyName(conf, Property.GENERAL_VOLUME_CHOOSER, VolumeChooser.class, new RandomVolumeChooser());
}
- public static org.apache.accumulo.server.fs.VolumeManager getLocal() throws IOException {
- return new VolumeManagerImpl(Collections.singletonMap("", FileSystem.getLocal(CachedConfiguration.getInstance())), "",
- DefaultConfiguration.getDefaultConfiguration());
+ private void invertVolumesByFileSystem(Map<String,Volume> forward, Multimap<FileSystem,Volume> inverted) {
+ for (Volume volume : forward.values()) {
+ inverted.put(volume.getFileSystem(), volume);
+ }
+ }
+
+ public static org.apache.accumulo.server.fs.VolumeManager getLocal(String localBasePath) throws IOException {
+ AccumuloConfiguration accConf = DefaultConfiguration.getDefaultConfiguration();
+ Volume defaultLocalVolume = VolumeConfiguration.create(FileSystem.getLocal(CachedConfiguration.getInstance()), localBasePath);
+
+ return new VolumeManagerImpl(Collections.singletonMap(DEFAULT, defaultLocalVolume), defaultLocalVolume, accConf);
}
@Override
public void close() throws IOException {
IOException ex = null;
- for (FileSystem fs : volumes.values()) {
+ for (Volume volume : volumesByName.values()) {
try {
- fs.close();
+ volume.getFileSystem().close();
} catch (IOException e) {
ex = e;
}
@@ -93,14 +111,20 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public FSDataOutputStream create(Path path) throws IOException {
- FileSystem fs = getFileSystemByPath(path);
- return fs.create(path);
+ checkNotNull(path);
+
+ Volume v = getVolumeByPath(path);
+
+ return v.getFileSystem().create(path);
}
@Override
public FSDataOutputStream create(Path path, boolean overwrite) throws IOException {
- FileSystem fs = getFileSystemByPath(path);
- return fs.create(path, overwrite);
+ checkNotNull(path);
+
+ Volume v = getVolumeByPath(path);
+
+ return v.getFileSystem().create(path, overwrite);
}
private static long correctBlockSize(Configuration conf, long blockSize) {
@@ -121,7 +145,11 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public FSDataOutputStream create(Path path, boolean overwrite, int bufferSize, short replication, long blockSize) throws IOException {
- FileSystem fs = getFileSystemByPath(path);
+ checkNotNull(path);
+
+ Volume v = getVolumeByPath(path);
+ FileSystem fs = v.getFileSystem();
+
if (bufferSize == 0) {
fs.getConf().getInt("io.file.buffer.size", 4096);
}
@@ -130,13 +158,16 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public boolean createNewFile(Path path) throws IOException {
- FileSystem fs = getFileSystemByPath(path);
- return fs.createNewFile(path);
+ checkNotNull(path);
+
+ Volume v = getVolumeByPath(path);
+ return v.getFileSystem().createNewFile(path);
}
@Override
public FSDataOutputStream createSyncable(Path logPath, int bufferSize, short replication, long blockSize) throws IOException {
- FileSystem fs = getFileSystemByPath(logPath);
+ Volume v = getVolumeByPath(logPath);
+ FileSystem fs = v.getFileSystem();
blockSize = correctBlockSize(fs.getConf(), blockSize);
bufferSize = correctBufferSize(fs.getConf(), bufferSize);
try {
@@ -174,18 +205,18 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public boolean delete(Path path) throws IOException {
- return getFileSystemByPath(path).delete(path, false);
+ return getVolumeByPath(path).getFileSystem().delete(path, false);
}
@Override
public boolean deleteRecursively(Path path) throws IOException {
- return getFileSystemByPath(path).delete(path, true);
+ return getVolumeByPath(path).getFileSystem().delete(path, true);
}
protected void ensureSyncIsEnabled() {
- for (Entry<String,? extends FileSystem> entry : getFileSystems().entrySet()) {
+ for (Entry<String,Volume> entry : getFileSystems().entrySet()) {
final String volumeName = entry.getKey();
- FileSystem fs = entry.getValue();
+ FileSystem fs = entry.getValue().getFileSystem();
if (ViewFSUtils.isViewFS(fs)) {
try {
@@ -255,19 +286,36 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public boolean exists(Path path) throws IOException {
- return getFileSystemByPath(path).exists(path);
+ return getVolumeByPath(path).getFileSystem().exists(path);
}
@Override
public FileStatus getFileStatus(Path path) throws IOException {
- return getFileSystemByPath(path).getFileStatus(path);
+ return getVolumeByPath(path).getFileSystem().getFileStatus(path);
}
@Override
- public FileSystem getFileSystemByPath(Path path) {
+ public Volume getVolumeByPath(Path path) {
if (path.toString().contains(":")) {
try {
- return path.getFileSystem(CachedConfiguration.getInstance());
+ FileSystem pathFs = path.getFileSystem(CachedConfiguration.getInstance());
+ Collection<Volume> candidateVolumes = volumesByFileSystem.get(pathFs);
+ if (null != candidateVolumes) {
+ for (Volume candidateVolume : candidateVolumes) {
+ if (candidateVolume.isValidPath(path)) {
+ return candidateVolume;
+ }
+ }
+
+ // For the same reason as we can have multiple Volumes within a single filesystem
+ // we could also not find a matching one. We should defer back to the defaultVolume
+ // e.g. volume rename with old path references
+ log.debug("Defaulting to " + defaultVolume + " as a valid volume could not be determined for " + path);
+
+ return defaultVolume;
+ }
+
+ log.debug("Could not determine volume for Path '" + path + "' from defined volumes");
} catch (IOException ex) {
throw new RuntimeException(ex);
}
@@ -276,29 +324,31 @@ public class VolumeManagerImpl implements VolumeManager {
return defaultVolume;
}
- private Map<String,? extends FileSystem> getFileSystems() {
- return volumes;
+ private Map<String,Volume> getFileSystems() {
+ return volumesByName;
}
@Override
public FileStatus[] listStatus(Path path) throws IOException {
- return getFileSystemByPath(path).listStatus(path);
+ return getVolumeByPath(path).getFileSystem().listStatus(path);
}
@Override
public boolean mkdirs(Path path) throws IOException {
- return getFileSystemByPath(path).mkdirs(path);
+ return getVolumeByPath(path).getFileSystem().mkdirs(path);
}
@Override
public FSDataInputStream open(Path path) throws IOException {
- return getFileSystemByPath(path).open(path);
+ return getVolumeByPath(path).getFileSystem().open(path);
}
@Override
public boolean rename(Path path, Path newPath) throws IOException {
- FileSystem source = getFileSystemByPath(path);
- FileSystem dest = getFileSystemByPath(newPath);
+ Volume srcVolume = getVolumeByPath(path);
+ Volume destVolume = getVolumeByPath(newPath);
+ FileSystem source = srcVolume.getFileSystem();
+ FileSystem dest = destVolume.getFileSystem();
if (source != dest) {
throw new NotImplementedException("Cannot rename files across volumes: " + path + " -> " + newPath);
}
@@ -307,14 +357,15 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public boolean moveToTrash(Path path) throws IOException {
- FileSystem fs = getFileSystemByPath(path);
+ FileSystem fs = getVolumeByPath(path).getFileSystem();
Trash trash = new Trash(fs, fs.getConf());
return trash.moveToTrash(path);
}
@Override
public short getDefaultReplication(Path path) {
- FileSystem fs = getFileSystemByPath(path);
+ Volume v = getVolumeByPath(path);
+ FileSystem fs = v.getFileSystem();
try {
// try calling hadoop 2 method
Method method = fs.getClass().getMethod("getDefaultReplication", Path.class);
@@ -336,7 +387,7 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public boolean isFile(Path path) throws IOException {
- return getFileSystemByPath(path).isFile(path);
+ return getVolumeByPath(path).getFileSystem().isFile(path);
}
public static VolumeManager get() throws IOException {
@@ -347,26 +398,30 @@ public class VolumeManagerImpl implements VolumeManager {
static private final String DEFAULT = "";
public static VolumeManager get(AccumuloConfiguration conf) throws IOException {
- Map<String,FileSystem> fileSystems = new HashMap<String,FileSystem>();
- Configuration hadoopConf = CachedConfiguration.getInstance();
- fileSystems.put(DEFAULT, VolumeConfiguration.getDefaultFilesystem(hadoopConf, conf));
- for (String space : VolumeConfiguration.getConfiguredBaseDirs(conf)) {
- if (space.equals(DEFAULT))
+ final Map<String,Volume> volumes = new HashMap<String,Volume>();
+ final Configuration hadoopConf = CachedConfiguration.getInstance();
+
+ // The "default" Volume for Accumulo (in case no volumes are specified)
+ for (String volumeUriOrDir : VolumeConfiguration.getVolumeUris(conf)) {
+ if (volumeUriOrDir.equals(DEFAULT))
+ // Cannot re-define the default volume
throw new IllegalArgumentException();
- if (space.contains(":")) {
- fileSystems.put(space, new Path(space).getFileSystem(hadoopConf));
+ // We require a URI here, fail if it doesn't look like one
+ if (volumeUriOrDir.contains(":")) {
+ volumes.put(volumeUriOrDir, VolumeConfiguration.create(new Path(volumeUriOrDir), hadoopConf));
} else {
- throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_VOLUMES.getKey() + " got " + space);
+ throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_VOLUMES.getKey() + " got " + volumeUriOrDir);
}
}
- return new VolumeManagerImpl(fileSystems, DEFAULT, conf);
+ return new VolumeManagerImpl(volumes, VolumeConfiguration.getDefaultVolume(hadoopConf, conf), conf);
}
@Override
public boolean isReady() throws IOException {
- for (FileSystem fs : getFileSystems().values()) {
+ for (Volume volume : getFileSystems().values()) {
+ FileSystem fs = volume.getFileSystem();
if (ViewFSUtils.isViewFS(fs)) {
try {
@@ -421,7 +476,7 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public FileStatus[] globStatus(Path pathPattern) throws IOException {
- return getFileSystemByPath(pathPattern).globStatus(pathPattern);
+ return getVolumeByPath(pathPattern).getFileSystem().globStatus(pathPattern);
}
@Override
@@ -476,18 +531,18 @@ public class VolumeManagerImpl implements VolumeManager {
return new Path(path);
// normalize the path
- Path fullPath = new Path(ServerConstants.getDefaultBaseDir(), fileType.getDirectory());
+ Path fullPath = new Path(defaultVolume.getBasePath(), fileType.getDirectory());
if (path.startsWith("/"))
path = path.substring(1);
fullPath = new Path(fullPath, path);
- FileSystem fs = getFileSystemByPath(fullPath);
+ FileSystem fs = getVolumeByPath(fullPath).getFileSystem();
return fs.makeQualified(fullPath);
}
@Override
public ContentSummary getContentSummary(Path dir) throws IOException {
- return getFileSystemByPath(dir).getContentSummary(dir);
+ return getVolumeByPath(dir).getFileSystem().getContentSummary(dir);
}
@Override
@@ -495,4 +550,14 @@ public class VolumeManagerImpl implements VolumeManager {
return chooser.choose(options);
}
+ @Override
+ public Volume getDefaultVolume() {
+ return defaultVolume;
+ }
+
+ @Override
+ public Collection<Volume> getVolumes() {
+ return volumesByName.values();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index da3baa6..bcfb008 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -52,7 +52,7 @@ import org.apache.log4j.Logger;
public class VolumeUtil {
private static final Logger log = Logger.getLogger(VolumeUtil.class);
-
+
private static boolean isActiveVolume(Path dir) {
// consider relative path as active and take no action
@@ -81,8 +81,10 @@ public class VolumeUtil {
}
public static String switchVolume(String path, FileType ft, List<Pair<Path,Path>> replacements) {
- if (replacements.size() == 0)
+ if (replacements.size() == 0) {
+ log.trace("Not switching volume because there are no replacements");
return null;
+ }
Path p = new Path(path);
@@ -92,10 +94,15 @@ public class VolumeUtil {
for (Pair<Path,Path> pair : replacements) {
Path key = removeTrailingSlash(pair.getFirst());
- if (key.equals(volume))
- return new Path(pair.getSecond(), ft.removeVolume(p)).toString();
+ if (key.equals(volume)) {
+ String replacement = new Path(pair.getSecond(), ft.removeVolume(p)).toString();
+ log.trace("Replacing " + path + " with " + replacement);
+ return replacement;
+ }
}
+ log.trace("Could not find replacement for " + ft + " at " + path);
+
return null;
}
@@ -119,13 +126,17 @@ public class VolumeUtil {
}
- if (numSwitched == 0)
+ if (numSwitched == 0) {
+ log.trace("Did not switch " + le);
return null;
+ }
LogEntry newLogEntry = new LogEntry(le);
newLogEntry.filename = switchedPath;
newLogEntry.logSet = switchedLogs;
+ log.trace("Switched " + le + " to " + newLogEntry);
+
return newLogEntry;
}
@@ -165,6 +176,7 @@ public class VolumeUtil {
*/
public static TabletFiles updateTabletVolumes(ZooLock zooLock, VolumeManager vm, KeyExtent extent, TabletFiles tabletFiles) throws IOException {
List<Pair<Path,Path>> replacements = ServerConstants.getVolumeReplacements();
+ log.trace("Using volume replacements: " + replacements);
List<LogEntry> logsToRemove = new ArrayList<LogEntry>();
List<LogEntry> logsToAdd = new ArrayList<LogEntry>();
@@ -239,8 +251,8 @@ public class VolumeUtil {
// this code needs to be idempotent
- FileSystem fs1 = vm.getFileSystemByPath(dir);
- FileSystem fs2 = vm.getFileSystemByPath(newDir);
+ FileSystem fs1 = vm.getVolumeByPath(dir).getFileSystem();
+ FileSystem fs2 = vm.getVolumeByPath(newDir).getFileSystem();
if (!same(fs1, dir, fs2, newDir)) {
if (fs2.exists(newDir)) {
@@ -335,4 +347,5 @@ public class VolumeUtil {
SecureRandom rand = new SecureRandom();
return new Path(path.getParent(), path.getName() + "_" + System.currentTimeMillis() + "_" + Math.abs(rand.nextInt()) + ".bak");
}
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index 925f602..206a721 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@ -40,7 +40,6 @@ import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVWriter;
-import org.apache.accumulo.core.file.VolumeConfiguration;
import org.apache.accumulo.core.iterators.user.VersioningIterator;
import org.apache.accumulo.core.master.state.tables.TableState;
import org.apache.accumulo.core.master.thrift.MasterGoalState;
@@ -52,6 +51,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Da
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
import org.apache.accumulo.core.security.SecurityUtil;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
@@ -151,7 +151,7 @@ public class Initialize {
else
fsUri = FileSystem.getDefaultUri(conf).toString();
log.info("Hadoop Filesystem is " + fsUri);
- log.info("Accumulo data dirs are " + Arrays.asList(VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration())));
+ log.info("Accumulo data dirs are " + Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())));
log.info("Zookeeper server is " + sconf.get(Property.INSTANCE_ZK_HOST));
log.info("Checking if Zookeeper is available. If this hangs, then you need to make sure zookeeper is running");
if (!zookeeperAvailable()) {
@@ -173,7 +173,7 @@ public class Initialize {
try {
if (isInitialized(fs)) {
String instanceDfsDir = sconf.get(Property.INSTANCE_DFS_DIR);
- log.fatal("It appears the directories " + Arrays.asList(VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration()))
+ log.fatal("It appears the directories " + Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()))
+ " were previously initialized.");
String instanceVolumes = sconf.get(Property.INSTANCE_VOLUMES);
String instanceDfsUri = sconf.get(Property.INSTANCE_DFS_URI);
@@ -220,7 +220,7 @@ public class Initialize {
UUID uuid = UUID.randomUUID();
// the actual disk locations of the root table and tablets
- String[] configuredTableDirs = VolumeConfiguration.prefix(VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration()),
+ String[] configuredTableDirs = VolumeConfiguration.prefix(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()),
ServerConstants.TABLE_DIR);
final Path rootTablet = new Path(fs.choose(configuredTableDirs) + "/" + RootTable.ID + RootTable.ROOT_TABLET_LOCATION);
try {
@@ -296,7 +296,7 @@ public class Initialize {
private static void initFileSystem(Opts opts, VolumeManager fs, UUID uuid, Path rootTablet) throws IOException {
FileStatus fstat;
- initDirs(fs, uuid, VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration()), false);
+ initDirs(fs, uuid, VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()), false);
// the actual disk locations of the metadata table and tablets
final Path[] metadataTableDirs = paths(ServerConstants.getMetadataTableDirs());
@@ -341,7 +341,7 @@ public class Initialize {
// the root tablet contains the key extent and locations of all the
// metadata tablets
String initRootTabFile = rootTablet + "/00000_00000." + FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration());
- FileSystem ns = fs.getFileSystemByPath(new Path(initRootTabFile));
+ FileSystem ns = fs.getVolumeByPath(new Path(initRootTabFile)).getFileSystem();
FileSKVWriter mfw = FileOperations.getInstance().openWriter(initRootTabFile, ns, ns.getConf(), AccumuloConfiguration.getDefaultConfiguration());
mfw.startDefaultLocalityGroup();
@@ -553,7 +553,7 @@ public class Initialize {
}
public static boolean isInitialized(VolumeManager fs) throws IOException {
- for (String baseDir : VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration())) {
+ for (String baseDir : VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())) {
if (fs.exists(new Path(baseDir, ServerConstants.INSTANCE_ID_DIR)) || fs.exists(new Path(baseDir, ServerConstants.VERSION_DIR)))
return true;
}
@@ -564,10 +564,10 @@ public class Initialize {
private static void addVolumes(VolumeManager fs) throws IOException {
HashSet<String> initializedDirs = new HashSet<String>();
initializedDirs
- .addAll(Arrays.asList(ServerConstants.checkBaseDirs(VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration()), true)));
+ .addAll(Arrays.asList(ServerConstants.checkBaseUris(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()), true)));
HashSet<String> uinitializedDirs = new HashSet<String>();
- uinitializedDirs.addAll(Arrays.asList(VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration())));
+ uinitializedDirs.addAll(Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())));
uinitializedDirs.removeAll(initializedDirs);
Path aBasePath = new Path(initializedDirs.iterator().next());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
index 7edc0cf..0e7f83a 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
@@ -37,7 +37,7 @@ public class HadoopLogCloser implements LogCloser {
@Override
public long close(AccumuloConfiguration conf, VolumeManager fs, Path source) throws IOException {
- FileSystem ns = fs.getFileSystemByPath(source);
+ FileSystem ns = fs.getVolumeByPath(source).getFileSystem();
// if path points to a viewfs path, then resolve to underlying filesystem
if (ViewFSUtils.isViewFS(ns)) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
index bba7ac5..d737109 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
@@ -32,7 +32,7 @@ public class MapRLogCloser implements LogCloser {
@Override
public long close(AccumuloConfiguration conf, VolumeManager fs, Path path) throws IOException {
log.info("Recovering file " + path.toString() + " by changing permission to readonly");
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
FsPermission roPerm = new FsPermission((short) 0444);
try {
ns.setPermission(path, roPerm);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index ac13034..3f33a0e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@ -24,7 +24,7 @@ import java.util.UUID;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.ZooReader;
@@ -32,8 +32,9 @@ import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.ACL;
@@ -57,7 +58,7 @@ public class ChangeSecret {
argsList.add("--new");
argsList.addAll(Arrays.asList(args));
opts.parseArgs(ChangeSecret.class.getName(), argsList.toArray(new String[0]));
- FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+ VolumeManager fs = VolumeManagerImpl.get();
Instance inst = opts.getInstance();
if (!verifyAccumuloIsDown(inst, opts.oldPass))
System.exit(-1);
@@ -142,10 +143,13 @@ public class ChangeSecret {
return newInstanceId;
}
- private static void updateHdfs(FileSystem fs, Instance inst, String newInstanceId) throws IOException {
- fs.delete(ServerConstants.getInstanceIdLocation(), true);
- fs.mkdirs(ServerConstants.getInstanceIdLocation());
- fs.create(new Path(ServerConstants.getInstanceIdLocation(), newInstanceId)).close();
+ private static void updateHdfs(VolumeManager fs, Instance inst, String newInstanceId) throws IOException {
+ // Need to recreate the instanceId on all of them to keep consistency
+ for (Volume v : fs.getVolumes()) {
+ v.getFileSystem().delete(ServerConstants.getInstanceIdLocation(v), true);
+ v.getFileSystem().mkdirs(ServerConstants.getInstanceIdLocation(v));
+ v.getFileSystem().create(new Path(ServerConstants.getInstanceIdLocation(v), newInstanceId)).close();
+ }
}
private static void deleteInstance(Instance origInstance, String oldPass) throws Exception {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
index 8e38cbd..277a0c2 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
@@ -125,7 +125,7 @@ public class FileUtil {
String newMapFile = String.format("%s/%04d.%s", newDir, count++, RFile.EXTENSION);
outFiles.add(newMapFile);
- FileSystem ns = fs.getFileSystemByPath(new Path(newMapFile));
+ FileSystem ns = fs.getVolumeByPath(new Path(newMapFile)).getFileSystem();
FileSKVWriter writer = new RFileOperations().openWriter(newMapFile.toString(), ns, ns.getConf(), acuConf);
writer.startDefaultLocalityGroup();
List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(inFiles.size());
@@ -133,7 +133,7 @@ public class FileUtil {
FileSKVIterator reader = null;
try {
for (String s : inFiles) {
- ns = fs.getFileSystemByPath(new Path(s));
+ ns = fs.getVolumeByPath(new Path(s)).getFileSystem();
reader = FileOperations.getInstance().openIndex(s, ns, ns.getConf(), acuConf);
iters.add(reader);
}
@@ -390,7 +390,7 @@ public class FileUtil {
for (String ref : mapFiles) {
FileSKVIterator reader = null;
Path path = new Path(ref);
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
try {
if (useIndex)
reader = FileOperations.getInstance().openIndex(path.toString(), ns, ns.getConf(), acuConf);
@@ -435,7 +435,7 @@ public class FileUtil {
for (FileRef mapfile : mapfiles) {
FileSKVIterator reader = null;
- FileSystem ns = fs.getFileSystemByPath(mapfile.path());
+ FileSystem ns = fs.getVolumeByPath(mapfile.path()).getFileSystem();
try {
reader = FileOperations.getInstance().openReader(mapfile.toString(), false, ns, ns.getConf(), acuConf);
@@ -470,7 +470,7 @@ public class FileUtil {
for (FileRef ref : mapFiles) {
Path path = ref.path();
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), acuConf);
try {
@@ -513,7 +513,7 @@ public class FileUtil {
counts.put(keyExtent, new MLong(0));
Text row = new Text();
- FileSystem ns = fs.getFileSystemByPath(mapFile);
+ FileSystem ns = fs.getVolumeByPath(mapFile).getFileSystem();
FileSKVIterator index = FileOperations.getInstance().openIndex(mapFile.toString(), ns, ns.getConf(), acuConf);
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
index a96e791..35bb8f5 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
@@ -88,7 +88,7 @@ public class LocalityCheck {
}
for (String file : files) {
Path filePath = new Path(file);
- FileSystem ns = fs.getFileSystemByPath(filePath);
+ FileSystem ns = fs.getVolumeByPath(filePath).getFileSystem();
FileStatus fileStatus = ns.getFileStatus(filePath);
BlockLocation[] fileBlockLocations = ns.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
for (BlockLocation blockLocation : fileBlockLocations) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java b/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
index b237cd0..b8e7113 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
@@ -46,7 +46,7 @@ public class TabletOperations {
lowDirectory = Constants.DEFAULT_TABLET_LOCATION;
Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" + lowDirectory);
if (fs.exists(lowDirectoryPath) || fs.mkdirs(lowDirectoryPath)) {
- FileSystem pathFs = fs.getFileSystemByPath(lowDirectoryPath);
+ FileSystem pathFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem();
return lowDirectoryPath.makeQualified(pathFs.getUri(), pathFs.getWorkingDirectory()).toString();
}
log.warn("Failed to create " + lowDirectoryPath + " for unknown reason");
@@ -56,7 +56,7 @@ public class TabletOperations {
if (fs.exists(lowDirectoryPath))
throw new IllegalStateException("Dir exist when it should not " + lowDirectoryPath);
if (fs.mkdirs(lowDirectoryPath)) {
- FileSystem lowDirectoryFs = fs.getFileSystemByPath(lowDirectoryPath);
+ FileSystem lowDirectoryFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem();
return lowDirectoryPath.makeQualified(lowDirectoryFs.getUri(), lowDirectoryFs.getWorkingDirectory()).toString();
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java b/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java
index 37edb1a..7cac046 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java
@@ -41,7 +41,7 @@ public class ZooKeeperMain {
Opts opts = new Opts();
opts.parseArgs(ZooKeeperMain.class.getName(), args);
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
- String baseDir = ServerConstants.getBaseDirs()[0];
+ String baseDir = ServerConstants.getBaseUris()[0];
System.out.println("Using " + fs.makeQualified(new Path(baseDir + "/instance_id")) + " to lookup accumulo instance");
Instance instance = HdfsZooInstance.getInstance();
if (opts.servers == null) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/test/java/org/apache/accumulo/server/ServerConstantsTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/ServerConstantsTest.java b/server/base/src/test/java/org/apache/accumulo/server/ServerConstantsTest.java
index a316155..8c8b2f2 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/ServerConstantsTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/ServerConstantsTest.java
@@ -59,26 +59,26 @@ public class ServerConstantsTest {
}
private void verifyAllPass(ArrayList<String> paths) {
- Assert.assertEquals(paths, Arrays.asList(ServerConstants.checkBaseDirs(paths.toArray(new String[paths.size()]), true)));
- Assert.assertEquals(paths, Arrays.asList(ServerConstants.checkBaseDirs(paths.toArray(new String[paths.size()]), false)));
+ Assert.assertEquals(paths, Arrays.asList(ServerConstants.checkBaseUris(paths.toArray(new String[paths.size()]), true)));
+ Assert.assertEquals(paths, Arrays.asList(ServerConstants.checkBaseUris(paths.toArray(new String[paths.size()]), false)));
}
private void verifySomePass(ArrayList<String> paths, int numExpected) {
- Assert.assertEquals(paths.subList(0, 2), Arrays.asList(ServerConstants.checkBaseDirs(paths.toArray(new String[paths.size()]), true)));
+ Assert.assertEquals(paths.subList(0, 2), Arrays.asList(ServerConstants.checkBaseUris(paths.toArray(new String[paths.size()]), true)));
try {
- ServerConstants.checkBaseDirs(paths.toArray(new String[paths.size()]), false);
+ ServerConstants.checkBaseUris(paths.toArray(new String[paths.size()]), false);
Assert.fail();
} catch (Exception e) {}
}
private void verifyError(ArrayList<String> paths) {
try {
- ServerConstants.checkBaseDirs(paths.toArray(new String[paths.size()]), true);
+ ServerConstants.checkBaseUris(paths.toArray(new String[paths.size()]), true);
Assert.fail();
} catch (Exception e) {}
try {
- ServerConstants.checkBaseDirs(paths.toArray(new String[paths.size()]), false);
+ ServerConstants.checkBaseUris(paths.toArray(new String[paths.size()]), false);
Assert.fail();
} catch (Exception e) {}
}
[07/25] git commit: ACCUMULO-2061 Add comments,
javadoc and other cleanup
Posted by el...@apache.org.
ACCUMULO-2061 Add comments, javadoc and other cleanup
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/550e5e61
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/550e5e61
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/550e5e61
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 550e5e61affc490f0931a8991d40f283b0282115
Parents: d45b723
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 12:11:48 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Mar 20 18:58:53 2014 -0400
----------------------------------------------------------------------
.../accumulo/core/file/rfile/PrintInfo.java | 9 +++++-
.../org/apache/accumulo/core/volume/Volume.java | 15 ++++------
.../core/volume/VolumeConfiguration.java | 30 ++++++++++++--------
.../apache/accumulo/core/volume/VolumeImpl.java | 22 +++++++-------
.../accumulo/server/fs/VolumeManager.java | 2 --
.../accumulo/server/fs/VolumeManagerImpl.java | 19 +++++++------
.../apache/accumulo/server/fs/VolumeUtil.java | 2 +-
.../accumulo/server/util/ChangeSecret.java | 7 +++--
.../accumulo/server/fs/VolumeUtilTest.java | 2 +-
9 files changed, 59 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 4e39fc7..7c0f067 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -31,10 +31,12 @@ import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
import com.beust.jcommander.Parameter;
public class PrintInfo {
+ private static final Logger log = Logger.getLogger(PrintInfo.class);
static class Opts extends Help {
@Parameter(names = {"-d", "--dump"}, description = "dump the key/value pairs")
@@ -50,6 +52,8 @@ public class PrintInfo {
@SuppressWarnings("deprecation")
AccumuloConfiguration aconf = AccumuloConfiguration.getSiteConfiguration();
+ // TODO This will only work for RFiles in HDFS when the filesystem is defined in the core-site.xml
+ // on the classpath if a path, and not a URI, is given
FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem();
FileSystem localFs = FileSystem.getLocal(conf);
Opts opts = new Opts();
@@ -68,8 +72,11 @@ public class PrintInfo {
FileSystem fs;
if (arg.contains(":"))
fs = path.getFileSystem(conf);
- else
+ else {
+ // Recommend a URI is given for the above todo reason
+ log.warn("Attempting to find file across filesystems. Consider providing URI instead of path");
fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
+ }
CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null, null, aconf);
Reader iter = new RFile.Reader(_rdr);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
index 08f61d4..17b2bf3 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
@@ -21,39 +21,36 @@ import org.apache.hadoop.fs.Path;
/**
* Encapsulates a {@link FileSystem} and a base {@link Path} within that filesystem. This
- * also avoid the necessity to pass around a Configuration.
+ * also avoid the necessity to pass around a Configuration.
*/
public interface Volume {
/**
* A {@link FileSystem} that Accumulo will use
- * @return
*/
public FileSystem getFileSystem();
/**
* The base path which Accumulo will use within the given {@link FileSystem}
- * @return
*/
public String getBasePath();
-
+
/**
* Convert the given Path into a Path that is relative to the base path for this Volume
- * @param p
- * @return
+ * @param p The suffix to use
+ * @return A Path for this Volume with the provided suffix
*/
public Path prefixChild(Path p);
/**
* Convert the given child path into a Path that is relative to the base path for this Volume
- * @param p
- * @return
+ * @param p The suffix to use
+ * @return A Path for this Volume with the provided suffix
*/
public Path prefixChild(String p);
/**
* Determine if the Path is valid on this Volume (contained by the basePath)
- * @param p
* @return True if path is contained within the basePath, false otherwise
*/
public boolean isValidPath(Path p);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
index 3005174..5db5bb2 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
@@ -30,10 +30,10 @@ import org.apache.hadoop.fs.Path;
import com.google.common.base.Preconditions;
public class VolumeConfiguration {
-
+
public static Volume getVolume(String path, Configuration conf, AccumuloConfiguration acuconf) throws IOException {
Preconditions.checkNotNull(path);
-
+
if (path.contains(":")) {
// An absolute path
return create(new Path(path), conf);
@@ -59,12 +59,15 @@ public class VolumeConfiguration {
}
}
+ /**
+ * @see org.apache.accumulo.core.volume.VolumeConfiguration#getVolumeUris(AccumuloConfiguration)
+ */
@Deprecated
public static String getConfiguredBaseDir(AccumuloConfiguration conf) {
String singleNamespace = conf.get(Property.INSTANCE_DFS_DIR);
String dfsUri = conf.get(Property.INSTANCE_DFS_URI);
String baseDir;
-
+
if (dfsUri == null || dfsUri.isEmpty()) {
Configuration hadoopConfig = CachedConfiguration.getInstance();
try {
@@ -82,14 +85,15 @@ public class VolumeConfiguration {
/**
* Compute the URIs to be used by Accumulo
+ *
* @param conf
* @return
*/
public static String[] getVolumeUris(AccumuloConfiguration conf) {
String ns = conf.get(Property.INSTANCE_VOLUMES);
-
+
String configuredBaseDirs[];
-
+
if (ns == null || ns.isEmpty()) {
// Fall back to using the old config values
configuredBaseDirs = new String[] {getConfiguredBaseDir(conf)};
@@ -101,7 +105,7 @@ public class VolumeConfiguration {
if (!namespace.contains(":")) {
throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_VOLUMES.getKey() + " got " + namespace);
}
-
+
try {
// pass through URI to unescape hex encoded chars (e.g. convert %2C to "," char)
configuredBaseDirs[i++] = new Path(new URI(namespace)).toString();
@@ -110,7 +114,7 @@ public class VolumeConfiguration {
}
}
}
-
+
return configuredBaseDirs;
}
@@ -126,24 +130,26 @@ public class VolumeConfiguration {
/**
* Create a Volume with the given FileSystem that writes to the default path
- * @param fs A FileSystem to write to
- * @return A Volume instance writing to the given FileSystem in the default path
+ *
+ * @param fs
+ * A FileSystem to write to
+ * @return A Volume instance writing to the given FileSystem in the default path
*/
@SuppressWarnings("deprecation")
public static <T extends FileSystem> Volume create(T fs, AccumuloConfiguration acuconf) {
String dfsDir = acuconf.get(Property.INSTANCE_DFS_DIR);
return new VolumeImpl(fs, null == dfsDir ? Property.INSTANCE_DFS_DIR.getDefaultValue() : dfsDir);
}
-
+
public static <T extends FileSystem> Volume create(T fs, String basePath) {
return new VolumeImpl(fs, basePath);
}
-
+
public static Volume create(String path, Configuration conf) throws IOException {
Preconditions.checkNotNull(path);
return create(new Path(path), conf);
}
-
+
public static Volume create(Path path, Configuration conf) throws IOException {
return new VolumeImpl(path, conf);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
index 0aaf482..55ccfbc 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
@@ -24,30 +24,30 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-
/**
- *
+ * Basic Volume implementation that contains a FileSystem and a base path
+ * that should be used within that filesystem.
*/
public class VolumeImpl implements Volume {
protected final FileSystem fs;
protected final String basePath;
-
+
public VolumeImpl(Path path, Configuration conf) throws IOException {
checkNotNull(path);
checkNotNull(conf);
-
+
this.fs = path.getFileSystem(conf);
this.basePath = path.toUri().getPath();
}
-
+
public VolumeImpl(FileSystem fs, String basePath) {
checkNotNull(fs);
checkNotNull(basePath);
-
+
this.fs = fs;
this.basePath = basePath;
}
-
+
@Override
public FileSystem getFileSystem() {
return fs;
@@ -66,20 +66,20 @@ public class VolumeImpl implements Volume {
@Override
public boolean isValidPath(Path p) {
checkNotNull(p);
-
+
return p.toUri().getPath().startsWith(basePath);
}
-
+
@Override
public boolean equals(Object o) {
if (o instanceof VolumeImpl) {
VolumeImpl other = (VolumeImpl) o;
return getFileSystem().equals(other.getFileSystem()) && getBasePath().equals(other.getBasePath());
}
-
+
return false;
}
-
+
@Override
public String toString() {
return getFileSystem() + " " + basePath;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
index 9b8fb98..cbfdb5e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
@@ -160,13 +160,11 @@ public interface VolumeManager {
/**
* Fetch the default Volume
- * @return
*/
public Volume getDefaultVolume();
/**
* Fetch the configured Volumes, excluding the default Volume
- * @return
*/
public Collection<Volume> getVolumes();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index ca5167d..b860f53 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -86,11 +86,12 @@ public class VolumeManagerImpl implements VolumeManager {
inverted.put(volume.getFileSystem(), volume);
}
}
-
+
public static org.apache.accumulo.server.fs.VolumeManager getLocal(String localBasePath) throws IOException {
AccumuloConfiguration accConf = DefaultConfiguration.getDefaultConfiguration();
Volume defaultLocalVolume = VolumeConfiguration.create(FileSystem.getLocal(CachedConfiguration.getInstance()), localBasePath);
-
+
+ // The default volume gets placed in the map, but local filesystem is only used for testing purposes
return new VolumeManagerImpl(Collections.singletonMap(DEFAULT, defaultLocalVolume), defaultLocalVolume, accConf);
}
@@ -112,16 +113,16 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public FSDataOutputStream create(Path path) throws IOException {
checkNotNull(path);
-
+
Volume v = getVolumeByPath(path);
-
+
return v.getFileSystem().create(path);
}
@Override
public FSDataOutputStream create(Path path, boolean overwrite) throws IOException {
checkNotNull(path);
-
+
Volume v = getVolumeByPath(path);
return v.getFileSystem().create(path, overwrite);
@@ -149,7 +150,7 @@ public class VolumeManagerImpl implements VolumeManager {
Volume v = getVolumeByPath(path);
FileSystem fs = v.getFileSystem();
-
+
if (bufferSize == 0) {
fs.getConf().getInt("io.file.buffer.size", 4096);
}
@@ -314,7 +315,7 @@ public class VolumeManagerImpl implements VolumeManager {
return defaultVolume;
}
-
+
log.debug("Could not determine volume for Path '" + path + "' from defined volumes");
} catch (IOException ex) {
throw new RuntimeException(ex);
@@ -404,7 +405,7 @@ public class VolumeManagerImpl implements VolumeManager {
// The "default" Volume for Accumulo (in case no volumes are specified)
for (String volumeUriOrDir : VolumeConfiguration.getVolumeUris(conf)) {
if (volumeUriOrDir.equals(DEFAULT))
- // Cannot re-define the default volume
+ // Cannot re-define the default volume
throw new IllegalArgumentException();
// We require a URI here, fail if it doesn't look like one
@@ -554,7 +555,7 @@ public class VolumeManagerImpl implements VolumeManager {
public Volume getDefaultVolume() {
return defaultVolume;
}
-
+
@Override
public Collection<Volume> getVolumes() {
return volumesByName.values();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index bcfb008..2ef438f 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -52,7 +52,7 @@ import org.apache.log4j.Logger;
public class VolumeUtil {
private static final Logger log = Logger.getLogger(VolumeUtil.class);
-
+
private static boolean isActiveVolume(Path dir) {
// consider relative path as active and take no action
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index 3f33a0e..f0dcd14 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@ -146,9 +146,10 @@ public class ChangeSecret {
private static void updateHdfs(VolumeManager fs, Instance inst, String newInstanceId) throws IOException {
// Need to recreate the instanceId on all of them to keep consistency
for (Volume v : fs.getVolumes()) {
- v.getFileSystem().delete(ServerConstants.getInstanceIdLocation(v), true);
- v.getFileSystem().mkdirs(ServerConstants.getInstanceIdLocation(v));
- v.getFileSystem().create(new Path(ServerConstants.getInstanceIdLocation(v), newInstanceId)).close();
+ final Path instanceId = ServerConstants.getInstanceIdLocation(v);
+ v.getFileSystem().delete(instanceId, true);
+ v.getFileSystem().mkdirs(instanceId);
+ v.getFileSystem().create(new Path(instanceId, newInstanceId)).close();
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/550e5e61/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
index 3b905c9..0013d04 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
@@ -202,7 +202,7 @@ public class VolumeUtilTest {
List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
replacements.add(new Pair<Path,Path>(new Path("file:/foo/v1"), new Path("file:/foo/v8")));
replacements.add(new Pair<Path,Path>(new Path("file:/foo/v2"), new Path("file:/foo/v9")));
-
+
FileType ft = FileType.TABLE;
Assert.assertEquals("file:/foo/v8/tables/+r/root_tablet",