You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2020/03/09 18:02:41 UTC
[accumulo] branch master updated: Simplify some VolumeManager
tooling (#1553)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new dd4a488 Simplify some VolumeManager tooling (#1553)
dd4a488 is described below
commit dd4a488a9077cdd5e90cbff89fbcf7e9147381a7
Author: Christopher Tubbs <ct...@apache.org>
AuthorDate: Mon Mar 9 14:02:28 2020 -0400
Simplify some VolumeManager tooling (#1553)
* Replace getVolumeByPath(p).getFileSystem() with getFileSystemByPath(p)
(no caller used getVolumeByPath without immediately calling getFileSystem)
* Remove redundant null checks for path argument by pushing down into
getFileSystemByPath method
* Remove unneeded NonConfiguredVolume class and test
* Remove unneeded (and unused) VolumeManager.getContentSummary
* Rename create method with overwrite capability to make API more clear
* Retain all exceptions when closing filesystems (as suppressed)
* Simplify/shorten implementations for VolumeManagerImpl methods
* Make VolumeManager AutoCloseable (it already had a close method)
* Fix tests (ZooLockTest -> ZooLockIT, and numerous incorrect usages of
TemporaryFolder)
* Remove ViewFSUtils and corresponding test, since we don't need it
* Tweak timeout of failing IT while testing
---
.../accumulo/core/volume/NonConfiguredVolume.java | 87 ---------
.../accumulo/core/volume/VolumeConfiguration.java | 14 +-
.../apache/accumulo/core/volume/VolumeImpl.java | 12 +-
.../core/volume/NonConfiguredVolumeTest.java | 71 --------
.../miniclusterImpl/MiniAccumuloClusterImpl.java | 7 +-
.../MiniAccumuloConfigImplTest.java | 16 +-
.../accumulo/server/client/BulkImporter.java | 6 +-
.../org/apache/accumulo/server/fs/ViewFSUtils.java | 90 ----------
.../server/fs/VolumeChooserEnvironmentImpl.java | 2 +-
.../apache/accumulo/server/fs/VolumeManager.java | 13 +-
.../accumulo/server/fs/VolumeManagerImpl.java | 199 ++++++++-------------
.../apache/accumulo/server/init/Initialize.java | 57 +++---
.../server/master/recovery/HadoopLogCloser.java | 6 +-
.../apache/accumulo/server/util/ChangeSecret.java | 43 ++---
.../org/apache/accumulo/server/util/FileUtil.java | 18 +-
.../apache/accumulo/server/util/LocalityCheck.java | 2 +-
.../accumulo/server/client/BulkImporterTest.java | 50 +++---
.../apache/accumulo/server/fs/ViewFSUtilsTest.java | 120 -------------
.../accumulo/server/fs/VolumeManagerImplTest.java | 28 ++-
.../apache/accumulo/server/util/FileUtilTest.java | 67 +++----
.../apache/accumulo/master/FateServiceHandler.java | 4 +-
.../java/org/apache/accumulo/master/Master.java | 13 +-
.../master/metrics/ReplicationMetrics.java | 3 +-
.../accumulo/master/recovery/RecoveryManager.java | 4 +-
.../master/replication/ReplicationDriver.java | 2 +-
.../master/tableOps/bulkVer1/BulkImport.java | 2 +-
.../master/tableOps/bulkVer1/CopyFailed.java | 2 +-
.../master/tableOps/bulkVer1/LoadFiles.java | 4 +-
.../master/tableOps/bulkVer2/BulkImportMove.java | 2 +-
.../tableOps/bulkVer2/CleanUpBulkImport.java | 4 +-
.../master/tableOps/bulkVer2/LoadFiles.java | 2 +-
.../master/tableOps/bulkVer2/PrepBulkImport.java | 4 +-
.../accumulo/master/tableOps/create/ChooseDir.java | 4 +-
.../master/tableOps/create/FinishCreateTable.java | 2 +-
.../accumulo/master/tableOps/delete/CleanUp.java | 2 +-
.../tableOps/tableExport/WriteExportFiles.java | 8 +-
.../tableOps/tableImport/CreateImportDir.java | 2 +-
.../tableOps/tableImport/FinishImportTable.java | 2 +-
.../tableImport/ImportPopulateZookeeper.java | 4 +-
.../master/tableOps/tableImport/ImportTable.java | 2 +-
.../tableOps/tableImport/MapImportFileNames.java | 4 +-
.../tableOps/tableImport/MoveExportedFiles.java | 2 +-
.../tableImport/PopulateMetadataTable.java | 2 +-
.../accumulo/master/upgrade/Upgrader9to10.java | 2 +-
.../master/metrics/ReplicationMetricsTest.java | 4 +-
.../master/upgrade/RootFilesUpgradeTest.java | 52 +++---
.../accumulo/tserver/BulkFailedCopyProcessor.java | 6 +-
.../org/apache/accumulo/tserver/FileManager.java | 2 +-
.../org/apache/accumulo/tserver/TabletServer.java | 6 +-
.../tserver/compaction/MajorCompactionRequest.java | 4 +-
.../org/apache/accumulo/tserver/log/LogSorter.java | 2 +-
.../accumulo/tserver/log/RecoveryLogReader.java | 2 +-
.../apache/accumulo/tserver/logger/LogReader.java | 104 +++++------
.../tserver/replication/AccumuloReplicaSystem.java | 14 +-
.../apache/accumulo/tserver/tablet/Compactor.java | 4 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 2 +-
.../tserver/TabletServerSyncCheckTest.java | 16 +-
.../tserver/log/RecoveryLogsReaderTest.java | 24 ++-
.../tserver/log/SortedLogRecoveryTest.java | 30 ++--
.../tserver/log/TestUpgradePathForWALogs.java | 40 ++---
.../vfs/AccumuloReloadingVFSClassLoaderTest.java | 11 +-
.../vfs/AccumuloVFSClassLoaderTest.java | 17 +-
.../start/classloader/vfs/ContextManagerTest.java | 33 ++--
.../org/apache/accumulo/test/DumpConfigIT.java | 21 +--
.../apache/accumulo/test/MultiTableRecoveryIT.java | 2 +-
.../accumulo/test/fate/zookeeper/ZooLockIT.java} | 64 +++----
.../test/performance/scan/CollectTabletStats.java | 6 +-
67 files changed, 466 insertions(+), 990 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
deleted file mode 100644
index db1aaf0..0000000
--- a/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.core.volume;
-
-import org.apache.accumulo.core.conf.Property;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-/**
- * Volume implementation which represents a Volume for which we have a FileSystem but no base path
- * because it is not configured via {@link Property#INSTANCE_VOLUMES}
- *
- * This is useful to handle volumes that have been removed from accumulo.properties but references
- * to these volumes have not been updated. This Volume should never be used to create new files,
- * only to read existing files.
- */
-public class NonConfiguredVolume implements Volume {
- private FileSystem fs;
-
- public NonConfiguredVolume(FileSystem fs) {
- this.fs = fs;
- }
-
- @Override
- public FileSystem getFileSystem() {
- return fs;
- }
-
- @Override
- public String getBasePath() {
- throw new UnsupportedOperationException(
- "No base path known because this Volume isn't configured in accumulo.properties");
- }
-
- @Override
- public Path prefixChild(Path p) {
- throw new UnsupportedOperationException(
- "Cannot prefix path because this Volume isn't configured in accumulo.properties");
- }
-
- @Override
- public Path prefixChild(String p) {
- throw new UnsupportedOperationException(
- "Cannot prefix path because this Volume isn't configured in accumulo.properties");
- }
-
- @Override
- public boolean isValidPath(Path p) {
- throw new UnsupportedOperationException("Cannot determine if path is valid"
- + " because this Volume isn't configured in accumulo.properties");
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof NonConfiguredVolume) {
- NonConfiguredVolume other = (NonConfiguredVolume) o;
- return this.fs.equals(other.getFileSystem());
- }
- return false;
- }
-
- @Override
- public String toString() {
- return "NonConfiguredVolume: " + this.fs;
- }
-
- @Override
- public int hashCode() {
- return NonConfiguredVolume.class.hashCode() ^ this.fs.hashCode();
- }
-}
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
index fc2ab67..7872b7f 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
@@ -38,11 +38,9 @@ public class VolumeConfiguration {
public static Volume getVolume(String path, Configuration conf, AccumuloConfiguration acuconf)
throws IOException {
- requireNonNull(path);
-
- if (path.contains(":")) {
+ if (requireNonNull(path).contains(":")) {
// An absolute path
- return create(new Path(path), conf);
+ return new VolumeImpl(new Path(path), conf);
} else {
// A relative path
return getDefaultVolume(conf, acuconf);
@@ -151,12 +149,4 @@ public class VolumeConfiguration {
dfsDir == null ? Property.INSTANCE_DFS_DIR.getDefaultValue() : dfsDir);
}
- public static <T extends FileSystem> Volume create(T fs, String basePath) {
- return new VolumeImpl(fs, basePath);
- }
-
- public static Volume create(Path path, Configuration conf) throws IOException {
- return new VolumeImpl(path, conf);
- }
-
}
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
index 3eba610..cafa87d 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
@@ -41,20 +41,14 @@ public class VolumeImpl implements Volume {
private final Configuration hadoopConf;
public VolumeImpl(Path path, Configuration conf) throws IOException {
- requireNonNull(path);
- requireNonNull(conf);
-
- this.fs = path.getFileSystem(conf);
+ this.fs = requireNonNull(path).getFileSystem(requireNonNull(conf));
this.basePath = path.toUri().getPath();
this.hadoopConf = conf;
}
public VolumeImpl(FileSystem fs, String basePath) {
- requireNonNull(fs);
- requireNonNull(basePath);
-
- this.fs = fs;
- this.basePath = basePath;
+ this.fs = requireNonNull(fs);
+ this.basePath = requireNonNull(basePath);
this.hadoopConf = fs.getConf();
}
diff --git a/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java b/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
deleted file mode 100644
index 7e7ce97..0000000
--- a/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.core.volume;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.Before;
-import org.junit.Test;
-
-public class NonConfiguredVolumeTest {
-
- private NonConfiguredVolume volume;
-
- @Before
- public void create() throws IOException {
- volume = new NonConfiguredVolume(FileSystem.getLocal(new Configuration()));
- }
-
- @Test
- public void testSameFileSystem() throws IOException {
- assertEquals(FileSystem.getLocal(new Configuration()), volume.getFileSystem());
- }
-
- @Test(expected = UnsupportedOperationException.class)
- public void testGetBasePathFails() {
- volume.getBasePath();
- }
-
- @Test(expected = UnsupportedOperationException.class)
- public void testPrefixChildPath() {
- volume.prefixChild(new Path("/foo"));
- }
-
- @Test(expected = UnsupportedOperationException.class)
- public void testPrefixChildString() {
- volume.prefixChild("/foo");
- }
-
- @Test
- public void testEquality() throws IOException {
- Volume newVolume = new NonConfiguredVolume(FileSystem.getLocal(new Configuration()));
- assertEquals(volume, newVolume);
- }
-
- @Test
- public void testHashCode() throws IOException {
- Volume newVolume = new NonConfiguredVolume(FileSystem.getLocal(new Configuration()));
- assertEquals(volume.hashCode(), newVolume.hashCode());
- }
-}
diff --git a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java
index 9a6a2c7..9451123 100644
--- a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java
@@ -460,13 +460,12 @@ public class MiniAccumuloClusterImpl implements AccumuloCluster {
Configuration hadoopConf = config.getHadoopConfiguration();
ConfigurationCopy cc = new ConfigurationCopy(acuConf);
- VolumeManager fs;
- try {
- fs = VolumeManagerImpl.get(cc, hadoopConf);
+ Path instanceIdPath;
+ try (var fs = VolumeManagerImpl.get(cc, hadoopConf)) {
+ instanceIdPath = ServerUtil.getAccumuloInstanceIdPath(fs);
} catch (IOException e) {
throw new RuntimeException(e);
}
- Path instanceIdPath = ServerUtil.getAccumuloInstanceIdPath(fs);
String instanceIdFromFile =
VolumeManager.getInstanceIDFromHdfs(instanceIdPath, cc, hadoopConf);
diff --git a/minicluster/src/test/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImplTest.java b/minicluster/src/test/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImplTest.java
index 73abaf3..37c49a2 100644
--- a/minicluster/src/test/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImplTest.java
+++ b/minicluster/src/test/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImplTest.java
@@ -22,15 +22,13 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
-import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.minicluster.MemoryUnit;
import org.apache.accumulo.minicluster.ServerType;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -42,14 +40,10 @@ public class MiniAccumuloConfigImplTest {
@SuppressWarnings("deprecation")
private static final Property INSTANCE_DFS_URI = Property.INSTANCE_DFS_URI;
- static TemporaryFolder tempFolder =
+ @Rule
+ public TemporaryFolder tempFolder =
new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- @BeforeClass
- public static void setUp() throws IOException {
- tempFolder.create();
- }
-
@Test
public void testZookeeperPort() {
@@ -98,8 +92,4 @@ public class MiniAccumuloConfigImplTest {
assertEquals(96 * 1024 * 1024L, config.getMemory(ServerType.TABLET_SERVER));
}
- @AfterClass
- public static void tearDown() {
- tempFolder.delete();
- }
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
index a878287..10b0433 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
@@ -89,7 +89,7 @@ public class BulkImporter {
private StopWatch<Timers> timer;
- private static enum Timers {
+ private enum Timers {
EXAMINE_MAP_FILES, QUERY_METADATA, IMPORT_MAP_FILES, SLEEP, TOTAL
}
@@ -344,7 +344,7 @@ public class BulkImporter {
try {
for (Path path : paths) {
- FileSystem fs = vm.getVolumeByPath(path).getFileSystem();
+ FileSystem fs = vm.getFileSystemByPath(path);
mapFileSizes.put(path, fs.getContentSummary(path).getLength());
}
} catch (IOException e) {
@@ -667,7 +667,7 @@ public class BulkImporter {
Collection<ByteSequence> columnFamilies = Collections.emptyList();
String filename = file.toString();
// log.debug(filename + " finding overlapping tablets " + startRow + " -> " + endRow);
- FileSystem fs = vm.getVolumeByPath(file).getFileSystem();
+ FileSystem fs = vm.getFileSystemByPath(file);
try (FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder()
.forFile(filename, fs, fs.getConf(), context.getCryptoService())
.withTableConfiguration(context.getConfiguration()).seekToBeginning().build()) {
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/ViewFSUtils.java b/server/base/src/main/java/org/apache/accumulo/server/fs/ViewFSUtils.java
deleted file mode 100644
index af5a466..0000000
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/ViewFSUtils.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.fs;
-
-import java.io.IOException;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-public class ViewFSUtils {
-
- public static final String VIEWFS_CLASSNAME = "org.apache.hadoop.fs.viewfs.ViewFileSystem";
-
- public static boolean isViewFSSupported() {
- try {
- Class.forName(VIEWFS_CLASSNAME);
- return true;
- } catch (ClassNotFoundException e) {
- return false;
- }
- }
-
- public static boolean isViewFS(Path source, Configuration conf) throws IOException {
- return isViewFS(source.getFileSystem(conf));
- }
-
- public static boolean isViewFS(FileSystem fs) {
- return fs.getClass().getName().equals(VIEWFS_CLASSNAME);
- }
-
- public static Path matchingFileSystem(Path source, Set<String> options, Configuration conf)
- throws IOException {
-
- if (!isViewFS(source, conf))
- throw new IllegalArgumentException("source " + source + " is not view fs");
-
- String sourceUriPath = source.toUri().getPath();
-
- Path match = null;
- int matchPrefixLen = 0;
-
- // find the option with the longest common path prefix
- for (String option : options) {
- Path optionPath = new Path(option);
- if (isViewFS(optionPath, conf)) {
- String optionUriPath = optionPath.toUri().getPath();
-
- int commonPrefixLen = 0;
- for (int i = 0; i < Math.min(sourceUriPath.length(), optionUriPath.length()); i++) {
- if (sourceUriPath.charAt(i) == optionUriPath.charAt(i)) {
- if (sourceUriPath.charAt(i) == '/')
- commonPrefixLen++;
- } else {
- break;
- }
- }
-
- if (commonPrefixLen > matchPrefixLen) {
- matchPrefixLen = commonPrefixLen;
- match = optionPath;
- } else if (match != null && commonPrefixLen == matchPrefixLen
- && optionPath.depth() < match.depth()) {
- // take path with less depth when match prefix length is the same
- match = optionPath;
- }
- }
- }
-
- return match;
- }
-
-}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironmentImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironmentImpl.java
index 84b02a6..ff93eda 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironmentImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironmentImpl.java
@@ -103,7 +103,7 @@ public class VolumeChooserEnvironmentImpl implements VolumeChooserEnvironment {
@Override
public FileSystem getFileSystem(String option) {
- return context.getVolumeManager().getVolumeByPath(new Path(option)).getFileSystem();
+ return context.getVolumeManager().getFileSystemByPath(new Path(option));
}
@Override
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
index 9c0634f..27ccfd3 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
@@ -29,7 +29,6 @@ import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.server.ServerConstants;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -44,7 +43,7 @@ import org.slf4j.LoggerFactory;
* This also concentrates a bunch of meta-operations like waiting for SAFE_MODE, and closing WALs.
* N.B. implementations must be thread safe.
*/
-public interface VolumeManager {
+public interface VolumeManager extends AutoCloseable {
enum FileType {
TABLE(ServerConstants.TABLE_DIR),
@@ -96,13 +95,14 @@ public interface VolumeManager {
}
// close the underlying FileSystems
+ @Override
void close() throws IOException;
// forward to the appropriate FileSystem object
FSDataOutputStream create(Path dest) throws IOException;
- // forward to the appropriate FileSystem object
- FSDataOutputStream create(Path path, boolean b) throws IOException;
+ // forward to the appropriate FileSystem object's create method with overwrite flag set to true
+ FSDataOutputStream overwrite(Path path) throws IOException;
// forward to the appropriate FileSystem object
FSDataOutputStream create(Path path, boolean b, int int1, short int2, long long1)
@@ -128,7 +128,7 @@ public interface VolumeManager {
FileStatus getFileStatus(Path path) throws IOException;
// find the appropriate FileSystem object given a path
- Volume getVolumeByPath(Path path);
+ FileSystem getFileSystemByPath(Path path);
// return the item in options that is in the same volume as source
Path matchingFileSystem(Path source, Set<String> options);
@@ -161,9 +161,6 @@ public interface VolumeManager {
// forward to the appropriate FileSystem object
FileStatus[] globStatus(Path path) throws IOException;
- // forward to the appropriate FileSystem object
- ContentSummary getContentSummary(Path dir) throws IOException;
-
// decide on which of the given locations to create a new file
String choose(VolumeChooserEnvironment env, Set<String> options);
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index d181bad..f14a4dc 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -30,17 +30,17 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Objects;
import java.util.Set;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.volume.NonConfiguredVolume;
import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.volume.VolumeConfiguration;
+import org.apache.accumulo.core.volume.VolumeImpl;
import org.apache.accumulo.server.fs.VolumeChooser.VolumeChooserException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -75,8 +75,7 @@ public class VolumeManagerImpl implements VolumeManager {
this.volumesByName = volumes;
this.defaultVolume = defaultVolume;
// We may have multiple directories used in a single FileSystem (e.g. testing)
- this.volumesByFileSystemUri = HashMultimap.create();
- invertVolumesByFileSystem(volumesByName, volumesByFileSystemUri);
+ this.volumesByFileSystemUri = invertVolumesByFileSystem(volumesByName);
ensureSyncIsEnabled();
// if they supplied a property and we cannot load it, then fail hard
VolumeChooser chooser1;
@@ -95,23 +94,18 @@ public class VolumeManagerImpl implements VolumeManager {
this.hadoopConf = hadoopConf;
}
- private void invertVolumesByFileSystem(Map<String,Volume> forward,
- Multimap<URI,Volume> inverted) {
- for (Volume volume : forward.values()) {
- inverted.put(volume.getFileSystem().getUri(), volume);
- }
+ private Multimap<URI,Volume> invertVolumesByFileSystem(Map<String,Volume> forward) {
+ Multimap<URI,Volume> inverted = HashMultimap.create();
+ forward.values().forEach(volume -> inverted.put(volume.getFileSystem().getUri(), volume));
+ return inverted;
}
- public static org.apache.accumulo.server.fs.VolumeManager getLocal(String localBasePath)
- throws IOException {
+ // for testing only
+ public static VolumeManager getLocalForTesting(String localBasePath) throws IOException {
AccumuloConfiguration accConf = DefaultConfiguration.getInstance();
Configuration hadoopConf = new Configuration();
- Volume defaultLocalVolume =
- VolumeConfiguration.create(FileSystem.getLocal(hadoopConf), localBasePath);
-
- // The default volume gets placed in the map, but local filesystem is only used for testing
- // purposes
- return new VolumeManagerImpl(Collections.singletonMap(DEFAULT, defaultLocalVolume),
+ Volume defaultLocalVolume = new VolumeImpl(FileSystem.getLocal(hadoopConf), localBasePath);
+ return new VolumeManagerImpl(Collections.singletonMap("", defaultLocalVolume),
defaultLocalVolume, accConf, hadoopConf);
}
@@ -122,7 +116,11 @@ public class VolumeManagerImpl implements VolumeManager {
try {
volume.getFileSystem().close();
} catch (IOException e) {
- ex = e;
+ if (ex == null) {
+ ex = e;
+ } else {
+ ex.addSuppressed(e);
+ }
}
}
if (ex != null) {
@@ -132,45 +130,30 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public FSDataOutputStream create(Path path) throws IOException {
- requireNonNull(path);
-
- Volume v = getVolumeByPath(path);
-
- return v.getFileSystem().create(path);
+ return getFileSystemByPath(path).create(path);
}
@Override
- public FSDataOutputStream create(Path path, boolean overwrite) throws IOException {
- requireNonNull(path);
-
- Volume v = getVolumeByPath(path);
-
- return v.getFileSystem().create(path, overwrite);
+ public FSDataOutputStream overwrite(Path path) throws IOException {
+ return getFileSystemByPath(path).create(path, true);
}
private static long correctBlockSize(Configuration conf, long blockSize) {
if (blockSize <= 0)
- blockSize = conf.getLong("dfs.block.size", 67108864);
-
+ blockSize = conf.getLong("dfs.block.size", 67108864); // 64MB default
int checkSum = conf.getInt("io.bytes.per.checksum", 512);
blockSize -= blockSize % checkSum;
- blockSize = Math.max(blockSize, checkSum);
- return blockSize;
+ return Math.max(blockSize, checkSum);
}
private static int correctBufferSize(Configuration conf, int bufferSize) {
- if (bufferSize <= 0)
- bufferSize = conf.getInt("io.file.buffer.size", 4096);
- return bufferSize;
+ return bufferSize <= 0 ? conf.getInt("io.file.buffer.size", 4096) : bufferSize;
}
@Override
public FSDataOutputStream create(Path path, boolean overwrite, int bufferSize, short replication,
long blockSize) throws IOException {
- requireNonNull(path);
-
- Volume v = getVolumeByPath(path);
- FileSystem fs = v.getFileSystem();
+ FileSystem fs = getFileSystemByPath(path);
blockSize = correctBlockSize(fs.getConf(), blockSize);
bufferSize = correctBufferSize(fs.getConf(), bufferSize);
return fs.create(path, overwrite, bufferSize, replication, blockSize);
@@ -178,17 +161,13 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public boolean createNewFile(Path path) throws IOException {
- requireNonNull(path);
-
- Volume v = getVolumeByPath(path);
- return v.getFileSystem().createNewFile(path);
+ return getFileSystemByPath(path).createNewFile(path);
}
@Override
public FSDataOutputStream createSyncable(Path logPath, int bufferSize, short replication,
long blockSize) throws IOException {
- Volume v = getVolumeByPath(logPath);
- FileSystem fs = v.getFileSystem();
+ FileSystem fs = getFileSystemByPath(logPath);
blockSize = correctBlockSize(fs.getConf(), blockSize);
bufferSize = correctBufferSize(fs.getConf(), bufferSize);
EnumSet<CreateFlag> set = EnumSet.of(CreateFlag.SYNC_BLOCK, CreateFlag.CREATE);
@@ -204,22 +183,22 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public boolean delete(Path path) throws IOException {
- return getVolumeByPath(path).getFileSystem().delete(path, false);
+ return getFileSystemByPath(path).delete(path, false);
}
@Override
public boolean deleteRecursively(Path path) throws IOException {
- return getVolumeByPath(path).getFileSystem().delete(path, true);
+ return getFileSystemByPath(path).delete(path, true);
}
protected void ensureSyncIsEnabled() {
- for (Entry<String,Volume> entry : getFileSystems().entrySet()) {
+ for (Entry<String,Volume> entry : volumesByName.entrySet()) {
FileSystem fs = entry.getValue().getFileSystem();
if (fs instanceof DistributedFileSystem) {
// Avoid use of DFSConfigKeys since it's private
- final String DFS_SUPPORT_APPEND = "dfs.support.append",
- DFS_DATANODE_SYNCONCLOSE = "dfs.datanode.synconclose";
+ final String DFS_SUPPORT_APPEND = "dfs.support.append";
+ final String DFS_DATANODE_SYNCONCLOSE = "dfs.datanode.synconclose";
final String ticketMessage = "See ACCUMULO-623 and ACCUMULO-1637 for more details.";
// If either of these parameters are configured to be false, fail.
@@ -249,70 +228,60 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public boolean exists(Path path) throws IOException {
- return getVolumeByPath(path).getFileSystem().exists(path);
+ return getFileSystemByPath(path).exists(path);
}
@Override
public FileStatus getFileStatus(Path path) throws IOException {
- return getVolumeByPath(path).getFileSystem().getFileStatus(path);
+ return getFileSystemByPath(path).getFileStatus(path);
}
@Override
- public Volume getVolumeByPath(Path path) {
- if (path.toString().contains(":")) {
- try {
- FileSystem desiredFs = path.getFileSystem(hadoopConf);
- URI desiredFsUri = desiredFs.getUri();
- Collection<Volume> candidateVolumes = volumesByFileSystemUri.get(desiredFsUri);
- if (candidateVolumes != null) {
- for (Volume candidateVolume : candidateVolumes) {
- if (candidateVolume.isValidPath(path)) {
- return candidateVolume;
- }
- }
- } else {
- log.debug("Could not determine volume for Path: {}", path);
- }
-
- return new NonConfiguredVolume(desiredFs);
- } catch (IOException ex) {
- throw new UncheckedIOException(ex);
- }
+ public FileSystem getFileSystemByPath(Path path) {
+ if (!requireNonNull(path).toString().contains(":")) {
+ return defaultVolume.getFileSystem();
+ }
+ FileSystem desiredFs;
+ try {
+ desiredFs = path.getFileSystem(hadoopConf);
+ } catch (IOException ex) {
+ throw new UncheckedIOException(ex);
+ }
+ URI desiredFsUri = desiredFs.getUri();
+ Collection<Volume> candidateVolumes = volumesByFileSystemUri.get(desiredFsUri);
+ if (candidateVolumes != null) {
+ return candidateVolumes.stream().filter(volume -> volume.isValidPath(path))
+ .map(Volume::getFileSystem).findFirst().orElse(desiredFs);
+ } else {
+ log.debug("Could not determine volume for Path: {}", path);
+ return desiredFs;
}
-
- return defaultVolume;
- }
-
- private Map<String,Volume> getFileSystems() {
- return volumesByName;
}
@Override
public FileStatus[] listStatus(Path path) throws IOException {
- return getVolumeByPath(path).getFileSystem().listStatus(path);
+ return getFileSystemByPath(path).listStatus(path);
}
@Override
public boolean mkdirs(Path path) throws IOException {
- return getVolumeByPath(path).getFileSystem().mkdirs(path);
+ return getFileSystemByPath(path).mkdirs(path);
}
@Override
public boolean mkdirs(Path path, FsPermission permission) throws IOException {
- return getVolumeByPath(path).getFileSystem().mkdirs(path, permission);
+ return getFileSystemByPath(path).mkdirs(path, permission);
}
@Override
public FSDataInputStream open(Path path) throws IOException {
- return getVolumeByPath(path).getFileSystem().open(path);
+ return getFileSystemByPath(path).open(path);
}
@Override
public boolean rename(Path path, Path newPath) throws IOException {
- Volume srcVolume = getVolumeByPath(path);
- Volume destVolume = getVolumeByPath(newPath);
- FileSystem source = srcVolume.getFileSystem();
- FileSystem dest = destVolume.getFileSystem();
+ FileSystem source = getFileSystemByPath(path);
+ FileSystem dest = getFileSystemByPath(newPath);
if (source != dest) {
throw new UnsupportedOperationException(
"Cannot rename files across volumes: " + path + " -> " + newPath);
@@ -322,35 +291,31 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public boolean moveToTrash(Path path) throws IOException {
- FileSystem fs = getVolumeByPath(path).getFileSystem();
+ FileSystem fs = getFileSystemByPath(path);
Trash trash = new Trash(fs, fs.getConf());
return trash.moveToTrash(path);
}
@Override
public short getDefaultReplication(Path path) {
- Volume v = getVolumeByPath(path);
- return v.getFileSystem().getDefaultReplication(path);
+ return getFileSystemByPath(path).getDefaultReplication(path);
}
- private static final String DEFAULT = "";
-
public static VolumeManager get(AccumuloConfiguration conf, final Configuration hadoopConf)
throws IOException {
final Map<String,Volume> volumes = new HashMap<>();
// The "default" Volume for Accumulo (in case no volumes are specified)
for (String volumeUriOrDir : VolumeConfiguration.getVolumeUris(conf, hadoopConf)) {
- if (volumeUriOrDir.equals(DEFAULT))
- throw new IllegalArgumentException("Cannot re-define the default volume");
+ if (volumeUriOrDir.isBlank())
+ throw new IllegalArgumentException("Empty volume specified in configuration");
if (volumeUriOrDir.startsWith("viewfs"))
throw new IllegalArgumentException("Cannot use viewfs as a volume");
// We require a URI here, fail if it doesn't look like one
if (volumeUriOrDir.contains(":")) {
- volumes.put(volumeUriOrDir,
- VolumeConfiguration.create(new Path(volumeUriOrDir), hadoopConf));
+ volumes.put(volumeUriOrDir, new VolumeImpl(new Path(volumeUriOrDir), hadoopConf));
} else {
throw new IllegalArgumentException("Expected fully qualified URI for "
+ Property.INSTANCE_VOLUMES.getKey() + " got " + volumeUriOrDir);
@@ -363,14 +328,11 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public boolean isReady() throws IOException {
- for (Volume volume : getFileSystems().values()) {
+ for (Volume volume : volumesByName.values()) {
final FileSystem fs = volume.getFileSystem();
-
if (!(fs instanceof DistributedFileSystem))
continue;
-
final DistributedFileSystem dfs = (DistributedFileSystem) fs;
-
// Returns true when safemode is on
if (dfs.setSafeMode(SafeModeAction.SAFEMODE_GET)) {
return false;
@@ -381,35 +343,17 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public FileStatus[] globStatus(Path pathPattern) throws IOException {
- return getVolumeByPath(pathPattern).getFileSystem().globStatus(pathPattern);
+ return getFileSystemByPath(pathPattern).globStatus(pathPattern);
}
@Override
public Path matchingFileSystem(Path source, Set<String> options) {
- try {
- if (ViewFSUtils.isViewFS(source, hadoopConf)) {
- return ViewFSUtils.matchingFileSystem(source, options, hadoopConf);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- URI uri1 = source.toUri();
- for (String option : options) {
- URI uri3 = URI.create(option);
- if (uri1.getScheme().equals(uri3.getScheme())) {
- String a1 = uri1.getAuthority();
- String a2 = uri3.getAuthority();
- if ((a1 == null && a2 == null) || (a1 != null && a1.equals(a2)))
- return new Path(option);
- }
- }
- return null;
- }
-
- @Override
- public ContentSummary getContentSummary(Path dir) throws IOException {
- return getVolumeByPath(dir).getFileSystem().getContentSummary(dir);
+ URI sourceUri = source.toUri();
+ return options.stream().filter(opt -> {
+ URI optUri = URI.create(opt);
+ return sourceUri.getScheme().equals(optUri.getScheme())
+ && Objects.equals(sourceUri.getAuthority(), optUri.getAuthority());
+ }).map((String opt) -> new Path(opt)).findFirst().orElse(null);
}
@Override
@@ -421,14 +365,12 @@ public class VolumeManagerImpl implements VolumeManager {
+ "', or one of its delegates returned a volume not in the set of options provided";
throw new VolumeChooserException(msg);
}
-
return choice;
}
@Override
public Set<String> choosable(VolumeChooserEnvironment env, Set<String> options) {
final Set<String> choices = chooser.choosable(env, options);
-
for (String choice : choices) {
if (!options.contains(choice)) {
String msg = "The configured volume chooser, '" + chooser.getClass()
@@ -436,7 +378,6 @@ public class VolumeManagerImpl implements VolumeManager {
throw new VolumeChooserException(msg);
}
}
-
return choices;
}
@@ -445,7 +386,7 @@ public class VolumeManagerImpl implements VolumeManager {
// the assumption is all filesystems support sync/flush except
// for HDFS erasure coding. not checking hdfs config options
// since that's already checked in ensureSyncIsEnabled()
- FileSystem fs = getVolumeByPath(path).getFileSystem();
+ FileSystem fs = getFileSystemByPath(path);
if (fs instanceof DistributedFileSystem) {
DistributedFileSystem dfs = (DistributedFileSystem) fs;
try {
diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index 1dc0e74..9623c24 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@ -574,7 +574,7 @@ public class Initialize implements KeywordExecutable {
for (Tablet tablet : tablets) {
createEntriesForTablet(sorted, tablet);
}
- FileSystem fs = volmanager.getVolumeByPath(new Path(fileName)).getFileSystem();
+ FileSystem fs = volmanager.getFileSystemByPath(new Path(fileName));
CryptoService cs = CryptoServiceFactory.newInstance(conf, ClassloaderType.ACCUMULO);
@@ -981,38 +981,41 @@ public class Initialize implements KeywordExecutable {
SecurityUtil.serverLogin(siteConfig);
Configuration hadoopConfig = new Configuration();
- VolumeManager fs = VolumeManagerImpl.get(siteConfig, hadoopConfig);
-
- if (opts.resetSecurity) {
- log.info("Resetting security on accumulo.");
- try (ServerContext context = new ServerContext(siteConfig)) {
- if (isInitialized(fs, siteConfig, hadoopConfig)) {
- if (!opts.forceResetSecurity) {
- ConsoleReader c = getConsoleReader();
- String userEnteredName = c.readLine("WARNING: This will remove all"
- + " users from Accumulo! If you wish to proceed enter the instance" + " name: ");
- if (userEnteredName != null && !context.getInstanceName().equals(userEnteredName)) {
- log.error("Aborted reset security: Instance name did not match current instance.");
- return;
+ try (var fs = VolumeManagerImpl.get(siteConfig, hadoopConfig)) {
+
+ if (opts.resetSecurity) {
+ log.info("Resetting security on accumulo.");
+ try (ServerContext context = new ServerContext(siteConfig)) {
+ if (isInitialized(fs, siteConfig, hadoopConfig)) {
+ if (!opts.forceResetSecurity) {
+ ConsoleReader c = getConsoleReader();
+ String userEnteredName = c.readLine("WARNING: This will remove all"
+ + " users from Accumulo! If you wish to proceed enter the instance"
+ + " name: ");
+ if (userEnteredName != null && !context.getInstanceName().equals(userEnteredName)) {
+ log.error(
+ "Aborted reset security: Instance name did not match current instance.");
+ return;
+ }
}
- }
- final String rootUser = getRootUserName(siteConfig, opts);
- opts.rootpass = getRootPassword(siteConfig, opts, rootUser);
- initSecurity(context, opts, rootUser);
- } else {
- log.error("FATAL: Attempted to reset security on accumulo before it was initialized");
+ final String rootUser = getRootUserName(siteConfig, opts);
+ opts.rootpass = getRootPassword(siteConfig, opts, rootUser);
+ initSecurity(context, opts, rootUser);
+ } else {
+ log.error("FATAL: Attempted to reset security on accumulo before it was initialized");
+ }
}
}
- }
- if (opts.addVolumes) {
- addVolumes(fs, siteConfig, hadoopConfig);
- }
+ if (opts.addVolumes) {
+ addVolumes(fs, siteConfig, hadoopConfig);
+ }
- if (!opts.resetSecurity && !opts.addVolumes) {
- if (!doInit(siteConfig, opts, hadoopConfig, fs)) {
- System.exit(-1);
+ if (!opts.resetSecurity && !opts.addVolumes) {
+ if (!doInit(siteConfig, opts, hadoopConfig, fs)) {
+ System.exit(-1);
+ }
}
}
} catch (Exception e) {
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
index ce3c749..74a4407 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
@@ -23,13 +23,13 @@ import java.io.IOException;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.server.fs.ViewFSUtils;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.fs.viewfs.ViewFileSystem;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,10 +41,10 @@ public class HadoopLogCloser implements LogCloser {
@Override
public long close(AccumuloConfiguration conf, Configuration hadoopConf, VolumeManager fs,
Path source) throws IOException {
- FileSystem ns = fs.getVolumeByPath(source).getFileSystem();
+ FileSystem ns = fs.getFileSystemByPath(source);
// if path points to a viewfs path, then resolve to underlying filesystem
- if (ViewFSUtils.isViewFS(ns)) {
+ if (ns instanceof ViewFileSystem) {
Path newSource = ns.resolvePath(source);
if (!newSource.equals(source) && newSource.toUri().getScheme() != null) {
ns = newSource.getFileSystem(hadoopConf);
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index 04cb35e..e05b252 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@ -63,28 +63,29 @@ public class ChangeSecret {
public static void main(String[] args) throws Exception {
var siteConfig = SiteConfiguration.auto();
- VolumeManager fs = VolumeManagerImpl.get(siteConfig, new Configuration());
- verifyHdfsWritePermission(fs);
-
- Opts opts = new Opts();
- List<String> argsList = new ArrayList<>(args.length + 2);
- argsList.add("--old");
- argsList.add("--new");
- argsList.addAll(Arrays.asList(args));
- try (TraceScope clientSpan =
- opts.parseArgsAndTrace(ChangeSecret.class.getName(), argsList.toArray(new String[0]))) {
-
- ServerContext context = opts.getServerContext();
- verifyAccumuloIsDown(context, opts.oldPass);
-
- final String newInstanceId = UUID.randomUUID().toString();
- updateHdfs(fs, newInstanceId);
- rewriteZooKeeperInstance(context, newInstanceId, opts.oldPass, opts.newPass);
- if (opts.oldPass != null) {
- deleteInstance(context, opts.oldPass);
+ try (var fs = VolumeManagerImpl.get(siteConfig, new Configuration())) {
+ verifyHdfsWritePermission(fs);
+
+ Opts opts = new Opts();
+ List<String> argsList = new ArrayList<>(args.length + 2);
+ argsList.add("--old");
+ argsList.add("--new");
+ argsList.addAll(Arrays.asList(args));
+ try (TraceScope clientSpan =
+ opts.parseArgsAndTrace(ChangeSecret.class.getName(), argsList.toArray(new String[0]))) {
+
+ ServerContext context = opts.getServerContext();
+ verifyAccumuloIsDown(context, opts.oldPass);
+
+ final String newInstanceId = UUID.randomUUID().toString();
+ updateHdfs(fs, newInstanceId);
+ rewriteZooKeeperInstance(context, newInstanceId, opts.oldPass, opts.newPass);
+ if (opts.oldPass != null) {
+ deleteInstance(context, opts.oldPass);
+ }
+ System.out.println("New instance id is " + newInstanceId);
+ System.out.println("Be sure to put your new secret in accumulo.properties");
}
- System.out.println("New instance id is " + newInstanceId);
- System.out.println("Be sure to put your new secret in accumulo.properties");
}
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
index bf9b401..47499f2 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
@@ -47,7 +47,6 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.util.LocalityGroupUtil;
-import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.hadoop.conf.Configuration;
@@ -141,8 +140,7 @@ public class FileUtil {
new TabletFile(new Path(String.format("%s/%04d.%s", newDir, count++, RFile.EXTENSION)));
outFiles.add(newMapFile);
- FileSystem ns =
- context.getVolumeManager().getVolumeByPath(newMapFile.getPath()).getFileSystem();
+ FileSystem ns = context.getVolumeManager().getFileSystemByPath(newMapFile.getPath());
FileSKVWriter writer = new RFileOperations().newWriterBuilder()
.forFile(newMapFile.getPathStr(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(acuConf).build();
@@ -152,7 +150,7 @@ public class FileUtil {
FileSKVIterator reader = null;
try {
for (TabletFile file : inFiles) {
- ns = context.getVolumeManager().getVolumeByPath(file.getPath()).getFileSystem();
+ ns = context.getVolumeManager().getFileSystemByPath(file.getPath());
reader = FileOperations.getInstance().newIndexReaderBuilder()
.forFile(file.getPathStr(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(acuConf).build();
@@ -415,8 +413,8 @@ public class FileUtil {
}
if (tmpDir != null) {
- Volume v = fs.getVolumeByPath(tmpDir);
- if (v.getFileSystem().exists(tmpDir)) {
+ FileSystem actualFs = fs.getFileSystemByPath(tmpDir);
+ if (actualFs.exists(tmpDir)) {
fs.deleteRecursively(tmpDir);
return;
}
@@ -436,7 +434,7 @@ public class FileUtil {
// count the total number of index entries
for (TabletFile file : mapFiles) {
FileSKVIterator reader = null;
- FileSystem ns = context.getVolumeManager().getVolumeByPath(file.getPath()).getFileSystem();
+ FileSystem ns = context.getVolumeManager().getFileSystemByPath(file.getPath());
try {
if (useIndex)
reader = FileOperations.getInstance().newIndexReaderBuilder()
@@ -492,7 +490,7 @@ public class FileUtil {
for (TabletFile mapfile : mapfiles) {
FileSKVIterator reader = null;
- FileSystem ns = context.getVolumeManager().getVolumeByPath(mapfile.getPath()).getFileSystem();
+ FileSystem ns = context.getVolumeManager().getFileSystemByPath(mapfile.getPath());
try {
reader = FileOperations.getInstance().newReaderBuilder()
.forFile(mapfile.getPathStr(), ns, ns.getConf(), context.getCryptoService())
@@ -531,7 +529,7 @@ public class FileUtil {
Key lastKey = null;
for (TabletFile file : mapFiles) {
- FileSystem ns = context.getVolumeManager().getVolumeByPath(file.getPath()).getFileSystem();
+ FileSystem ns = context.getVolumeManager().getFileSystemByPath(file.getPath());
FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder()
.forFile(file.getPathStr(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(context.getConfiguration()).seekToBeginning().build();
@@ -562,7 +560,7 @@ public class FileUtil {
public static Map<KeyExtent,Long> estimateSizes(ServerContext context, Path mapFile,
long fileSize, List<KeyExtent> extents) throws IOException {
- FileSystem ns = context.getVolumeManager().getVolumeByPath(mapFile).getFileSystem();
+ FileSystem ns = context.getVolumeManager().getFileSystemByPath(mapFile);
return BulkImport.estimateSizes(context.getConfiguration(), mapFile, fileSize, extents, ns,
null, context.getCryptoService());
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
index 985152f..f0c4bba 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
@@ -95,7 +95,7 @@ public class LocalityCheck {
}
for (String file : files) {
Path filePath = new Path(file);
- FileSystem ns = fs.getVolumeByPath(filePath).getFileSystem();
+ FileSystem ns = fs.getFileSystemByPath(filePath);
FileStatus fileStatus = ns.getFileStatus(filePath);
BlockLocation[] fileBlockLocations =
ns.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
diff --git a/server/base/src/test/java/org/apache/accumulo/server/client/BulkImporterTest.java b/server/base/src/test/java/org/apache/accumulo/server/client/BulkImporterTest.java
index e8b15d5..98debc1 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/client/BulkImporterTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/client/BulkImporterTest.java
@@ -42,7 +42,6 @@ import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVWriter;
import org.apache.accumulo.server.ServerContext;
-import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -146,30 +145,31 @@ public class BulkImporterTest {
writer.append(new Key("iterator", "cf", "cq5"), empty);
writer.append(new Key("xyzzy", "cf", "cq"), empty);
writer.close();
- VolumeManager vm = VolumeManagerImpl.get(context.getConfiguration(), new Configuration());
- List<TabletLocation> overlaps =
- BulkImporter.findOverlappingTablets(context, vm, locator, new Path(file));
- assertEquals(5, overlaps.size());
- Collections.sort(overlaps);
- assertEquals(new KeyExtent(tableId, new Text("a"), null), overlaps.get(0).tablet_extent);
- assertEquals(new KeyExtent(tableId, new Text("d"), new Text("cm")),
- overlaps.get(1).tablet_extent);
- assertEquals(new KeyExtent(tableId, new Text("dm"), new Text("d")),
- overlaps.get(2).tablet_extent);
- assertEquals(new KeyExtent(tableId, new Text("j"), new Text("i")),
- overlaps.get(3).tablet_extent);
- assertEquals(new KeyExtent(tableId, null, new Text("l")), overlaps.get(4).tablet_extent);
-
- List<TabletLocation> overlaps2 = BulkImporter.findOverlappingTablets(context, vm, locator,
- new Path(file), new KeyExtent(tableId, new Text("h"), new Text("b")));
- assertEquals(3, overlaps2.size());
- assertEquals(new KeyExtent(tableId, new Text("d"), new Text("cm")),
- overlaps2.get(0).tablet_extent);
- assertEquals(new KeyExtent(tableId, new Text("dm"), new Text("d")),
- overlaps2.get(1).tablet_extent);
- assertEquals(new KeyExtent(tableId, new Text("j"), new Text("i")),
- overlaps2.get(2).tablet_extent);
- assertEquals(locator.invalidated, 1);
+ try (var vm = VolumeManagerImpl.get(context.getConfiguration(), new Configuration())) {
+ List<TabletLocation> overlaps =
+ BulkImporter.findOverlappingTablets(context, vm, locator, new Path(file));
+ assertEquals(5, overlaps.size());
+ Collections.sort(overlaps);
+ assertEquals(new KeyExtent(tableId, new Text("a"), null), overlaps.get(0).tablet_extent);
+ assertEquals(new KeyExtent(tableId, new Text("d"), new Text("cm")),
+ overlaps.get(1).tablet_extent);
+ assertEquals(new KeyExtent(tableId, new Text("dm"), new Text("d")),
+ overlaps.get(2).tablet_extent);
+ assertEquals(new KeyExtent(tableId, new Text("j"), new Text("i")),
+ overlaps.get(3).tablet_extent);
+ assertEquals(new KeyExtent(tableId, null, new Text("l")), overlaps.get(4).tablet_extent);
+
+ List<TabletLocation> overlaps2 = BulkImporter.findOverlappingTablets(context, vm, locator,
+ new Path(file), new KeyExtent(tableId, new Text("h"), new Text("b")));
+ assertEquals(3, overlaps2.size());
+ assertEquals(new KeyExtent(tableId, new Text("d"), new Text("cm")),
+ overlaps2.get(0).tablet_extent);
+ assertEquals(new KeyExtent(tableId, new Text("dm"), new Text("d")),
+ overlaps2.get(1).tablet_extent);
+ assertEquals(new KeyExtent(tableId, new Text("j"), new Text("i")),
+ overlaps2.get(2).tablet_extent);
+ assertEquals(locator.invalidated, 1);
+ }
}
@Test
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/ViewFSUtilsTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/ViewFSUtilsTest.java
deleted file mode 100644
index 0826c5c..0000000
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/ViewFSUtilsTest.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.fs;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.junit.Test;
-
-public class ViewFSUtilsTest {
-
- private Set<String> shuffle(String... inputs) {
- List<String> inputsArray = Arrays.asList(inputs);
- // shuffle will modify array, because ArrayList implements RandomAccess
- Collections.shuffle(inputsArray);
- // preserve the shuffled array as an insertion-ordered set
- return inputsArray.stream().collect(Collectors.toCollection(LinkedHashSet::new));
- }
-
- @Test
- public void testDisjointMountPoints() throws IllegalArgumentException, IOException {
- if (ViewFSUtils.isViewFSSupported()) {
- Configuration conf = new Configuration(false);
- conf.set("fs.viewfs.mounttable.default.link./ns", "file:///tmp/ns");
- conf.set("fs.viewfs.mounttable.default.link./ns1", "file:///tmp/ns1");
- conf.set("fs.viewfs.mounttable.default.link./ns2", "file:///tmp/ns2");
- conf.set("fs.viewfs.mounttable.default.link./ns22", "file:///tmp/ns22");
-
- Set<String> tablesDirs1 =
- shuffle("viewfs:///ns1/accumulo/tables", "viewfs:///ns2/accumulo/tables",
- "viewfs:///ns22/accumulo/tables", "viewfs:///ns/accumulo/tables");
- Set<String> tablesDirs2 =
- shuffle("viewfs:/ns1/accumulo/tables", "viewfs:/ns2/accumulo/tables",
- "viewfs:/ns22/accumulo/tables", "viewfs:/ns/accumulo/tables");
-
- for (String ns : Arrays.asList("ns1", "ns2", "ns22", "ns")) {
- Path match = ViewFSUtils.matchingFileSystem(new Path("viewfs:/" + ns + "/bulk_import_01"),
- tablesDirs2, conf);
- assertEquals(new Path("viewfs:/" + ns + "/accumulo/tables"), match);
-
- match = ViewFSUtils.matchingFileSystem(new Path("viewfs:///" + ns + "/bulk_import_01"),
- tablesDirs1, conf);
- assertEquals(new Path("viewfs:/" + ns + "/accumulo/tables"), match);
-
- match = ViewFSUtils.matchingFileSystem(new Path("viewfs:/" + ns + "/bulk_import_01"),
- tablesDirs2, conf);
- assertEquals(new Path("viewfs:/" + ns + "/accumulo/tables"), match);
-
- match = ViewFSUtils.matchingFileSystem(new Path("viewfs:///" + ns + "/bulk_import_01"),
- tablesDirs1, conf);
- assertEquals(new Path("viewfs:/" + ns + "/accumulo/tables"), match);
- }
- }
- }
-
- @Test
- public void testOverlappingMountPoints() throws IllegalArgumentException, IOException {
- if (ViewFSUtils.isViewFSSupported()) {
- Configuration conf = new Configuration(false);
- conf.set("fs.viewfs.mounttable.default.link./", "file:///tmp/0");
- conf.set("fs.viewfs.mounttable.default.link./ns1", "file:///tmp/1");
- conf.set("fs.viewfs.mounttable.default.link./ns1/A", "file:///tmp/2");
- conf.set("fs.viewfs.mounttable.default.link./ns1/AA", "file:///tmp/3");
- conf.set("fs.viewfs.mounttable.default.link./ns1/C", "file:///tmp/3");
- conf.set("fs.viewfs.mounttable.default.link./ns2", "file:///tmp/3");
-
- Set<String> tablesDirs1 =
- shuffle("viewfs:///ns1/accumulo/tables", "viewfs:///ns1/A/accumulo/tables",
- "viewfs:///ns1/AA/accumulo/tables", "viewfs:///ns1/C/accumulo/tables",
- "viewfs:///ns2/accumulo/tables", "viewfs:///accumulo/tables");
- Set<String> tablesDirs2 =
- shuffle("viewfs:/ns1/accumulo/tables", "viewfs:/ns1/A/accumulo/tables",
- "viewfs:/ns1/AA/accumulo/tables", "viewfs:/ns1/C/accumulo/tables",
- "viewfs:/ns2/accumulo/tables", "viewfs:/accumulo/tables");
-
- for (String ns : Arrays.asList("", "/ns1", "/ns1/A", "/ns1/AA", "/ns1/C", "/ns2")) {
- Path match = ViewFSUtils.matchingFileSystem(new Path("viewfs:" + ns + "/bulk_import_01"),
- tablesDirs2, conf);
- assertEquals(new Path("viewfs:" + ns + "/accumulo/tables"), match);
-
- match = ViewFSUtils.matchingFileSystem(new Path("viewfs://" + ns + "/bulk_import_01"),
- tablesDirs1, conf);
- assertEquals(new Path("viewfs:" + ns + "/accumulo/tables"), match);
-
- match = ViewFSUtils.matchingFileSystem(new Path("viewfs:" + ns + "/bulk_import_01"),
- tablesDirs2, conf);
- assertEquals(new Path("viewfs:" + ns + "/accumulo/tables"), match);
-
- match = ViewFSUtils.matchingFileSystem(new Path("viewfs://" + ns + "/bulk_import_01"),
- tablesDirs1, conf);
- assertEquals(new Path("viewfs:" + ns + "/accumulo/tables"), match);
- }
- }
- }
-}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
index db93f0f..1e1d084 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
@@ -18,7 +18,7 @@
*/
package org.apache.accumulo.server.fs;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.util.Arrays;
import java.util.List;
@@ -28,24 +28,17 @@ import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.TableId;
import org.apache.hadoop.conf.Configuration;
-import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
public class VolumeManagerImplTest {
- protected VolumeManager fs;
private Configuration hadoopConf = new Configuration();
@Rule
public ExpectedException thrown = ExpectedException.none();
- @Before
- public void setup() throws Exception {
- fs = VolumeManagerImpl.getLocal(System.getProperty("user.dir"));
- }
-
@Test
public void invalidChooserConfigured() throws Exception {
List<String> volumes = Arrays.asList("file://one/", "file://two/", "file://three/");
@@ -55,7 +48,9 @@ public class VolumeManagerImplTest {
conf.set(Property.GENERAL_VOLUME_CHOOSER,
"org.apache.accumulo.server.fs.ChooserThatDoesntExist");
thrown.expect(RuntimeException.class);
- VolumeManagerImpl.get(conf, hadoopConf);
+ try (var vm = VolumeManagerImpl.get(conf, hadoopConf)) {
+ fail("shouldn't reach here " + vm);
+ }
}
@Test
@@ -63,7 +58,9 @@ public class VolumeManagerImplTest {
ConfigurationCopy conf = new ConfigurationCopy();
conf.set(Property.INSTANCE_VOLUMES, "viewfs://dummy");
thrown.expect(IllegalArgumentException.class);
- VolumeManagerImpl.get(conf, hadoopConf);
+ try (var vm = VolumeManagerImpl.get(conf, hadoopConf)) {
+ fail("shouldn't reach here " + vm);
+ }
}
public static class WrongVolumeChooser implements VolumeChooser {
@@ -90,10 +87,11 @@ public class VolumeManagerImplTest {
conf.set(Property.INSTANCE_VOLUMES, String.join(",", volumes));
conf.set(Property.GENERAL_VOLUME_CHOOSER, WrongVolumeChooser.class.getName());
thrown.expect(RuntimeException.class);
- VolumeManager vm = VolumeManagerImpl.get(conf, hadoopConf);
- VolumeChooserEnvironment chooserEnv =
- new VolumeChooserEnvironmentImpl(TableId.of("sometable"), null, null);
- String choice = vm.choose(chooserEnv, volumes);
- assertTrue("shouldn't see invalid options from misbehaving chooser.", volumes.contains(choice));
+ try (var vm = VolumeManagerImpl.get(conf, hadoopConf)) {
+ VolumeChooserEnvironment chooserEnv =
+ new VolumeChooserEnvironmentImpl(TableId.of("sometable"), null, null);
+ vm.choose(chooserEnv, volumes);
+ fail("shouldn't reach here");
+ }
}
}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/FileUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/FileUtilTest.java
index aeab364..5380128 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/util/FileUtilTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/util/FileUtilTest.java
@@ -27,7 +27,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.hadoop.fs.Path;
import org.junit.Before;
@@ -67,9 +66,9 @@ public class FileUtilTest {
HashMap<Property,String> testProps = new HashMap<>();
testProps.put(INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath());
- VolumeManager fs = VolumeManagerImpl.getLocal(accumuloDir.getAbsolutePath());
-
- FileUtil.cleanupIndexOp(tmpPath1, fs, new ArrayList<>());
+ try (var fs = VolumeManagerImpl.getLocalForTesting(accumuloDir.getAbsolutePath())) {
+ FileUtil.cleanupIndexOp(tmpPath1, fs, new ArrayList<>());
+ }
assertFalse("Expected " + tmp1 + " to be cleaned up but it wasn't", tmp1.exists());
}
@@ -93,15 +92,12 @@ public class FileUtilTest {
HashMap<Property,String> testProps = new HashMap<>();
testProps.put(Property.INSTANCE_VOLUMES, v1.toURI() + "," + v2.toURI());
- VolumeManager fs = VolumeManagerImpl.getLocal(accumuloDir.getAbsolutePath());
-
- FileUtil.cleanupIndexOp(tmpPath1, fs, new ArrayList<>());
-
- assertFalse("Expected " + tmp1 + " to be cleaned up but it wasn't", tmp1.exists());
-
- FileUtil.cleanupIndexOp(tmpPath2, fs, new ArrayList<>());
-
- assertFalse("Expected " + tmp2 + " to be cleaned up but it wasn't", tmp2.exists());
+ try (var fs = VolumeManagerImpl.getLocalForTesting(accumuloDir.getAbsolutePath())) {
+ FileUtil.cleanupIndexOp(tmpPath1, fs, new ArrayList<>());
+ assertFalse("Expected " + tmp1 + " to be cleaned up but it wasn't", tmp1.exists());
+ FileUtil.cleanupIndexOp(tmpPath2, fs, new ArrayList<>());
+ assertFalse("Expected " + tmp2 + " to be cleaned up but it wasn't", tmp2.exists());
+ }
}
@Test
@@ -126,15 +122,12 @@ public class FileUtilTest {
HashMap<Property,String> testProps = new HashMap<>();
testProps.put(Property.INSTANCE_VOLUMES, v1.toURI() + "," + v2.toURI());
- VolumeManager fs = VolumeManagerImpl.getLocal(accumuloDir.getAbsolutePath());
-
- FileUtil.cleanupIndexOp(tmpPath1, fs, new ArrayList<>());
-
- assertFalse("Expected " + tmp1 + " to be cleaned up but it wasn't", tmp1.exists());
-
- FileUtil.cleanupIndexOp(tmpPath2, fs, new ArrayList<>());
-
- assertFalse("Expected " + tmp2 + " to be cleaned up but it wasn't", tmp2.exists());
+ try (var fs = VolumeManagerImpl.getLocalForTesting(accumuloDir.getAbsolutePath())) {
+ FileUtil.cleanupIndexOp(tmpPath1, fs, new ArrayList<>());
+ assertFalse("Expected " + tmp1 + " to be cleaned up but it wasn't", tmp1.exists());
+ FileUtil.cleanupIndexOp(tmpPath2, fs, new ArrayList<>());
+ assertFalse("Expected " + tmp2 + " to be cleaned up but it wasn't", tmp2.exists());
+ }
}
@Test
@@ -153,15 +146,12 @@ public class FileUtilTest {
HashMap<Property,String> testProps = new HashMap<>();
testProps.put(Property.INSTANCE_VOLUMES, v1.toURI() + "," + v2.toURI());
- VolumeManager fs = VolumeManagerImpl.getLocal(accumuloDir.getAbsolutePath());
-
- FileUtil.cleanupIndexOp(tmpPath1, fs, new ArrayList<>());
-
- assertFalse("Expected " + tmp1 + " to be cleaned up but it wasn't", tmp1.exists());
-
- FileUtil.cleanupIndexOp(tmpPath2, fs, new ArrayList<>());
-
- assertFalse("Expected " + tmp2 + " to be cleaned up but it wasn't", tmp2.exists());
+ try (var fs = VolumeManagerImpl.getLocalForTesting(accumuloDir.getAbsolutePath())) {
+ FileUtil.cleanupIndexOp(tmpPath1, fs, new ArrayList<>());
+ assertFalse("Expected " + tmp1 + " to be cleaned up but it wasn't", tmp1.exists());
+ FileUtil.cleanupIndexOp(tmpPath2, fs, new ArrayList<>());
+ assertFalse("Expected " + tmp2 + " to be cleaned up but it wasn't", tmp2.exists());
+ }
}
@Test
@@ -183,14 +173,11 @@ public class FileUtilTest {
HashMap<Property,String> testProps = new HashMap<>();
testProps.put(Property.INSTANCE_VOLUMES, v1.toURI() + "," + v2.toURI());
- VolumeManager fs = VolumeManagerImpl.getLocal(accumuloDir.getAbsolutePath());
-
- FileUtil.cleanupIndexOp(tmpPath1, fs, new ArrayList<>());
-
- assertFalse("Expected " + tmp1 + " to be cleaned up but it wasn't", tmp1.exists());
-
- FileUtil.cleanupIndexOp(tmpPath2, fs, new ArrayList<>());
-
- assertFalse("Expected " + tmp2 + " to be cleaned up but it wasn't", tmp2.exists());
+ try (var fs = VolumeManagerImpl.getLocalForTesting(accumuloDir.getAbsolutePath())) {
+ FileUtil.cleanupIndexOp(tmpPath1, fs, new ArrayList<>());
+ assertFalse("Expected " + tmp1 + " to be cleaned up but it wasn't", tmp1.exists());
+ FileUtil.cleanupIndexOp(tmpPath2, fs, new ArrayList<>());
+ assertFalse("Expected " + tmp2 + " to be cleaned up but it wasn't", tmp2.exists());
+ }
}
}
diff --git a/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java b/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
index db8931a..0373b71 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
@@ -757,7 +757,7 @@ class FateServiceHandler implements FateService.Iface {
* failure and/or FateServiceHandler retries.
*/
private void removeAndCreateTempFile(String path) throws IOException {
- FileSystem fs = master.getFileSystem().getDefaultVolume().getFileSystem();
+ FileSystem fs = master.getVolumeManager().getDefaultVolume().getFileSystem();
if (fs.exists(new Path(path)))
fs.delete(new Path(path), true);
fs.create(new Path(path));
@@ -793,7 +793,7 @@ class FateServiceHandler implements FateService.Iface {
* Get full path to location where initial splits are stored on file system.
*/
private String getSplitPath(String relPath) {
- Volume defaultVolume = master.getFileSystem().getDefaultVolume();
+ Volume defaultVolume = master.getVolumeManager().getDefaultVolume();
String uri = defaultVolume.getFileSystem().getUri().toString();
String basePath = defaultVolume.getBasePath();
return uri + basePath + relPath;
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index da8d902..be683d8 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -138,7 +138,6 @@ import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.accumulo.start.classloader.vfs.ContextManager;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
@@ -181,7 +180,6 @@ public class Master extends AbstractServer
private static final int MAX_BAD_STATUS_COUNT = 3;
private static final double MAX_SHUTDOWNS_PER_SEC = 10D / 60D;
- final VolumeManager fs;
private final Object balancedNotifier = new Object();
final LiveTServerSet tserverSet;
private final List<TabletGroupWatcher> watchers = new ArrayList<>();
@@ -377,7 +375,6 @@ public class Master extends AbstractServer
super("master", opts, args);
ServerContext context = super.getContext();
this.serverConfig = context.getServerConfFactory();
- this.fs = context.getVolumeManager();
AccumuloConfiguration aconf = serverConfig.getSystemConfiguration();
@@ -1598,8 +1595,8 @@ public class Master extends AbstractServer
return serverConfig;
}
- public VolumeManager getFileSystem() {
- return this.fs;
+ public VolumeManager getVolumeManager() {
+ return getContext().getVolumeManager();
}
public void assignedTablet(KeyExtent extent) {
@@ -1723,13 +1720,11 @@ public class Master extends AbstractServer
}
public FSDataOutputStream getOutputStream(final String path) throws IOException {
- FileSystem fileSystem = fs.getDefaultVolume().getFileSystem();
- return fileSystem.create(new Path(path));
+ return getVolumeManager().getDefaultVolume().getFileSystem().create(new Path(path));
}
public FSDataInputStream getInputStream(final String path) throws IOException {
- FileSystem fileSystem = fs.getDefaultVolume().getFileSystem();
- return fileSystem.open(new Path(path));
+ return getVolumeManager().getDefaultVolume().getFileSystem().open(new Path(path));
}
}
diff --git a/server/master/src/main/java/org/apache/accumulo/master/metrics/ReplicationMetrics.java b/server/master/src/main/java/org/apache/accumulo/master/metrics/ReplicationMetrics.java
index cc1a44e..7ba0134 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/metrics/ReplicationMetrics.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/metrics/ReplicationMetrics.java
@@ -118,7 +118,8 @@ public class ReplicationMetrics extends MasterMetrics {
for (Path path : paths) {
if (!pathModTimes.containsKey(path)) {
try {
- pathModTimes.put(path, master.getFileSystem().getFileStatus(path).getModificationTime());
+ pathModTimes.put(path,
+ master.getVolumeManager().getFileStatus(path).getModificationTime());
} catch (IOException e) {
// Ignore all IOExceptions
// Either the system is unavailable or the file was deleted
diff --git a/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java b/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
index 940461d..d158e0d 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
@@ -104,7 +104,7 @@ public class RecoveryManager {
boolean rescheduled = false;
try {
long time = closer.close(master.getConfiguration(), master.getContext().getHadoopConf(),
- master.getFileSystem(), new Path(source));
+ master.getVolumeManager(), new Path(source));
if (time > 0) {
executor.schedule(this, time, TimeUnit.MILLISECONDS);
@@ -143,7 +143,7 @@ public class RecoveryManager {
private boolean exists(final Path path) throws IOException {
try {
- return existenceCache.get(path, () -> master.getFileSystem().exists(path));
+ return existenceCache.get(path, () -> master.getVolumeManager().exists(path));
} catch (ExecutionException e) {
throw new IOException(e);
}
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
index e984979..8050838 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
@@ -67,7 +67,7 @@ public class ReplicationDriver extends Daemon {
while (master.stillMaster()) {
if (workMaker == null) {
client = master.getContext();
- statusMaker = new StatusMaker(client, master.getFileSystem());
+ statusMaker = new StatusMaker(client, master.getVolumeManager());
workMaker = new WorkMaker(master.getContext(), client);
finishedWorkUpdater = new FinishedWorkUpdater(client);
rcrr = new RemoveCompleteReplicationRecords(client);
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer1/BulkImport.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer1/BulkImport.java
index 9eb9776..f9a36d7 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer1/BulkImport.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer1/BulkImport.java
@@ -118,7 +118,7 @@ public class BulkImport extends MasterRepo {
Utils.getReadLock(master, tableId, tid).lock();
// check that the error directory exists and is empty
- VolumeManager fs = master.getFileSystem();
+ VolumeManager fs = master.getVolumeManager();
Path errorPath = new Path(errorDir);
FileStatus errorStatus = null;
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer1/CopyFailed.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer1/CopyFailed.java
index e84ffa0..922137a 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer1/CopyFailed.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer1/CopyFailed.java
@@ -95,7 +95,7 @@ class CopyFailed extends MasterRepo {
public Repo<Master> call(long tid, Master master) throws Exception {
// This needs to execute after the arbiter is stopped
master.updateBulkImportStatus(source, BulkImportState.COPY_FILES);
- VolumeManager fs = master.getFileSystem();
+ VolumeManager fs = master.getVolumeManager();
if (!fs.exists(new Path(error, BulkImport.FAILURES_TXT)))
return new CleanUpBulkImport(tableId, source, bulk, error);
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer1/LoadFiles.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer1/LoadFiles.java
index adf0303..77b5587 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer1/LoadFiles.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer1/LoadFiles.java
@@ -105,7 +105,7 @@ class LoadFiles extends MasterRepo {
master.updateBulkImportStatus(source, BulkImportState.LOADING);
ExecutorService executor = getThreadPool(master);
final AccumuloConfiguration conf = master.getConfiguration();
- VolumeManager fs = master.getFileSystem();
+ VolumeManager fs = master.getVolumeManager();
List<FileStatus> files = new ArrayList<>();
for (FileStatus entry : fs.listStatus(new Path(bulk))) {
files.add(entry);
@@ -202,7 +202,7 @@ class LoadFiles extends MasterRepo {
}
}
- FSDataOutputStream failFile = fs.create(new Path(errorDir, BulkImport.FAILURES_TXT), true);
+ FSDataOutputStream failFile = fs.overwrite(new Path(errorDir, BulkImport.FAILURES_TXT));
try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(failFile, UTF_8))) {
for (String f : filesToLoad) {
out.write(f);
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/BulkImportMove.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/BulkImportMove.java
index c68ca78..0fa4910 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/BulkImportMove.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/BulkImportMove.java
@@ -81,7 +81,7 @@ class BulkImportMove extends MasterRepo {
log.debug("{} sourceDir {}", fmtTid, sourceDir);
- VolumeManager fs = master.getFileSystem();
+ VolumeManager fs = master.getVolumeManager();
if (bulkInfo.tableState == TableState.ONLINE) {
ZooArbitrator.start(master.getContext(), Constants.BULK_ARBITRATOR_TYPE, tid);
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/CleanUpBulkImport.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/CleanUpBulkImport.java
index 0f46e55..d0d219b 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/CleanUpBulkImport.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/CleanUpBulkImport.java
@@ -64,8 +64,8 @@ public class CleanUpBulkImport extends MasterRepo {
Path renamingFile = new Path(bulkDir, Constants.BULK_RENAME_FILE);
Path mappingFile = new Path(bulkDir, Constants.BULK_LOAD_MAPPING);
try {
- master.getFileSystem().delete(renamingFile);
- master.getFileSystem().delete(mappingFile);
+ master.getVolumeManager().delete(renamingFile);
+ master.getVolumeManager().delete(mappingFile);
} catch (IOException ioe) {
log.debug("Failed to delete renames and/or loadmap", ioe);
}
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
index e031a0f..b378975 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
@@ -94,7 +94,7 @@ class LoadFiles extends MasterRepo {
+ FateTxId.formatTid(tid) + ")");
return 100;
}
- VolumeManager fs = master.getFileSystem();
+ VolumeManager fs = master.getVolumeManager();
final Path bulkDir = new Path(bulkInfo.bulkDir);
try (LoadMappingIterator lmi = BulkSerialize.getUpdatedLoadMapping(bulkDir.toString(),
bulkInfo.tableId, p -> fs.open(p))) {
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java
index 1641f12..6403bae 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java
@@ -157,7 +157,7 @@ public class PrepBulkImport extends MasterRepo {
private void checkForMerge(final Master master) throws Exception {
- VolumeManager fs = master.getFileSystem();
+ VolumeManager fs = master.getVolumeManager();
final Path bulkDir = new Path(bulkInfo.sourceDir);
try (LoadMappingIterator lmi =
BulkSerialize.readLoadMapping(bulkDir.toString(), bulkInfo.tableId, p -> fs.open(p))) {
@@ -178,7 +178,7 @@ public class PrepBulkImport extends MasterRepo {
bulkInfo.tableState = Tables.getTableState(master.getContext(), bulkInfo.tableId);
- VolumeManager fs = master.getFileSystem();
+ VolumeManager fs = master.getVolumeManager();
final UniqueNameAllocator namer = master.getContext().getUniqueNameAllocator();
Path sourceDir = new Path(bulkInfo.sourceDir);
List<FileStatus> files = BulkImport.filterInvalid(fs.listStatus(sourceDir));
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java
index 5fbf141..2533676 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java
@@ -59,7 +59,7 @@ class ChooseDir extends MasterRepo {
@Override
public void undo(long tid, Master master) throws Exception {
- VolumeManager fs = master.getFileSystem();
+ VolumeManager fs = master.getVolumeManager();
fs.deleteRecursively(new Path(tableInfo.getSplitDirsFile()));
}
@@ -95,7 +95,7 @@ class ChooseDir extends MasterRepo {
*/
private void writeTabletDirectoriesToFileSystem(Master master, SortedSet<Text> dirs)
throws IOException {
- FileSystem fs = master.getFileSystem().getDefaultVolume().getFileSystem();
+ FileSystem fs = master.getVolumeManager().getDefaultVolume().getFileSystem();
if (fs.exists(new Path(tableInfo.getSplitDirsFile())))
fs.delete(new Path(tableInfo.getSplitDirsFile()), true);
try (FSDataOutputStream stream = master.getOutputStream(tableInfo.getSplitDirsFile())) {
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/FinishCreateTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/FinishCreateTable.java
index fd85add..34be770 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/FinishCreateTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/FinishCreateTable.java
@@ -69,7 +69,7 @@ class FinishCreateTable extends MasterRepo {
}
private void cleanupSplitFiles(Master env) throws IOException {
- Volume defaultVolume = env.getFileSystem().getDefaultVolume();
+ Volume defaultVolume = env.getVolumeManager().getDefaultVolume();
FileSystem fs = defaultVolume.getFileSystem();
fs.delete(new Path(tableInfo.getSplitFile()), true);
fs.delete(new Path(tableInfo.getSplitDirsFile()), true);
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/delete/CleanUp.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/delete/CleanUp.java
index e8a44c7..c18f4d0 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/delete/CleanUp.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/delete/CleanUp.java
@@ -175,7 +175,7 @@ class CleanUp extends MasterRepo {
if (refCount == 0) {
// delete the map files
try {
- VolumeManager fs = master.getFileSystem();
+ VolumeManager fs = master.getVolumeManager();
for (String dir : ServerConstants.getTablesDirs(master.getContext())) {
fs.deleteRecursively(new Path(dir, tableId.canonical()));
}
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableExport/WriteExportFiles.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableExport/WriteExportFiles.java
index 21f5397..0227817 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableExport/WriteExportFiles.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableExport/WriteExportFiles.java
@@ -129,7 +129,7 @@ class WriteExportFiles extends MasterRepo {
@Override
public Repo<Master> call(long tid, Master master) throws Exception {
try {
- exportTable(master.getFileSystem(), master.getContext(), tableInfo.tableName,
+ exportTable(master.getVolumeManager(), master.getContext(), tableInfo.tableName,
tableInfo.tableID, tableInfo.exportDir);
} catch (IOException ioe) {
throw new AcceptableThriftTableOperationException(tableInfo.tableID.canonical(),
@@ -152,10 +152,10 @@ class WriteExportFiles extends MasterRepo {
TableId tableID, String exportDir) throws Exception {
fs.mkdirs(new Path(exportDir));
- Path exportMetaFilePath = fs.getVolumeByPath(new Path(exportDir)).getFileSystem()
+ Path exportMetaFilePath = fs.getFileSystemByPath(new Path(exportDir))
.makeQualified(new Path(exportDir, Constants.EXPORT_FILE));
- FSDataOutputStream fileOut = fs.create(exportMetaFilePath, false);
+ FSDataOutputStream fileOut = fs.create(exportMetaFilePath);
ZipOutputStream zipOut = new ZipOutputStream(fileOut);
BufferedOutputStream bufOut = new BufferedOutputStream(zipOut);
DataOutputStream dataOut = new DataOutputStream(bufOut);
@@ -194,7 +194,7 @@ class WriteExportFiles extends MasterRepo {
private static void createDistcpFile(VolumeManager fs, String exportDir, Path exportMetaFilePath,
Map<String,String> uniqueFiles) throws IOException {
BufferedWriter distcpOut = new BufferedWriter(
- new OutputStreamWriter(fs.create(new Path(exportDir, "distcp.txt"), false), UTF_8));
+ new OutputStreamWriter(fs.create(new Path(exportDir, "distcp.txt")), UTF_8));
try {
for (String file : uniqueFiles.values()) {
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/CreateImportDir.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/CreateImportDir.java
index a6ae987..48b68db 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/CreateImportDir.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/CreateImportDir.java
@@ -50,7 +50,7 @@ class CreateImportDir extends MasterRepo {
Set<String> tableDirs = ServerConstants.getTablesDirs(master.getContext());
log.info("Looking for matching filesystem for {} from options {}", exportDir, tableDirs);
- Path base = master.getFileSystem().matchingFileSystem(exportDir, tableDirs);
+ Path base = master.getVolumeManager().matchingFileSystem(exportDir, tableDirs);
if (base == null) {
throw new IOException(tableInfo.exportDir + " is not in a volume configured for Accumulo");
}
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/FinishImportTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/FinishImportTable.java
index aa816b6..47e8d0e 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/FinishImportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/FinishImportTable.java
@@ -44,7 +44,7 @@ class FinishImportTable extends MasterRepo {
@Override
public Repo<Master> call(long tid, Master env) throws Exception {
- env.getFileSystem().deleteRecursively(new Path(tableInfo.importDir, "mappings.txt"));
+ env.getVolumeManager().deleteRecursively(new Path(tableInfo.importDir, "mappings.txt"));
env.getTableManager().transitionTableState(tableInfo.tableId, TableState.ONLINE);
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/ImportPopulateZookeeper.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/ImportPopulateZookeeper.java
index c43447a..ad6767f 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/ImportPopulateZookeeper.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/ImportPopulateZookeeper.java
@@ -61,7 +61,7 @@ class ImportPopulateZookeeper extends MasterRepo {
Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
try {
- FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+ FileSystem ns = fs.getFileSystemByPath(path);
return TableOperationsImpl.getExportedProps(ns, path);
} catch (IOException ioe) {
throw new AcceptableThriftTableOperationException(tableInfo.tableId.canonical(),
@@ -90,7 +90,7 @@ class ImportPopulateZookeeper extends MasterRepo {
Utils.getTableNameLock().unlock();
}
- for (Entry<String,String> entry : getExportedProps(env.getFileSystem()).entrySet())
+ for (Entry<String,String> entry : getExportedProps(env.getVolumeManager()).entrySet())
if (!TablePropUtil.setTableProperty(env.getContext(), tableInfo.tableId, entry.getKey(),
entry.getValue())) {
throw new AcceptableThriftTableOperationException(tableInfo.tableId.canonical(),
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/ImportTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/ImportTable.java
index b23c95c..4c9f9b3 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/ImportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/ImportTable.java
@@ -92,7 +92,7 @@ public class ImportTable extends MasterRepo {
Integer exportVersion = null;
Integer dataVersion = null;
- try (ZipInputStream zis = new ZipInputStream(env.getFileSystem().open(path))) {
+ try (ZipInputStream zis = new ZipInputStream(env.getVolumeManager().open(path))) {
ZipEntry zipEntry;
while ((zipEntry = zis.getNextEntry()) != null) {
if (zipEntry.getName().equals(Constants.EXPORT_INFO_FILE)) {
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/MapImportFileNames.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/MapImportFileNames.java
index 7c3c121..b1e7a3c 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/MapImportFileNames.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/MapImportFileNames.java
@@ -58,7 +58,7 @@ class MapImportFileNames extends MasterRepo {
BufferedWriter mappingsWriter = null;
try {
- VolumeManager fs = environment.getFileSystem();
+ VolumeManager fs = environment.getVolumeManager();
fs.mkdirs(new Path(tableInfo.importDir));
@@ -113,6 +113,6 @@ class MapImportFileNames extends MasterRepo {
@Override
public void undo(long tid, Master env) throws Exception {
- env.getFileSystem().deleteRecursively(new Path(tableInfo.importDir));
+ env.getVolumeManager().deleteRecursively(new Path(tableInfo.importDir));
}
}
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/MoveExportedFiles.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/MoveExportedFiles.java
index bf42605..b5bb6be 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/MoveExportedFiles.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/MoveExportedFiles.java
@@ -47,7 +47,7 @@ class MoveExportedFiles extends MasterRepo {
@Override
public Repo<Master> call(long tid, Master master) throws Exception {
try {
- VolumeManager fs = master.getFileSystem();
+ VolumeManager fs = master.getVolumeManager();
Map<String,String> fileNameMappings = PopulateMetadataTable.readMappingFile(fs, tableInfo);
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/PopulateMetadataTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/PopulateMetadataTable.java
index f5a34f4..d6989c5 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/PopulateMetadataTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/PopulateMetadataTable.java
@@ -92,7 +92,7 @@ class PopulateMetadataTable extends MasterRepo {
ZipInputStream zis = null;
try {
- VolumeManager fs = master.getFileSystem();
+ VolumeManager fs = master.getVolumeManager();
mbw = master.getContext().createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
diff --git a/server/master/src/main/java/org/apache/accumulo/master/upgrade/Upgrader9to10.java b/server/master/src/main/java/org/apache/accumulo/master/upgrade/Upgrader9to10.java
index 09e6c62..839bcbc 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/upgrade/Upgrader9to10.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/upgrade/Upgrader9to10.java
@@ -302,7 +302,7 @@ public class Upgrader9to10 implements Upgrader {
for (String good : goodPaths) {
Path path = new Path(good);
- FileSystem ns = context.getVolumeManager().getVolumeByPath(path).getFileSystem();
+ FileSystem ns = context.getVolumeManager().getFileSystemByPath(path);
long maxTime = -1;
try (FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder()
.forFile(path.toString(), ns, ns.getConf(), context.getCryptoService())
diff --git a/server/master/src/test/java/org/apache/accumulo/master/metrics/ReplicationMetricsTest.java b/server/master/src/test/java/org/apache/accumulo/master/metrics/ReplicationMetricsTest.java
index 58a9299..b15c3ee 100644
--- a/server/master/src/test/java/org/apache/accumulo/master/metrics/ReplicationMetricsTest.java
+++ b/server/master/src/test/java/org/apache/accumulo/master/metrics/ReplicationMetricsTest.java
@@ -64,9 +64,9 @@ public class ReplicationMetricsTest {
// First call will initialize the map of paths to modification time
EasyMock.expect(master.getContext()).andReturn(context).anyTimes();
EasyMock.expect(util.getPendingReplicationPaths()).andReturn(Set.of(path1, path2));
- EasyMock.expect(master.getFileSystem()).andReturn(fileSystem);
+ EasyMock.expect(master.getVolumeManager()).andReturn(fileSystem);
EasyMock.expect(fileSystem.getFileStatus(path1)).andReturn(createStatus(100));
- EasyMock.expect(master.getFileSystem()).andReturn(fileSystem);
+ EasyMock.expect(master.getVolumeManager()).andReturn(fileSystem);
EasyMock.expect(fileSystem.getFileStatus(path2)).andReturn(createStatus(200));
// Second call will recognize the missing path1 and add the latency stat
diff --git a/server/master/src/test/java/org/apache/accumulo/master/upgrade/RootFilesUpgradeTest.java b/server/master/src/test/java/org/apache/accumulo/master/upgrade/RootFilesUpgradeTest.java
index 7f005ce..af9b9a0 100644
--- a/server/master/src/test/java/org/apache/accumulo/master/upgrade/RootFilesUpgradeTest.java
+++ b/server/master/src/test/java/org/apache/accumulo/master/upgrade/RootFilesUpgradeTest.java
@@ -167,31 +167,31 @@ public class RootFilesUpgradeTest {
conf.set(Property.INSTANCE_DFS_DIR, "/");
conf.set(Property.GENERAL_VOLUME_CHOOSER, RandomVolumeChooser.class.getName());
- VolumeManager vm = VolumeManagerImpl.get(conf, new Configuration());
-
- TestWrapper wrapper = new TestWrapper(vm, conf, "A00004.rf", "A00002.rf", "F00003.rf");
- wrapper.prepareReplacement();
- wrapper.renameReplacement();
- wrapper.finishReplacement();
- wrapper.assertFiles("A00004.rf");
-
- wrapper = new TestWrapper(vm, conf, "A00004.rf", "A00002.rf", "F00003.rf");
- wrapper.prepareReplacement();
- wrapper.cleanupReplacement("A00002.rf", "F00003.rf");
- wrapper.assertFiles("A00002.rf", "F00003.rf");
-
- wrapper = new TestWrapper(vm, conf, "A00004.rf", "A00002.rf", "F00003.rf");
- wrapper.prepareReplacement();
- wrapper.renameReplacement();
- wrapper.cleanupReplacement("A00004.rf");
- wrapper.assertFiles("A00004.rf");
-
- wrapper = new TestWrapper(vm, conf, "A00004.rf", "A00002.rf", "F00003.rf");
- wrapper.prepareReplacement();
- wrapper.renameReplacement();
- wrapper.finishReplacement();
- wrapper.cleanupReplacement("A00004.rf");
- wrapper.assertFiles("A00004.rf");
-
+ try (var vm = VolumeManagerImpl.get(conf, new Configuration())) {
+
+ TestWrapper wrapper = new TestWrapper(vm, conf, "A00004.rf", "A00002.rf", "F00003.rf");
+ wrapper.prepareReplacement();
+ wrapper.renameReplacement();
+ wrapper.finishReplacement();
+ wrapper.assertFiles("A00004.rf");
+
+ wrapper = new TestWrapper(vm, conf, "A00004.rf", "A00002.rf", "F00003.rf");
+ wrapper.prepareReplacement();
+ wrapper.cleanupReplacement("A00002.rf", "F00003.rf");
+ wrapper.assertFiles("A00002.rf", "F00003.rf");
+
+ wrapper = new TestWrapper(vm, conf, "A00004.rf", "A00002.rf", "F00003.rf");
+ wrapper.prepareReplacement();
+ wrapper.renameReplacement();
+ wrapper.cleanupReplacement("A00004.rf");
+ wrapper.assertFiles("A00004.rf");
+
+ wrapper = new TestWrapper(vm, conf, "A00004.rf", "A00002.rf", "F00003.rf");
+ wrapper.prepareReplacement();
+ wrapper.renameReplacement();
+ wrapper.finishReplacement();
+ wrapper.cleanupReplacement("A00004.rf");
+ wrapper.assertFiles("A00004.rf");
+ }
}
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
index 0ad002e..3b61314 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
@@ -60,15 +60,15 @@ public class BulkFailedCopyProcessor implements Processor {
VolumeManager vm = context.getVolumeManager();
try {
- FileSystem origFs = vm.getVolumeByPath(orig).getFileSystem();
- FileSystem destFs = vm.getVolumeByPath(dest).getFileSystem();
+ FileSystem origFs = vm.getFileSystemByPath(orig);
+ FileSystem destFs = vm.getFileSystemByPath(dest);
FileUtil.copy(origFs, orig, destFs, tmp, false, true, context.getHadoopConf());
destFs.rename(tmp, dest);
log.debug("copied {} to {}", orig, dest);
} catch (IOException ex) {
try {
- FileSystem destFs = vm.getVolumeByPath(dest).getFileSystem();
+ FileSystem destFs = vm.getFileSystemByPath(dest);
destFs.create(dest).close();
log.warn(" marked " + dest + " failed", ex);
} catch (IOException e) {
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
index 2a45236..66ee7ec 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
@@ -319,7 +319,7 @@ public class FileManager {
if (!file.contains(":"))
throw new IllegalArgumentException("Expected uri, got : " + file);
Path path = new Path(file);
- FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+ FileSystem ns = fs.getFileSystemByPath(path);
// log.debug("Opening "+file + " path " + path);
FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder()
.forFile(path.toString(), ns, ns.getConf(), context.getCryptoService())
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index a097399..d1ef5ef 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -535,7 +535,7 @@ public class TabletServer extends AbstractServer {
Map<TabletFile,MapFileInfo> fileRefMap = new HashMap<>();
for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
Path path = new Path(mapping.getKey());
- FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+ FileSystem ns = fs.getFileSystemByPath(path);
path = ns.makeQualified(path);
fileRefMap.put(new TabletFile(path), mapping.getValue());
}
@@ -577,7 +577,7 @@ public class TabletServer extends AbstractServer {
Map<TabletFile,MapFileInfo> newFileMap = new HashMap<>();
for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
Path path = new Path(dir, mapping.getKey());
- FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+ FileSystem ns = fs.getFileSystemByPath(path);
path = ns.makeQualified(path);
newFileMap.put(new TabletFile(path), mapping.getValue());
}
@@ -2115,7 +2115,7 @@ public class TabletServer extends AbstractServer {
BlockCache summaryCache = resourceManager.getSummaryCache();
BlockCache indexCache = resourceManager.getIndexCache();
Cache<String,Long> fileLenCache = resourceManager.getFileLenCache();
- FileSystemResolver volMgr = p -> fs.getVolumeByPath(p).getFileSystem();
+ FileSystemResolver volMgr = p -> fs.getFileSystemByPath(p);
Future<SummaryCollection> future =
new Gatherer(getContext(), request, tableCfg, getContext().getCryptoService())
.processFiles(volMgr, files, summaryCache, indexCache, fileLenCache, srp);
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
index 6a09f84..598df0a 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
@@ -158,7 +158,7 @@ public class MajorCompactionRequest implements Cloneable {
SummaryCollection sc = new SummaryCollection();
SummarizerFactory factory = new SummarizerFactory(tableConfig);
for (TabletFile file : files) {
- FileSystem fs = volumeManager.getVolumeByPath(file.getPath()).getFileSystem();
+ FileSystem fs = volumeManager.getFileSystemByPath(file.getPath());
Configuration conf = context.getHadoopConf();
SummaryCollection fsc = SummaryReader
.load(fs, conf, factory, file.getPath(), summarySelector, summaryCache, indexCache,
@@ -181,7 +181,7 @@ public class MajorCompactionRequest implements Cloneable {
// @TODO verify the file isn't some random file in HDFS
// @TODO ensure these files are always closed?
FileOperations fileFactory = FileOperations.getInstance();
- FileSystem ns = volumeManager.getVolumeByPath(tabletFile.getPath()).getFileSystem();
+ FileSystem ns = volumeManager.getFileSystemByPath(tabletFile.getPath());
return fileFactory.newReaderBuilder()
.forFile(tabletFile.getPathStr(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(tableConfig).seekToBeginning().build();
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
index a6f7f38..170d03a 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
@@ -177,7 +177,7 @@ public class LogSorter {
private void writeBuffer(String destPath, List<Pair<LogFileKey,LogFileValue>> buffer, int part)
throws IOException {
Path path = new Path(destPath, String.format("part-r-%05d", part));
- FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+ FileSystem ns = fs.getFileSystemByPath(path);
try (MapFile.Writer output = new MapFile.Writer(ns.getConf(), ns.makeQualified(path),
MapFile.Writer.keyClass(LogFileKey.class),
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/RecoveryLogReader.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/RecoveryLogReader.java
index ec8a4a07..041f103 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/RecoveryLogReader.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/RecoveryLogReader.java
@@ -133,7 +133,7 @@ public class RecoveryLogReader implements CloseableIterator<Entry<LogFileKey,Log
if (SortedLogState.FAILED.getMarker().equals(child.getPath().getName())) {
continue;
}
- FileSystem ns = fs.getVolumeByPath(child.getPath()).getFileSystem();
+ FileSystem ns = fs.getFileSystemByPath(child.getPath());
heap.add(new Index(new Reader(ns.makeQualified(child.getPath()), ns.getConf())));
}
if (!foundFinish)
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
index 3f80765..8456365 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
@@ -36,7 +36,6 @@ import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.tserver.log.DfsLogger;
import org.apache.accumulo.tserver.log.DfsLogger.DFSLoggerInputStreams;
@@ -79,65 +78,66 @@ public class LogReader {
Opts opts = new Opts();
opts.parseArgs(LogReader.class.getName(), args);
var siteConfig = SiteConfiguration.auto();
- VolumeManager fs = VolumeManagerImpl.get(siteConfig, new Configuration());
-
- Matcher rowMatcher = null;
- KeyExtent ke = null;
- Text row = null;
- if (opts.files.isEmpty()) {
- new JCommander(opts).usage();
- return;
- }
- if (opts.row != null) {
- row = new Text(opts.row);
- }
- if (opts.extent != null) {
- String[] sa = opts.extent.split(";");
- ke = new KeyExtent(TableId.of(sa[0]), new Text(sa[1]), new Text(sa[2]));
- }
- if (opts.regexp != null) {
- Pattern pattern = Pattern.compile(opts.regexp);
- rowMatcher = pattern.matcher("");
- }
+ try (var fs = VolumeManagerImpl.get(siteConfig, new Configuration())) {
+
+ Matcher rowMatcher = null;
+ KeyExtent ke = null;
+ Text row = null;
+ if (opts.files.isEmpty()) {
+ new JCommander(opts).usage();
+ return;
+ }
+ if (opts.row != null) {
+ row = new Text(opts.row);
+ }
+ if (opts.extent != null) {
+ String[] sa = opts.extent.split(";");
+ ke = new KeyExtent(TableId.of(sa[0]), new Text(sa[1]), new Text(sa[2]));
+ }
+ if (opts.regexp != null) {
+ Pattern pattern = Pattern.compile(opts.regexp);
+ rowMatcher = pattern.matcher("");
+ }
- Set<Integer> tabletIds = new HashSet<>();
+ Set<Integer> tabletIds = new HashSet<>();
- for (String file : opts.files) {
+ for (String file : opts.files) {
- Path path = new Path(file);
- LogFileKey key = new LogFileKey();
- LogFileValue value = new LogFileValue();
+ Path path = new Path(file);
+ LogFileKey key = new LogFileKey();
+ LogFileValue value = new LogFileValue();
- if (fs.getFileStatus(path).isFile()) {
- try (final FSDataInputStream fsinput = fs.open(path)) {
- // read log entries from a simple hdfs file
- DFSLoggerInputStreams streams;
- try {
- streams = DfsLogger.readHeaderAndReturnStream(fsinput, siteConfig);
- } catch (LogHeaderIncompleteException e) {
- log.warn("Could not read header for {} . Ignoring...", path);
- continue;
- }
+ if (fs.getFileStatus(path).isFile()) {
+ try (final FSDataInputStream fsinput = fs.open(path)) {
+ // read log entries from a simple hdfs file
+ DFSLoggerInputStreams streams;
+ try {
+ streams = DfsLogger.readHeaderAndReturnStream(fsinput, siteConfig);
+ } catch (LogHeaderIncompleteException e) {
+ log.warn("Could not read header for {} . Ignoring...", path);
+ continue;
+ }
- try (DataInputStream input = streams.getDecryptingInputStream()) {
- while (true) {
- try {
- key.readFields(input);
- value.readFields(input);
- } catch (EOFException ex) {
- break;
+ try (DataInputStream input = streams.getDecryptingInputStream()) {
+ while (true) {
+ try {
+ key.readFields(input);
+ value.readFields(input);
+ } catch (EOFException ex) {
+ break;
+ }
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
}
- printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
}
}
- }
- } else {
- // read the log entries sorted in a map file
- try (RecoveryLogReader input = new RecoveryLogReader(fs, path)) {
- while (input.hasNext()) {
- Entry<LogFileKey,LogFileValue> entry = input.next();
- printLogEvent(entry.getKey(), entry.getValue(), row, rowMatcher, ke, tabletIds,
- opts.maxMutations);
+ } else {
+ // read the log entries sorted in a map file
+ try (RecoveryLogReader input = new RecoveryLogReader(fs, path)) {
+ while (input.hasNext()) {
+ Entry<LogFileKey,LogFileValue> entry = input.next();
+ printLogEvent(entry.getKey(), entry.getValue(), row, rowMatcher, ke, tabletIds,
+ opts.maxMutations);
+ }
}
}
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java
index 0ca443b..1e11adf 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java
@@ -62,8 +62,6 @@ import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.core.util.HostAndPort;
import org.apache.accumulo.server.ServerContext;
-import org.apache.accumulo.server.fs.VolumeManager;
-import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.replication.ReplicaSystem;
import org.apache.accumulo.server.replication.ReplicaSystemHelper;
import org.apache.accumulo.server.replication.StatusUtil;
@@ -91,7 +89,7 @@ public class AccumuloReplicaSystem implements ReplicaSystem {
private String instanceName, zookeepers;
private AccumuloConfiguration conf;
- private VolumeManager fs;
+ private ServerContext context;
protected void setConf(AccumuloConfiguration conf) {
this.conf = conf;
@@ -122,13 +120,7 @@ public class AccumuloReplicaSystem implements ReplicaSystem {
instanceName = configuration.substring(0, index);
zookeepers = configuration.substring(index + 1);
conf = context.getConfiguration();
-
- try {
- fs = VolumeManagerImpl.get(conf, context.getHadoopConf());
- } catch (IOException e) {
- log.error("Could not connect to filesystem", e);
- throw new RuntimeException(e);
- }
+ this.context = context;
}
@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "path provided by admin")
@@ -329,7 +321,7 @@ public class AccumuloReplicaSystem implements ReplicaSystem {
log.debug("Replication WAL to peer tserver");
final Set<Integer> tids;
- try (final FSDataInputStream fsinput = fs.open(p);
+ try (final FSDataInputStream fsinput = context.getVolumeManager().open(p);
final DataInputStream input = getWalStream(p, fsinput)) {
log.debug("Skipping unwanted data in WAL");
try (TraceScope span = Trace.startSpan("Consume WAL prefix")) {
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
index 6eee8b0..d878eb4 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
@@ -208,7 +208,7 @@ public class Compactor implements Callable<CompactionStats> {
thread = Thread.currentThread();
try {
FileOperations fileFactory = FileOperations.getInstance();
- FileSystem ns = this.fs.getVolumeByPath(outputFile.getPath()).getFileSystem();
+ FileSystem ns = this.fs.getFileSystemByPath(outputFile.getPath());
mfw = fileFactory.newWriterBuilder()
.forFile(outputFile.getMetaInsert(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(acuTableConf).withRateLimiter(env.getWriteLimiter()).build();
@@ -291,7 +291,7 @@ public class Compactor implements Callable<CompactionStats> {
try {
FileOperations fileFactory = FileOperations.getInstance();
- FileSystem fs = this.fs.getVolumeByPath(mapFile.getPath()).getFileSystem();
+ FileSystem fs = this.fs.getFileSystemByPath(mapFile.getPath());
FileSKVIterator reader;
reader = fileFactory.newReaderBuilder()
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 68bde53..50dd5d5 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -1610,7 +1610,7 @@ public class Tablet {
final VolumeManager fs = getTabletServer().getFileSystem();
for (Entry<StoredTabletFile,DataFileValue> entry : allFiles.entrySet()) {
StoredTabletFile file = entry.getKey();
- FileSystem ns = fs.getVolumeByPath(file.getPath()).getFileSystem();
+ FileSystem ns = fs.getFileSystemByPath(file.getPath());
try (FileSKVIterator openReader = fileFactory.newReaderBuilder()
.forFile(file.getPathStr(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(this.getTableConfiguration()).seekToBeginning().build()) {
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
index b47d817..f5b4d43 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
@@ -28,7 +28,6 @@ import org.apache.accumulo.core.volume.VolumeImpl;
import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -46,9 +45,9 @@ public class TabletServerSyncCheckTest {
conf.set(DFS_SUPPORT_APPEND, "false");
FileSystem fs = new TestFileSystem(conf);
- TestVolumeManagerImpl vm = new TestVolumeManagerImpl(Map.of("foo", new VolumeImpl(fs, "/")));
-
- vm.ensureSyncIsEnabled();
+ try (var vm = new TestVolumeManagerImpl(Map.of("foo", new VolumeImpl(fs, "/")))) {
+ vm.ensureSyncIsEnabled();
+ }
}
private class TestFileSystem extends DistributedFileSystem {
@@ -88,7 +87,7 @@ public class TabletServerSyncCheckTest {
}
@Override
- public FSDataOutputStream create(Path path, boolean b) {
+ public FSDataOutputStream overwrite(Path path) {
return null;
}
@@ -129,7 +128,7 @@ public class TabletServerSyncCheckTest {
}
@Override
- public Volume getVolumeByPath(Path path) {
+ public FileSystem getFileSystemByPath(Path path) {
return null;
}
@@ -179,11 +178,6 @@ public class TabletServerSyncCheckTest {
}
@Override
- public ContentSummary getContentSummary(Path dir) {
- return null;
- }
-
- @Override
public String choose(VolumeChooserEnvironment env, Set<String> options) {
return null;
}
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsReaderTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsReaderTest.java
index 1d38683..5a3a0af 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsReaderTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsReaderTest.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.MapFile.Writer;
import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -51,18 +52,23 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "paths not set by user input")
public class RecoveryLogsReaderTest {
- VolumeManager fs;
- TemporaryFolder root = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+ private VolumeManager fs;
+ private File workDir;
+
+ @Rule
+ public TemporaryFolder tempFolder =
+ new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
@Before
public void setUp() throws Exception {
- root.create();
- String path = root.getRoot().getAbsolutePath() + "/manyMaps";
- fs = VolumeManagerImpl.getLocal(path);
+ workDir = tempFolder.newFolder();
+ String path = workDir.getAbsolutePath();
+ assertTrue(workDir.delete());
+ fs = VolumeManagerImpl.getLocalForTesting(path);
Path root = new Path("file://" + path);
fs.mkdirs(root);
fs.create(new Path(root, "finished")).close();
- FileSystem ns = fs.getVolumeByPath(root).getFileSystem();
+ FileSystem ns = fs.getFileSystemByPath(root);
Writer oddWriter = new Writer(ns.getConf(), ns.makeQualified(new Path(root, "odd")),
Writer.keyClass(IntWritable.class), Writer.valueClass(BytesWritable.class));
@@ -84,7 +90,7 @@ public class RecoveryLogsReaderTest {
@After
public void tearDown() throws Exception {
- root.create();
+ fs.close();
}
private void scan(RecoveryLogReader reader, int start) throws IOException {
@@ -111,7 +117,7 @@ public class RecoveryLogsReaderTest {
@Test
public void testMultiReader() throws IOException {
- Path manyMaps = new Path("file://" + root.getRoot().getAbsolutePath() + "/manyMaps");
+ Path manyMaps = new Path("file://" + workDir.getAbsolutePath());
RecoveryLogReader reader = new RecoveryLogReader(fs, manyMaps);
IntWritable key = new IntWritable();
BytesWritable value = new BytesWritable();
@@ -189,7 +195,7 @@ public class RecoveryLogsReaderTest {
*/
@Test
public void testFailed() throws Exception {
- Path manyMaps = new Path("file://" + root.getRoot().getAbsolutePath() + "/manyMaps");
+ Path manyMaps = new Path("file://" + workDir.getAbsolutePath());
fs.create(new Path(manyMaps, SortedLogState.FAILED.getMarker())).close();
RecoveryLogReader reader = new RecoveryLogReader(fs, manyMaps);
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
index 4c18dc2..b76f50b 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
@@ -45,7 +45,6 @@ import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.server.data.ServerMutation;
-import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.log.SortedLogState;
import org.apache.accumulo.tserver.logger.LogEvents;
@@ -72,6 +71,10 @@ public class SortedLogRecoveryTest {
static final Value value = new Value("value");
@Rule
+ public TemporaryFolder tempFolder =
+ new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ @Rule
public ExpectedException thrown = ExpectedException.none();
static class KeyValue implements Comparable<KeyValue> {
@@ -137,25 +140,20 @@ public class SortedLogRecoveryTest {
}
}
- private static List<Mutation> recover(Map<String,KeyValue[]> logs, KeyExtent extent)
- throws IOException {
+ private List<Mutation> recover(Map<String,KeyValue[]> logs, KeyExtent extent) throws IOException {
return recover(logs, new HashSet<>(), extent);
}
- private static List<Mutation> recover(Map<String,KeyValue[]> logs, Set<String> files,
- KeyExtent extent) throws IOException {
- TemporaryFolder root =
- new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- root.create();
- final String workdir = root.getRoot().getAbsolutePath() + "/workdir";
- VolumeManager fs = VolumeManagerImpl.getLocal(workdir);
- final Path workdirPath = new Path("file://" + workdir);
- fs.deleteRecursively(workdirPath);
- ArrayList<Path> dirs = new ArrayList<>();
- try {
+ private List<Mutation> recover(Map<String,KeyValue[]> logs, Set<String> files, KeyExtent extent)
+ throws IOException {
+ final String workdir = tempFolder.newFolder().getAbsolutePath();
+ try (var fs = VolumeManagerImpl.getLocalForTesting(workdir)) {
+ final Path workdirPath = new Path("file://" + workdir);
+ fs.deleteRecursively(workdirPath);
+ ArrayList<Path> dirs = new ArrayList<>();
for (Entry<String,KeyValue[]> entry : logs.entrySet()) {
String path = workdir + "/" + entry.getKey();
- FileSystem ns = fs.getVolumeByPath(new Path(path)).getFileSystem();
+ FileSystem ns = fs.getFileSystemByPath(new Path(path));
@SuppressWarnings("deprecation")
Writer map = new MapFile.Writer(ns.getConf(), ns, path + "/log1", LogFileKey.class,
LogFileValue.class);
@@ -171,8 +169,6 @@ public class SortedLogRecoveryTest {
CaptureMutations capture = new CaptureMutations();
recovery.recover(extent, dirs, files, capture);
return capture.result;
- } finally {
- root.delete();
}
}
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
index 1875bf1..d107e8a 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
@@ -31,11 +31,10 @@ import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.log.SortedLogState;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.Path;
+import org.junit.After;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -47,33 +46,30 @@ public class TestUpgradePathForWALogs {
private static final String WALOG_FROM_15 = "/walog-from-15.walog";
private static final String WALOG_FROM_16 = "/walog-from-16.walog";
- private static File testDir;
AccumuloConfiguration config = DefaultConfiguration.getInstance();
VolumeManager fs;
- @BeforeClass
- public static void createTestDirectory() {
- File baseDir = new File(System.getProperty("user.dir") + "/target/upgrade-tests");
- assertTrue(baseDir.mkdirs() || baseDir.isDirectory());
- testDir = new File(baseDir, TestUpgradePathForWALogs.class.getName());
- FileUtils.deleteQuietly(testDir);
- assertTrue(testDir.mkdir() || testDir.isDirectory());
- }
-
@Rule
- public TemporaryFolder root = new TemporaryFolder(testDir);
+ public TemporaryFolder tempFolder =
+ new TemporaryFolder(new File(System.getProperty("user.dir"), "target"));
@Before
public void setUp() throws Exception {
- root.create();
- String path = root.getRoot().getAbsolutePath() + "/manyMaps";
- fs = VolumeManagerImpl.getLocal(path);
+ File workDir = tempFolder.newFolder();
+ String path = workDir.getAbsolutePath();
+ assertTrue(workDir.delete());
+ fs = VolumeManagerImpl.getLocalForTesting(path);
Path manyMapsPath = new Path("file://" + path);
fs.mkdirs(manyMapsPath);
fs.create(SortedLogState.getFinishedMarkerPath(manyMapsPath)).close();
}
+ @After
+ public void tearDown() throws IOException {
+ fs.close();
+ }
+
@Test
public void testUpgradeOf15WALog() throws IOException {
InputStream walogStream = null;
@@ -83,7 +79,7 @@ public class TestUpgradePathForWALogs {
walogStream = getClass().getResourceAsStream(WALOG_FROM_15);
walogInHDFStream =
- new FileOutputStream(new File(root.getRoot().getAbsolutePath() + WALOG_FROM_15));
+ new FileOutputStream(new File(tempFolder.getRoot().getAbsolutePath() + WALOG_FROM_15));
IOUtils.copyLarge(walogStream, walogInHDFStream);
walogInHDFStream.flush();
@@ -94,8 +90,8 @@ public class TestUpgradePathForWALogs {
LogSorter.LogProcessor logProcessor = logSorter.new LogProcessor();
logProcessor.sort(WALOG_FROM_15,
- new Path("file://" + root.getRoot().getAbsolutePath() + WALOG_FROM_15),
- "file://" + root.getRoot().getAbsolutePath() + "/manyMaps");
+ new Path("file://" + tempFolder.getRoot().getAbsolutePath() + WALOG_FROM_15),
+ "file://" + tempFolder.getRoot().getAbsolutePath() + "/manyMaps");
} finally {
if (walogStream != null) {
@@ -119,7 +115,7 @@ public class TestUpgradePathForWALogs {
walogStream = getClass().getResourceAsStream(walogToTest);
walogInHDFStream =
- new FileOutputStream(new File(root.getRoot().getAbsolutePath() + walogToTest));
+ new FileOutputStream(new File(tempFolder.getRoot().getAbsolutePath() + walogToTest));
IOUtils.copyLarge(walogStream, walogInHDFStream);
walogInHDFStream.flush();
@@ -130,8 +126,8 @@ public class TestUpgradePathForWALogs {
LogSorter.LogProcessor logProcessor = logSorter.new LogProcessor();
logProcessor.sort(walogToTest,
- new Path("file://" + root.getRoot().getAbsolutePath() + walogToTest),
- "file://" + root.getRoot().getAbsolutePath() + "/manyMaps");
+ new Path("file://" + tempFolder.getRoot().getAbsolutePath() + walogToTest),
+ "file://" + tempFolder.getRoot().getAbsolutePath() + "/manyMaps");
} finally {
if (walogStream != null) {
diff --git a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
index b50439c..aaa33f7 100644
--- a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
+++ b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
@@ -31,8 +31,8 @@ import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.commons.vfs2.FileSystemManager;
import org.apache.commons.vfs2.impl.VFSClassLoader;
-import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -41,7 +41,8 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "paths not set by user input")
public class AccumuloReloadingVFSClassLoaderTest {
- private TemporaryFolder folder1 =
+ @Rule
+ public TemporaryFolder folder1 =
new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
String folderPath;
private FileSystemManager vfs;
@@ -50,7 +51,6 @@ public class AccumuloReloadingVFSClassLoaderTest {
public void setup() throws Exception {
vfs = ContextManagerTest.getVFS();
- folder1.create();
folderPath = folder1.getRoot().toURI() + ".*";
FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"),
@@ -178,9 +178,4 @@ public class AccumuloReloadingVFSClassLoaderTest {
arvcl.close();
}
- @After
- public void tearDown() throws Exception {
- folder1.delete();
- }
-
}
diff --git a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java
index f97a4d8..1311b52 100644
--- a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java
+++ b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java
@@ -23,15 +23,13 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileWriter;
-import java.io.IOException;
import java.net.URLClassLoader;
import org.apache.accumulo.start.classloader.AccumuloClassLoader;
import org.apache.commons.io.FileUtils;
import org.apache.commons.vfs2.FileSystemManager;
import org.apache.commons.vfs2.impl.VFSClassLoader;
-import org.junit.After;
-import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
@@ -54,19 +52,10 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
"org.apache.hadoop.*", "com.sun.org.apache.xerces.*"})
public class AccumuloVFSClassLoaderTest {
- private TemporaryFolder folder1 =
+ @Rule
+ public TemporaryFolder folder1 =
new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- @Before
- public void setup() throws IOException {
- folder1.create();
- }
-
- @After
- public void tearDown() {
- folder1.delete();
- }
-
/*
* Test that the default (empty dynamic class paths) does not create the 2nd level loader
*/
diff --git a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
index 00c3a86..dfb95d2 100644
--- a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
+++ b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
@@ -33,8 +33,8 @@ import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.commons.vfs2.FileSystemManager;
import org.apache.commons.vfs2.impl.VFSClassLoader;
-import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -43,11 +43,13 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "paths not set by user input")
public class ContextManagerTest {
- private TemporaryFolder folder1 =
- new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- private TemporaryFolder folder2 =
+ @Rule
+ public TemporaryFolder tempFolder =
new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
private FileSystemManager vfs;
+ private File folder1;
+ private File folder2;
private String uri1;
private String uri2;
@@ -63,17 +65,16 @@ public class ContextManagerTest {
public void setup() throws Exception {
vfs = getVFS();
-
- folder1.create();
- folder2.create();
+ folder1 = tempFolder.newFolder();
+ folder2 = tempFolder.newFolder();
FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"),
- folder1.newFile("HelloWorld.jar"));
+ new File(folder1, "HelloWorld.jar"));
FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"),
- folder2.newFile("HelloWorld.jar"));
+ new File(folder2, "HelloWorld.jar"));
- uri1 = new File(folder1.getRoot(), "HelloWorld.jar").toURI().toString();
- uri2 = folder2.getRoot().toURI() + ".*";
+ uri1 = new File(folder1, "HelloWorld.jar").toURI().toString();
+ uri2 = folder2.toURI() + ".*";
}
@@ -103,13 +104,13 @@ public class ContextManagerTest {
return null;
});
- FileObject testDir = vfs.resolveFile(folder1.getRoot().toURI().toString());
+ FileObject testDir = vfs.resolveFile(folder1.toURI().toString());
FileObject[] dirContents = testDir.getChildren();
ClassLoader cl1 = cm.getClassLoader("CX1");
FileObject[] files = ((VFSClassLoader) cl1).getFileObjects();
assertArrayEquals(createFileSystems(dirContents), files);
- FileObject testDir2 = vfs.resolveFile(folder2.getRoot().toURI().toString());
+ FileObject testDir2 = vfs.resolveFile(folder2.toURI().toString());
FileObject[] dirContents2 = testDir2.getChildren();
ClassLoader cl2 = cm.getClassLoader("CX2");
FileObject[] files2 = ((VFSClassLoader) cl2).getFileObjects();
@@ -149,10 +150,4 @@ public class ContextManagerTest {
assertNotSame(cm.getClassLoader("CX2").loadClass("test.HelloWorld"), pclass);
}
- @After
- public void tearDown() throws Exception {
- folder1.delete();
- folder2.delete();
- }
-
}
diff --git a/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java b/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java
index 8a0fab6..2073ddc 100644
--- a/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java
@@ -33,6 +33,7 @@ import org.apache.accumulo.server.util.Admin;
import org.apache.accumulo.test.functional.ConfigurableMacBase;
import org.apache.accumulo.test.functional.FunctionalTestUtils;
import org.apache.hadoop.conf.Configuration;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -40,6 +41,10 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
public class DumpConfigIT extends ConfigurableMacBase {
+ @Rule
+ public TemporaryFolder tempFolder =
+ new TemporaryFolder(new File(System.getProperty("user.dir"), "target"));
+
@Override
public int defaultTimeoutSeconds() {
return 2 * 60;
@@ -54,23 +59,19 @@ public class DumpConfigIT extends ConfigurableMacBase {
justification = "user.dir is suitable test input")
@Test
public void test() throws Exception {
- File target = new File(System.getProperty("user.dir"), "target");
- assertTrue(target.exists() || target.mkdirs());
- TemporaryFolder folder = new TemporaryFolder(target);
- folder.create();
- File siteFileBackup = new File(folder.getRoot(), "accumulo.properties.bak");
+ File folder = tempFolder.newFolder();
+ File siteFileBackup = new File(folder, "accumulo.properties.bak");
assertFalse(siteFileBackup.exists());
- assertEquals(0,
- exec(Admin.class, "dumpConfig", "-a", "-d", folder.getRoot().getPath()).waitFor());
+ assertEquals(0, exec(Admin.class, "dumpConfig", "-a", "-d", folder.getPath()).waitFor());
assertTrue(siteFileBackup.exists());
String site = FunctionalTestUtils.readAll(new FileInputStream(siteFileBackup));
assertTrue(site.contains(Property.TABLE_FILE_BLOCK_SIZE.getKey()));
assertTrue(site.contains("1234567"));
String meta = FunctionalTestUtils
- .readAll(new FileInputStream(new File(folder.getRoot(), MetadataTable.NAME + ".cfg")));
+ .readAll(new FileInputStream(new File(folder, MetadataTable.NAME + ".cfg")));
assertTrue(meta.contains(Property.TABLE_FILE_REPLICATION.getKey()));
- String systemPerm = FunctionalTestUtils
- .readAll(new FileInputStream(new File(folder.getRoot(), "root_user.cfg")));
+ String systemPerm =
+ FunctionalTestUtils.readAll(new FileInputStream(new File(folder, "root_user.cfg")));
assertTrue(systemPerm.contains("grant System.ALTER_USER -s -u root"));
assertTrue(systemPerm.contains("grant Table.READ -t " + MetadataTable.NAME + " -u root"));
assertFalse(systemPerm.contains("grant Table.DROP -t " + MetadataTable.NAME + " -u root"));
diff --git a/test/src/main/java/org/apache/accumulo/test/MultiTableRecoveryIT.java b/test/src/main/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
index 8619280..18e1d81 100644
--- a/test/src/main/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
@@ -61,7 +61,7 @@ public class MultiTableRecoveryIT extends ConfigurableMacBase {
@Override
protected int defaultTimeoutSeconds() {
- return 4 * 60;
+ return 5 * 60;
}
@Test
diff --git a/test/src/test/java/org/apache/accumulo/test/fate/zookeeper/ZooLockTest.java b/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/ZooLockIT.java
similarity index 82%
rename from test/src/test/java/org/apache/accumulo/test/fate/zookeeper/ZooLockTest.java
rename to test/src/main/java/org/apache/accumulo/test/fate/zookeeper/ZooLockIT.java
index 91955ba..83dc588 100644
--- a/test/src/test/java/org/apache/accumulo/test/fate/zookeeper/ZooLockTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/ZooLockIT.java
@@ -25,7 +25,6 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import java.io.File;
import java.lang.reflect.Field;
import java.util.Collections;
import java.util.List;
@@ -35,7 +34,7 @@ import org.apache.accumulo.fate.zookeeper.ZooLock;
import org.apache.accumulo.fate.zookeeper.ZooLock.AsyncLockWatcher;
import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
-import org.apache.accumulo.minicluster.MiniAccumuloCluster;
+import org.apache.accumulo.harness.SharedMiniClusterBase;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
@@ -45,14 +44,18 @@ import org.apache.zookeeper.ZooKeeper;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-public class ZooLockTest {
+public class ZooLockIT extends SharedMiniClusterBase {
- private static final TemporaryFolder folder =
- new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+ @BeforeClass
+ public static void setup() throws Exception {
+ SharedMiniClusterBase.startMiniCluster();
+ }
- private static MiniAccumuloCluster accumulo;
+ @AfterClass
+ public static void teardown() {
+ SharedMiniClusterBase.stopMiniCluster();
+ }
static class ConnectedWatcher implements Watcher {
volatile boolean connected = false;
@@ -108,28 +111,17 @@ public class ZooLockTest {
}
}
- @BeforeClass
- public static void setupMiniCluster() throws Exception {
-
- folder.create();
-
- accumulo = new MiniAccumuloCluster(folder.getRoot(), "superSecret");
-
- accumulo.start();
-
- }
-
private static final AtomicInteger pdCount = new AtomicInteger(0);
@Test(timeout = 10000)
public void testDeleteParent() throws Exception {
String parent = "/zltest-" + this.hashCode() + "-l" + pdCount.incrementAndGet();
- ZooLock zl = new ZooLock(accumulo.getZooKeepers(), 30000, "secret", parent);
+ ZooLock zl = new ZooLock(getCluster().getZooKeepers(), 30000, "secret", parent);
assertFalse(zl.isLocked());
- ZooReaderWriter zk = new ZooReaderWriter(accumulo.getZooKeepers(), 30000, "secret");
+ ZooReaderWriter zk = new ZooReaderWriter(getCluster().getZooKeepers(), 30000, "secret");
// intentionally created parent after lock
zk.mkdirs(parent);
@@ -156,7 +148,7 @@ public class ZooLockTest {
public void testNoParent() throws Exception {
String parent = "/zltest-" + this.hashCode() + "-l" + pdCount.incrementAndGet();
- ZooLock zl = new ZooLock(accumulo.getZooKeepers(), 30000, "secret", parent);
+ ZooLock zl = new ZooLock(getCluster().getZooKeepers(), 30000, "secret", parent);
assertFalse(zl.isLocked());
@@ -176,10 +168,10 @@ public class ZooLockTest {
public void testDeleteLock() throws Exception {
String parent = "/zltest-" + this.hashCode() + "-l" + pdCount.incrementAndGet();
- ZooReaderWriter zk = new ZooReaderWriter(accumulo.getZooKeepers(), 30000, "secret");
+ ZooReaderWriter zk = new ZooReaderWriter(getCluster().getZooKeepers(), 30000, "secret");
zk.mkdirs(parent);
- ZooLock zl = new ZooLock(accumulo.getZooKeepers(), 30000, "secret", parent);
+ ZooLock zl = new ZooLock(getCluster().getZooKeepers(), 30000, "secret", parent);
assertFalse(zl.isLocked());
@@ -207,10 +199,10 @@ public class ZooLockTest {
public void testDeleteWaiting() throws Exception {
String parent = "/zltest-" + this.hashCode() + "-l" + pdCount.incrementAndGet();
- ZooReaderWriter zk = new ZooReaderWriter(accumulo.getZooKeepers(), 30000, "secret");
+ ZooReaderWriter zk = new ZooReaderWriter(getCluster().getZooKeepers(), 30000, "secret");
zk.mkdirs(parent);
- ZooLock zl = new ZooLock(accumulo.getZooKeepers(), 30000, "secret", parent);
+ ZooLock zl = new ZooLock(getCluster().getZooKeepers(), 30000, "secret", parent);
assertFalse(zl.isLocked());
@@ -225,7 +217,7 @@ public class ZooLockTest {
assertNull(lw.exception);
assertNull(lw.reason);
- ZooLock zl2 = new ZooLock(accumulo.getZooKeepers(), 30000, "secret", parent);
+ ZooLock zl2 = new ZooLock(getCluster().getZooKeepers(), 30000, "secret", parent);
TestALW lw2 = new TestALW();
@@ -234,7 +226,7 @@ public class ZooLockTest {
assertFalse(lw2.locked);
assertFalse(zl2.isLocked());
- ZooLock zl3 = new ZooLock(accumulo.getZooKeepers(), 30000, "secret", parent);
+ ZooLock zl3 = new ZooLock(getCluster().getZooKeepers(), 30000, "secret", parent);
TestALW lw3 = new TestALW();
@@ -274,7 +266,7 @@ public class ZooLockTest {
String parent = "/zltest-" + this.hashCode() + "-l" + pdCount.incrementAndGet();
ConnectedWatcher watcher = new ConnectedWatcher();
- try (ZooKeeper zk = new ZooKeeper(accumulo.getZooKeepers(), 30000, watcher)) {
+ try (ZooKeeper zk = new ZooKeeper(getCluster().getZooKeepers(), 30000, watcher)) {
zk.addAuthInfo("digest", "accumulo:secret".getBytes(UTF_8));
while (!watcher.isConnected()) {
@@ -283,7 +275,7 @@ public class ZooLockTest {
zk.create(parent, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- ZooLock zl = new ZooLock(accumulo.getZooKeepers(), 30000, "secret", parent);
+ ZooLock zl = new ZooLock(getCluster().getZooKeepers(), 30000, "secret", parent);
assertFalse(zl.isLocked());
@@ -318,10 +310,10 @@ public class ZooLockTest {
public void testTryLock() throws Exception {
String parent = "/zltest-" + this.hashCode() + "-l" + pdCount.incrementAndGet();
- ZooLock zl = new ZooLock(accumulo.getZooKeepers(), 1000, "secret", parent);
+ ZooLock zl = new ZooLock(getCluster().getZooKeepers(), 30000, "secret", parent);
ConnectedWatcher watcher = new ConnectedWatcher();
- try (ZooKeeper zk = new ZooKeeper(accumulo.getZooKeepers(), 1000, watcher)) {
+ try (ZooKeeper zk = new ZooKeeper(getCluster().getZooKeepers(), 30000, watcher)) {
zk.addAuthInfo("digest", "accumulo:secret".getBytes(UTF_8));
while (!watcher.isConnected()) {
@@ -356,7 +348,7 @@ public class ZooLockTest {
public void testChangeData() throws Exception {
String parent = "/zltest-" + this.hashCode() + "-l" + pdCount.incrementAndGet();
ConnectedWatcher watcher = new ConnectedWatcher();
- try (ZooKeeper zk = new ZooKeeper(accumulo.getZooKeepers(), 1000, watcher)) {
+ try (ZooKeeper zk = new ZooKeeper(getCluster().getZooKeepers(), 30000, watcher)) {
zk.addAuthInfo("digest", "accumulo:secret".getBytes(UTF_8));
while (!watcher.isConnected()) {
@@ -365,7 +357,7 @@ public class ZooLockTest {
zk.create(parent, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- ZooLock zl = new ZooLock(accumulo.getZooKeepers(), 1000, "secret", parent);
+ ZooLock zl = new ZooLock(getCluster().getZooKeepers(), 30000, "secret", parent);
TestALW lw = new TestALW();
@@ -377,10 +369,4 @@ public class ZooLockTest {
}
}
- @AfterClass
- public static void tearDownMiniCluster() throws Exception {
- accumulo.stop();
- folder.delete();
- }
-
}
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
index 2bf7f55..0695ecb 100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
@@ -408,7 +408,7 @@ public class CollectTabletStats {
// assume it is a map file
status = fs.getFileStatus(new Path(file + "/data"));
}
- FileSystem ns = fs.getVolumeByPath(file.getPath()).getFileSystem();
+ FileSystem ns = fs.getFileSystemByPath(file.getPath());
BlockLocation[] locs = ns.getFileBlockLocations(status, 0, status.getLen());
System.out.println("\t\t\tBlocks for : " + file);
@@ -462,7 +462,7 @@ public class CollectTabletStats {
HashSet<ByteSequence> columnSet = createColumnBSS(columns);
for (TabletFile file : files) {
- FileSystem ns = fs.getVolumeByPath(file.getPath()).getFileSystem();
+ FileSystem ns = fs.getFileSystemByPath(file.getPath());
FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder()
.forFile(file.getPathStr(), ns, ns.getConf(), CryptoServiceFactory.newDefaultInstance())
.withTableConfiguration(aconf).build();
@@ -495,7 +495,7 @@ public class CollectTabletStats {
List<SortedKeyValueIterator<Key,Value>> readers = new ArrayList<>(files.size());
for (TabletFile file : files) {
- FileSystem ns = fs.getVolumeByPath(file.getPath()).getFileSystem();
+ FileSystem ns = fs.getFileSystemByPath(file.getPath());
readers.add(FileOperations.getInstance().newReaderBuilder()
.forFile(file.getPathStr(), ns, ns.getConf(), CryptoServiceFactory.newDefaultInstance())
.withTableConfiguration(aconf.getSystemConfiguration()).build());