You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mm...@apache.org on 2021/08/16 14:56:34 UTC
[accumulo] branch main updated: Replace static state in
ServerConstants (#2232)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new f2adfed Replace static state in ServerConstants (#2232)
f2adfed is described below
commit f2adfedda8d4efee48c5138b1373b2d4e2cd7598
Author: Mike Miller <mm...@apache.org>
AuthorDate: Mon Aug 16 10:56:23 2021 -0400
Replace static state in ServerConstants (#2232)
* Replace static state from ServerConstants with object methods and remove duplicate methods
* Add final object to ServerContext for getting baseUris and volumeReplacements
* Move static methods that use Volume to ServerUtil
---
.../apache/accumulo/server/ServerConstants.java | 156 ++++++++++-----------
.../org/apache/accumulo/server/ServerContext.java | 22 +++
.../org/apache/accumulo/server/ServerUtil.java | 16 ++-
.../org/apache/accumulo/server/fs/VolumeUtil.java | 4 +-
.../apache/accumulo/server/init/Initialize.java | 27 ++--
.../apache/accumulo/server/util/ChangeSecret.java | 6 +-
.../accumulo/server/ServerConstantsTest.java | 32 +++--
.../accumulo/gc/GarbageCollectWriteAheadLogs.java | 3 +-
.../apache/accumulo/gc/SimpleGarbageCollector.java | 6 +-
.../accumulo/manager/recovery/RecoveryManager.java | 6 +-
.../manager/tableOps/bulkVer1/BulkImport.java | 4 +-
.../manager/tableOps/bulkVer2/PrepBulkImport.java | 4 +-
.../accumulo/manager/tableOps/delete/CleanUp.java | 3 +-
.../tableOps/tableImport/CreateImportDir.java | 3 +-
.../org/apache/accumulo/tserver/TabletServer.java | 2 +-
.../org/apache/accumulo/tserver/log/DfsLogger.java | 2 +-
.../accumulo/tserver/tablet/DatafileManager.java | 3 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 7 +-
.../accumulo/test/functional/SplitRecoveryIT.java | 5 +-
19 files changed, 169 insertions(+), 142 deletions(-)
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index bb70aa8..46d39f1 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -25,13 +25,13 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeUtil;
@@ -81,27 +81,33 @@ public class ServerConstants {
public static final Set<Integer> CAN_RUN =
Set.of(SHORTEN_RFILE_KEYS, CRYPTO_CHANGES, DATA_VERSION);
public static final Set<Integer> NEEDS_UPGRADE = Sets.difference(CAN_RUN, Set.of(DATA_VERSION));
+ public static final String TABLE_DIR = "tables";
+ public static final String RECOVERY_DIR = "recovery";
+ public static final String WAL_DIR = "wal";
- private static Set<String> baseUris = null;
+ private Set<String> baseUris;
+ private Set<String> tablesDirs;
+ private Set<String> recoveryDirs;
- private static List<Pair<Path,Path>> replacementsList = null;
+ private final List<Pair<Path,Path>> replacementsList;
+ private final AccumuloConfiguration conf;
+ private final Configuration hadoopConf;
- public static Set<String> getBaseUris(ServerContext context) {
- return getBaseUris(context.getConfiguration(), context.getHadoopConf());
+ public ServerConstants(AccumuloConfiguration conf, Configuration hadoopConf) {
+ this.conf = Objects.requireNonNull(conf, "Configuration cannot be null");
+ this.hadoopConf = Objects.requireNonNull(hadoopConf, "Hadoop configuration cannot be null");
+ this.replacementsList = loadVolumeReplacements();
}
- // these are functions to delay loading the Accumulo configuration unless we must
- public static synchronized Set<String> getBaseUris(AccumuloConfiguration conf,
- Configuration hadoopConf) {
+ public Set<String> getBaseUris() {
if (baseUris == null) {
baseUris = Collections.unmodifiableSet(
checkBaseUris(hadoopConf, VolumeConfiguration.getVolumeUris(conf), false));
}
-
return baseUris;
}
- public static Set<String> checkBaseUris(Configuration hadoopConf, Set<String> configuredBaseDirs,
+ public Set<String> checkBaseUris(Configuration hadoopConf, Set<String> configuredBaseDirs,
boolean ignore) {
// all base dirs must have same instance id and data version, any dirs that have neither should
// be ignored
@@ -152,100 +158,94 @@ public class ServerConstants {
return baseDirsList;
}
- public static final String TABLE_DIR = "tables";
- public static final String RECOVERY_DIR = "recovery";
- public static final String WAL_DIR = "wal";
-
- private static Set<String> prefix(Set<String> bases, String suffix) {
+ private Set<String> prefix(Set<String> bases, String suffix) {
String actualSuffix = suffix.startsWith("/") ? suffix.substring(1) : suffix;
return bases.stream().map(base -> base + (base.endsWith("/") ? "" : "/") + actualSuffix)
.collect(Collectors.toCollection(LinkedHashSet::new));
}
- public static Set<String> getTablesDirs(ServerContext context) {
- return prefix(getBaseUris(context), TABLE_DIR);
+ public Set<String> getTablesDirs() {
+ if (tablesDirs == null) {
+ tablesDirs = prefix(getBaseUris(), TABLE_DIR);
+ }
+ return tablesDirs;
}
- public static Set<String> getRecoveryDirs(ServerContext context) {
- return prefix(getBaseUris(context), RECOVERY_DIR);
+ public Set<String> getRecoveryDirs() {
+ if (recoveryDirs == null) {
+ recoveryDirs = prefix(getBaseUris(), RECOVERY_DIR);
+ }
+ return recoveryDirs;
}
- public static Path getInstanceIdLocation(Volume v) {
- // all base dirs should have the same instance id, so can choose any one
- return v.prefixChild(INSTANCE_ID_DIR);
- }
+ private List<Pair<Path,Path>> loadVolumeReplacements() {
- public static Path getDataVersionLocation(Volume v) {
- // all base dirs should have the same version, so can choose any one
- return v.prefixChild(VERSION_DIR);
- }
+ List<Pair<Path,Path>> replacementsList;
+ String replacements = conf.get(Property.INSTANCE_VOLUMES_REPLACEMENTS);
- public static synchronized List<Pair<Path,Path>> getVolumeReplacements(AccumuloConfiguration conf,
- Configuration hadoopConf) {
+ if (replacements == null || replacements.trim().isEmpty()) {
+ return Collections.emptyList();
+ }
+ replacements = replacements.trim();
- if (replacementsList == null) {
- String replacements = conf.get(Property.INSTANCE_VOLUMES_REPLACEMENTS);
+ String[] pairs = replacements.split(",");
+ List<Pair<Path,Path>> ret = new ArrayList<>();
- replacements = replacements.trim();
+ for (String pair : pairs) {
- if (replacements.isEmpty()) {
- return Collections.emptyList();
+ String[] uris = pair.split("\\s+");
+ if (uris.length != 2) {
+ throw new IllegalArgumentException(
+ Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey() + " contains malformed pair " + pair);
}
- String[] pairs = replacements.split(",");
- List<Pair<Path,Path>> ret = new ArrayList<>();
-
- for (String pair : pairs) {
-
- String[] uris = pair.split("\\s+");
- if (uris.length != 2) {
- throw new IllegalArgumentException(
- Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey() + " contains malformed pair " + pair);
- }
-
- Path p1, p2;
- try {
- // URI constructor handles hex escaping
- p1 = new Path(new URI(VolumeUtil.removeTrailingSlash(uris[0].trim())));
- if (p1.toUri().getScheme() == null) {
- throw new IllegalArgumentException(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey()
- + " contains " + uris[0] + " which is not fully qualified");
- }
- } catch (URISyntaxException e) {
+ Path p1, p2;
+ try {
+ // URI constructor handles hex escaping
+ p1 = new Path(new URI(VolumeUtil.removeTrailingSlash(uris[0].trim())));
+ if (p1.toUri().getScheme() == null) {
throw new IllegalArgumentException(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey()
- + " contains " + uris[0] + " which has a syntax error", e);
+ + " contains " + uris[0] + " which is not fully qualified");
}
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey()
+ + " contains " + uris[0] + " which has a syntax error", e);
+ }
- try {
- p2 = new Path(new URI(VolumeUtil.removeTrailingSlash(uris[1].trim())));
- if (p2.toUri().getScheme() == null) {
- throw new IllegalArgumentException(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey()
- + " contains " + uris[1] + " which is not fully qualified");
- }
- } catch (URISyntaxException e) {
+ try {
+ p2 = new Path(new URI(VolumeUtil.removeTrailingSlash(uris[1].trim())));
+ if (p2.toUri().getScheme() == null) {
throw new IllegalArgumentException(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey()
- + " contains " + uris[1] + " which has a syntax error", e);
+ + " contains " + uris[1] + " which is not fully qualified");
}
-
- ret.add(new Pair<>(p1, p2));
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey()
+ + " contains " + uris[1] + " which has a syntax error", e);
}
- HashSet<Path> baseDirs = new HashSet<>();
- for (String baseDir : getBaseUris(conf, hadoopConf)) {
- // normalize using path
- baseDirs.add(new Path(baseDir));
- }
+ ret.add(new Pair<>(p1, p2));
+ }
- for (Pair<Path,Path> pair : ret) {
- if (!baseDirs.contains(pair.getSecond())) {
- throw new IllegalArgumentException(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey()
- + " contains " + pair.getSecond() + " which is not a configured volume");
- }
- }
+ HashSet<Path> baseDirs = new HashSet<>();
+ for (String baseDir : getBaseUris()) {
+ // normalize using path
+ baseDirs.add(new Path(baseDir));
+ }
- // only set if get here w/o exception
- replacementsList = ret;
+ for (Pair<Path,Path> pair : ret) {
+ if (!baseDirs.contains(pair.getSecond())) {
+ throw new IllegalArgumentException(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey()
+ + " contains " + pair.getSecond() + " which is not a configured volume");
+ }
}
+
+ // only set if get here w/o exception
+ replacementsList = ret;
+
return replacementsList;
}
+
+ public List<Pair<Path,Path>> getVolumeReplacements() {
+ return this.replacementsList;
+ }
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java
index c844fde..85507fe 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java
@@ -21,6 +21,8 @@ package org.apache.accumulo.server;
import static com.google.common.base.Preconditions.checkArgument;
import java.io.IOException;
+import java.util.List;
+import java.util.Set;
import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
@@ -35,6 +37,7 @@ import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.rpc.SslConnectionParams;
import org.apache.accumulo.core.singletons.SingletonReservation;
import org.apache.accumulo.core.spi.crypto.CryptoService;
+import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.fate.zookeeper.ZooCache;
import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.server.conf.NamespaceConfiguration;
@@ -49,6 +52,7 @@ import org.apache.accumulo.server.security.SecurityUtil;
import org.apache.accumulo.server.security.delegation.AuthenticationTokenSecretManager;
import org.apache.accumulo.server.tables.TableManager;
import org.apache.accumulo.server.tablets.UniqueNameAllocator;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
/**
@@ -59,6 +63,8 @@ public class ServerContext extends ClientContext {
private final ServerInfo info;
private final ZooReaderWriter zooReaderWriter;
+ private final ServerConstants serverConstants;
+
private TableManager tableManager;
private UniqueNameAllocator nameAllocator;
private ServerConfigurationFactory serverConfFactory = null;
@@ -75,6 +81,7 @@ public class ServerContext extends ClientContext {
super(SingletonReservation.noop(), info, info.getSiteConfiguration());
this.info = info;
zooReaderWriter = new ZooReaderWriter(info.getSiteConfiguration());
+ serverConstants = new ServerConstants(info.getSiteConfiguration(), info.getHadoopConf());
}
/**
@@ -257,4 +264,19 @@ public class ServerContext extends ClientContext {
return new ServerAmpleImpl(this);
}
+ public Set<String> getBaseUris() {
+ return serverConstants.getBaseUris();
+ }
+
+ public List<Pair<Path,Path>> getVolumeReplacements() {
+ return serverConstants.getVolumeReplacements();
+ }
+
+ public Set<String> getTablesDirs() {
+ return serverConstants.getTablesDirs();
+ }
+
+ public Set<String> getRecoveryDirs() {
+ return serverConstants.getRecoveryDirs();
+ }
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerUtil.java b/server/base/src/main/java/org/apache/accumulo/server/ServerUtil.java
index 3708b71..f864972 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerUtil.java
@@ -58,7 +58,7 @@ public class ServerUtil {
try {
if (getAccumuloPersistentVersion(volume) == oldVersion) {
log.debug("Attempting to upgrade {}", volume);
- Path dataVersionLocation = ServerConstants.getDataVersionLocation(volume);
+ Path dataVersionLocation = getDataVersionLocation(volume);
fs.create(new Path(dataVersionLocation, Integer.toString(ServerConstants.DATA_VERSION)))
.close();
// TODO document failure mode & recovery if FS permissions cause above to work and below
@@ -91,7 +91,7 @@ public class ServerUtil {
}
public static synchronized int getAccumuloPersistentVersion(Volume v) {
- Path path = ServerConstants.getDataVersionLocation(v);
+ Path path = getDataVersionLocation(v);
return getAccumuloPersistentVersion(v.getFileSystem(), path);
}
@@ -102,7 +102,17 @@ public class ServerUtil {
public static synchronized Path getAccumuloInstanceIdPath(VolumeManager vm) {
// It doesn't matter which Volume is used as they should all have the instance ID stored
- return ServerConstants.getInstanceIdLocation(vm.getFirst());
+ return getInstanceIdLocation(vm.getFirst());
+ }
+
+ public static Path getInstanceIdLocation(Volume v) {
+ // all base dirs should have the same instance id, so can choose any one
+ return v.prefixChild(ServerConstants.INSTANCE_ID_DIR);
+ }
+
+ public static Path getDataVersionLocation(Volume v) {
+ // all base dirs should have the same version, so can choose any one
+ return v.prefixChild(ServerConstants.VERSION_DIR);
}
public static void init(ServerContext context, String application) {
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index 95eb497..aed46a6 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -32,7 +32,6 @@ import org.apache.accumulo.core.protobuf.ProtobufUtil;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.fate.zookeeper.ServiceLock;
-import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.fs.VolumeManager.FileType;
import org.apache.accumulo.server.replication.StatusUtil;
@@ -137,8 +136,7 @@ public class VolumeUtil {
*/
public static TabletFiles updateTabletVolumes(ServerContext context, ServiceLock zooLock,
KeyExtent extent, TabletFiles tabletFiles, boolean replicate) {
- List<Pair<Path,Path>> replacements =
- ServerConstants.getVolumeReplacements(context.getConfiguration(), context.getHadoopConf());
+ List<Pair<Path,Path>> replacements = context.getVolumeReplacements();
if (replacements.isEmpty())
return tabletFiles;
log.trace("Using volume replacements: {}", replacements);
diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index c4d7636..5ae4bfa 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@ -466,23 +466,20 @@ public class Initialize implements KeywordExecutable {
new VolumeChooserEnvironmentImpl(Scope.INIT, MetadataTable.ID, splitPoint, serverContext);
String tableMetadataTabletDirName = TABLE_TABLETS_TABLET_DIR;
String tableMetadataTabletDirUri =
- fs.choose(chooserEnv, ServerConstants.getBaseUris(siteConfig, hadoopConf))
- + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + MetadataTable.ID + Path.SEPARATOR
- + tableMetadataTabletDirName;
+ fs.choose(chooserEnv, serverContext.getBaseUris()) + Constants.HDFS_TABLES_DIR
+ + Path.SEPARATOR + MetadataTable.ID + Path.SEPARATOR + tableMetadataTabletDirName;
chooserEnv =
new VolumeChooserEnvironmentImpl(Scope.INIT, ReplicationTable.ID, null, serverContext);
String replicationTableDefaultTabletDirName = ServerColumnFamily.DEFAULT_TABLET_DIR_NAME;
- String replicationTableDefaultTabletDirUri =
- fs.choose(chooserEnv, ServerConstants.getBaseUris(siteConfig, hadoopConf))
- + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + ReplicationTable.ID + Path.SEPARATOR
- + replicationTableDefaultTabletDirName;
+ String replicationTableDefaultTabletDirUri = fs.choose(chooserEnv, serverContext.getBaseUris())
+ + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + ReplicationTable.ID + Path.SEPARATOR
+ + replicationTableDefaultTabletDirName;
chooserEnv =
new VolumeChooserEnvironmentImpl(Scope.INIT, MetadataTable.ID, null, serverContext);
String defaultMetadataTabletDirName = ServerColumnFamily.DEFAULT_TABLET_DIR_NAME;
String defaultMetadataTabletDirUri =
- fs.choose(chooserEnv, ServerConstants.getBaseUris(siteConfig, hadoopConf))
- + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + MetadataTable.ID + Path.SEPARATOR
- + defaultMetadataTabletDirName;
+ fs.choose(chooserEnv, serverContext.getBaseUris()) + Constants.HDFS_TABLES_DIR
+ + Path.SEPARATOR + MetadataTable.ID + Path.SEPARATOR + defaultMetadataTabletDirName;
// create table and default tablets directories
createDirectories(fs, rootTabletDirUri, tableMetadataTabletDirUri, defaultMetadataTabletDirUri,
@@ -858,11 +855,11 @@ public class Initialize implements KeywordExecutable {
}
private static void addVolumes(VolumeManager fs, SiteConfiguration siteConfig,
- Configuration hadoopConf) throws IOException {
+ Configuration hadoopConf, ServerConstants serverConstants) throws IOException {
Set<String> volumeURIs = VolumeConfiguration.getVolumeUris(siteConfig);
- Set<String> initializedDirs = ServerConstants.checkBaseUris(hadoopConf, volumeURIs, true);
+ Set<String> initializedDirs = serverConstants.checkBaseUris(hadoopConf, volumeURIs, true);
HashSet<String> uinitializedDirs = new HashSet<>();
uinitializedDirs.addAll(volumeURIs);
@@ -873,8 +870,7 @@ public class Initialize implements KeywordExecutable {
Path versionPath = new Path(aBasePath, ServerConstants.VERSION_DIR);
UUID uuid = UUID.fromString(VolumeManager.getInstanceIDFromHdfs(iidPath, hadoopConf));
- for (Pair<Path,Path> replacementVolume : ServerConstants.getVolumeReplacements(siteConfig,
- hadoopConf)) {
+ for (Pair<Path,Path> replacementVolume : serverConstants.getVolumeReplacements()) {
if (aBasePath.equals(replacementVolume.getFirst())) {
log.error(
"{} is set to be replaced in {} and should not appear in {}."
@@ -946,6 +942,7 @@ public class Initialize implements KeywordExecutable {
setZooReaderWriter(new ZooReaderWriter(siteConfig));
SecurityUtil.serverLogin(siteConfig);
Configuration hadoopConfig = new Configuration();
+ ServerConstants serverConstants = new ServerConstants(siteConfig, hadoopConfig);
try (var fs = VolumeManagerImpl.get(siteConfig, hadoopConfig)) {
@@ -975,7 +972,7 @@ public class Initialize implements KeywordExecutable {
}
if (opts.addVolumes) {
- addVolumes(fs, siteConfig, hadoopConfig);
+ addVolumes(fs, siteConfig, hadoopConfig, serverConstants);
}
if (!opts.resetSecurity && !opts.addVolumes) {
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index 0c6124e..04c4731 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@ -32,8 +32,8 @@ import org.apache.accumulo.fate.zookeeper.ZooReader;
import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
-import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.ServerContext;
+import org.apache.accumulo.server.ServerUtil;
import org.apache.accumulo.server.cli.ServerUtilOpts;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
@@ -163,7 +163,7 @@ public class ChangeSecret {
private static void updateHdfs(VolumeManager fs, String newInstanceId) throws IOException {
// Need to recreate the instanceId on all of them to keep consistency
for (Volume v : fs.getVolumes()) {
- final Path instanceId = ServerConstants.getInstanceIdLocation(v);
+ final Path instanceId = ServerUtil.getInstanceIdLocation(v);
if (!v.getFileSystem().delete(instanceId, true)) {
throw new IOException("Could not recursively delete " + instanceId);
}
@@ -178,7 +178,7 @@ public class ChangeSecret {
private static void verifyHdfsWritePermission(VolumeManager fs) throws Exception {
for (Volume v : fs.getVolumes()) {
- final Path instanceId = ServerConstants.getInstanceIdLocation(v);
+ final Path instanceId = ServerUtil.getInstanceIdLocation(v);
FileStatus fileStatus = v.getFileSystem().getFileStatus(instanceId);
checkHdfsAccessPermissions(fileStatus, FsAction.WRITE);
}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/ServerConstantsTest.java b/server/base/src/test/java/org/apache/accumulo/server/ServerConstantsTest.java
index 35ab590..1cabad9 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/ServerConstantsTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/ServerConstantsTest.java
@@ -31,11 +31,13 @@ import java.util.UUID;
import java.util.stream.Collectors;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -45,8 +47,22 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "paths not set by user input")
public class ServerConstantsTest {
- AccumuloConfiguration conf = DefaultConfiguration.getInstance();
+ AccumuloConfiguration conf;
Configuration hadoopConf = new Configuration();
+ ServerConstants constants;
+
+ @Before
+ public void setup() throws IOException {
+ String uuid = UUID.randomUUID().toString();
+
+ var vols =
+ init(folder.newFolder(), Arrays.asList(uuid), Arrays.asList(ServerConstants.DATA_VERSION));
+
+ ConfigurationCopy copy = new ConfigurationCopy();
+ copy.set(Property.INSTANCE_VOLUMES.getKey(), String.join(",", vols));
+ conf = copy;
+ constants = new ServerConstants(conf, hadoopConf);
+ }
@Rule
public TemporaryFolder folder =
@@ -78,15 +94,15 @@ public class ServerConstantsTest {
}
private void verifyAllPass(Set<String> paths) {
- assertEquals(paths, ServerConstants.checkBaseUris(hadoopConf, paths, true));
- assertEquals(paths, ServerConstants.checkBaseUris(hadoopConf, paths, false));
+ assertEquals(paths, constants.checkBaseUris(hadoopConf, paths, true));
+ assertEquals(paths, constants.checkBaseUris(hadoopConf, paths, false));
}
private void verifySomePass(Set<String> paths) {
Set<String> subset = paths.stream().limit(2).collect(Collectors.toSet());
- assertEquals(subset, ServerConstants.checkBaseUris(hadoopConf, paths, true));
+ assertEquals(subset, constants.checkBaseUris(hadoopConf, paths, true));
try {
- ServerConstants.checkBaseUris(hadoopConf, paths, false);
+ constants.checkBaseUris(hadoopConf, paths, false);
fail();
} catch (Exception e) {
// ignored
@@ -95,14 +111,14 @@ public class ServerConstantsTest {
private void verifyError(Set<String> paths) {
try {
- ServerConstants.checkBaseUris(hadoopConf, paths, true);
+ constants.checkBaseUris(hadoopConf, paths, true);
fail();
} catch (Exception e) {
// ignored
}
try {
- ServerConstants.checkBaseUris(hadoopConf, paths, false);
+ constants.checkBaseUris(hadoopConf, paths, false);
fail();
} catch (Exception e) {
// ignored
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 72f47e9..1e1233f 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@ -46,7 +46,6 @@ import org.apache.accumulo.core.replication.ReplicationTable;
import org.apache.accumulo.core.replication.ReplicationTableOfflineException;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.log.WalStateManager;
@@ -393,7 +392,7 @@ public class GarbageCollectWriteAheadLogs {
protected Map<UUID,Path> getSortedWALogs() throws IOException {
Map<UUID,Path> result = new HashMap<>();
- for (String dir : ServerConstants.getRecoveryDirs(context)) {
+ for (String dir : context.getRecoveryDirs()) {
Path recoveryDir = new Path(dir);
if (fs.exists(recoveryDir)) {
for (FileStatus status : fs.listStatus(recoveryDir)) {
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 10bed91..3b7761f 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -83,7 +83,6 @@ import org.apache.accumulo.gc.metrics.GcCycleMetrics;
import org.apache.accumulo.gc.metrics.GcMetricsFactory;
import org.apache.accumulo.gc.replication.CloseWriteAheadLogReferences;
import org.apache.accumulo.server.AbstractServer;
-import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.ServerOpts;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManager.FileType;
@@ -306,8 +305,7 @@ public class SimpleGarbageCollector extends AbstractServer implements Iface {
ExecutorService deleteThreadPool =
ThreadPools.createExecutorService(getConfiguration(), Property.GC_DELETE_THREADS);
- final List<Pair<Path,Path>> replacements =
- ServerConstants.getVolumeReplacements(getConfiguration(), getContext().getHadoopConf());
+ final List<Pair<Path,Path>> replacements = getContext().getVolumeReplacements();
for (final String delete : confirmedDeletes.values()) {
@@ -403,7 +401,7 @@ public class SimpleGarbageCollector extends AbstractServer implements Iface {
final VolumeManager fs = getContext().getVolumeManager();
// if dir exist and is empty, then empty list is returned...
// hadoop 2.0 will throw an exception if the file does not exist
- for (String dir : ServerConstants.getTablesDirs(getContext())) {
+ for (String dir : getContext().getTablesDirs()) {
FileStatus[] tabletDirs = null;
try {
tabletDirs = fs.listStatus(new Path(dir + "/" + tableID));
diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java
index bace224..5d28f10 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java
@@ -39,7 +39,6 @@ import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.util.threads.ThreadPools;
import org.apache.accumulo.fate.zookeeper.ZooCache;
import org.apache.accumulo.manager.Manager;
-import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.fs.VolumeManager.FileType;
import org.apache.accumulo.server.fs.VolumeUtil;
import org.apache.accumulo.server.log.SortedLogState;
@@ -155,9 +154,8 @@ public class RecoveryManager {
for (Collection<String> logs : walogs) {
for (String walog : logs) {
- Path switchedWalog =
- VolumeUtil.switchVolume(walog, FileType.WAL, ServerConstants.getVolumeReplacements(
- manager.getConfiguration(), manager.getContext().getHadoopConf()));
+ Path switchedWalog = VolumeUtil.switchVolume(walog, FileType.WAL,
+ manager.getContext().getVolumeReplacements());
if (switchedWalog != null) {
// replaces the volume used for sorting, but do not change entry in metadata table. When
// the tablet loads it will change the metadata table entry. If
diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer1/BulkImport.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer1/BulkImport.java
index c710ea2..ca628f2 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer1/BulkImport.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer1/BulkImport.java
@@ -45,7 +45,6 @@ import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.manager.Manager;
import org.apache.accumulo.manager.tableOps.ManagerRepo;
import org.apache.accumulo.manager.tableOps.Utils;
-import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.tablets.UniqueNameAllocator;
@@ -159,8 +158,7 @@ public class BulkImport extends ManagerRepo {
private static Path createNewBulkDir(ServerContext context, VolumeManager fs, String sourceDir,
TableId tableId) throws IOException {
- Path tableDir =
- fs.matchingFileSystem(new Path(sourceDir), ServerConstants.getTablesDirs(context));
+ Path tableDir = fs.matchingFileSystem(new Path(sourceDir), context.getTablesDirs());
if (tableDir == null)
throw new IOException(
sourceDir + " is not in the same file system as any volume configured for Accumulo");
diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/PrepBulkImport.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/PrepBulkImport.java
index 3ca4b4f..fda73f4 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/PrepBulkImport.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/PrepBulkImport.java
@@ -48,7 +48,6 @@ import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.manager.Manager;
import org.apache.accumulo.manager.tableOps.ManagerRepo;
import org.apache.accumulo.manager.tableOps.Utils;
-import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.tablets.UniqueNameAllocator;
@@ -236,8 +235,7 @@ public class PrepBulkImport extends ManagerRepo {
private Path createNewBulkDir(ServerContext context, VolumeManager fs, TableId tableId)
throws IOException {
- Path tableDir =
- fs.matchingFileSystem(new Path(bulkInfo.sourceDir), ServerConstants.getTablesDirs(context));
+ Path tableDir = fs.matchingFileSystem(new Path(bulkInfo.sourceDir), context.getTablesDirs());
if (tableDir == null)
throw new IOException(bulkInfo.sourceDir
+ " is not in the same file system as any volume configured for Accumulo");
diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java
index 3022f4f..8108793 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java
@@ -46,7 +46,6 @@ import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.manager.Manager;
import org.apache.accumulo.manager.tableOps.ManagerRepo;
import org.apache.accumulo.manager.tableOps.Utils;
-import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.manager.state.MetaDataTableScanner;
import org.apache.accumulo.server.problems.ProblemReports;
@@ -176,7 +175,7 @@ class CleanUp extends ManagerRepo {
// delete the map files
try {
VolumeManager fs = manager.getVolumeManager();
- for (String dir : ServerConstants.getTablesDirs(manager.getContext())) {
+ for (String dir : manager.getContext().getTablesDirs()) {
fs.deleteRecursively(new Path(dir, tableId.canonical()));
}
} catch (IOException e) {
diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/CreateImportDir.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/CreateImportDir.java
index e427c52..40b2812 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/CreateImportDir.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/CreateImportDir.java
@@ -25,7 +25,6 @@ import org.apache.accumulo.core.Constants;
import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.manager.Manager;
import org.apache.accumulo.manager.tableOps.ManagerRepo;
-import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.tablets.UniqueNameAllocator;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
@@ -44,7 +43,7 @@ class CreateImportDir extends ManagerRepo {
@Override
public Repo<Manager> call(long tid, Manager manager) throws Exception {
- Set<String> tableDirs = ServerConstants.getTablesDirs(manager.getContext());
+ Set<String> tableDirs = manager.getContext().getTablesDirs();
create(tableDirs, manager);
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 1341838..8003f74 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -985,7 +985,7 @@ public class TabletServer extends AbstractServer {
VolumeChooserEnvironment chooserEnv =
new VolumeChooserEnvironmentImpl(VolumeChooserEnvironment.Scope.LOGGER, context);
Set<String> prefixes;
- var options = ServerConstants.getBaseUris(context);
+ var options = context.getBaseUris();
try {
prefixes = context.getVolumeManager().choosable(chooserEnv, options);
} catch (RuntimeException e) {
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
index 79366e6..3a4ea4e 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
@@ -406,7 +406,7 @@ public class DfsLogger implements Comparable<DfsLogger> {
var chooserEnv = new VolumeChooserEnvironmentImpl(
org.apache.accumulo.core.spi.fs.VolumeChooserEnvironment.Scope.LOGGER, context);
- logPath = fs.choose(chooserEnv, ServerConstants.getBaseUris(context)) + Path.SEPARATOR
+ logPath = fs.choose(chooserEnv, context.getBaseUris()) + Path.SEPARATOR
+ ServerConstants.WAL_DIR + Path.SEPARATOR + logger + Path.SEPARATOR + filename;
metaReference = toString();
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index e2ebc39..68458ea 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@ -45,7 +45,6 @@ import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
import org.apache.accumulo.core.replication.ReplicationConfigurationUtil;
import org.apache.accumulo.core.util.MapCounter;
import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.replication.StatusUtil;
import org.apache.accumulo.server.util.ManagerMetadataUtil;
@@ -207,7 +206,7 @@ class DatafileManager {
for (TabletFile tpath : paths.keySet()) {
boolean inTheRightDirectory = false;
Path parent = tpath.getPath().getParent().getParent();
- for (String tablesDir : ServerConstants.getTablesDirs(tablet.getContext())) {
+ for (String tablesDir : tablet.getContext().getTablesDirs()) {
if (parent.equals(new Path(tablesDir, tablet.getExtent().tableId().canonical()))) {
inTheRightDirectory = true;
break;
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index ad93e70..360ac77 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -94,7 +94,6 @@ import org.apache.accumulo.core.util.LocalityGroupUtil;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.ShutdownUtil;
import org.apache.accumulo.core.volume.Volume;
-import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.compaction.CompactionStats;
import org.apache.accumulo.server.conf.TableConfiguration;
@@ -259,10 +258,8 @@ public class Tablet {
private String chooseTabletDir() throws IOException {
VolumeChooserEnvironment chooserEnv =
new VolumeChooserEnvironmentImpl(extent.tableId(), extent.endRow(), context);
- String dirUri =
- tabletServer.getVolumeManager().choose(chooserEnv, ServerConstants.getBaseUris(context))
- + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + extent.tableId() + Path.SEPARATOR
- + dirName;
+ String dirUri = tabletServer.getVolumeManager().choose(chooserEnv, context.getBaseUris())
+ + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + extent.tableId() + Path.SEPARATOR + dirName;
checkTabletDir(new Path(dirUri));
return dirUri;
}
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java b/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
index 7a3ae75..0ddd3ce 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
@@ -68,7 +68,6 @@ import org.apache.accumulo.fate.zookeeper.ServiceLock.LockLossReason;
import org.apache.accumulo.fate.zookeeper.ServiceLock.LockWatcher;
import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
-import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.manager.state.Assignment;
import org.apache.accumulo.server.util.ManagerMetadataUtil;
@@ -159,8 +158,8 @@ public class SplitRecoveryIT extends ConfigurableMacBase {
KeyExtent extent = extents[i];
String dirName = "dir_" + i;
- String tdir = ServerConstants.getTablesDirs(context).iterator().next() + "/"
- + extent.tableId() + "/" + dirName;
+ String tdir =
+ context.getTablesDirs().iterator().next() + "/" + extent.tableId() + "/" + dirName;
MetadataTableUtil.addTablet(extent, dirName, context, TimeType.LOGICAL, zl);
SortedMap<TabletFile,DataFileValue> mapFiles = new TreeMap<>();
mapFiles.put(new TabletFile(new Path(tdir + "/" + RFile.EXTENSION + "_000_000")),