You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2017/04/20 20:51:09 UTC
[1/2] accumulo git commit: ACCUMULO-4085 Add non-table scope to
chooser env
Repository: accumulo
Updated Branches:
refs/heads/master f70f50703 -> 57e8f2195
ACCUMULO-4085 Add non-table scope to chooser env
Create a non-table volume chooser scope in the volume chooser
environment to support choosing volumes for situations other than for
compacting a particular table.
Initially support a logger scope. Can add others as needed.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/57e8f219
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/57e8f219
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/57e8f219
Branch: refs/heads/master
Commit: 57e8f21957b229c362a18b25809325e184a09cf2
Parents: 21c27c4
Author: Christopher Tubbs <ct...@apache.org>
Authored: Wed Apr 19 14:53:47 2017 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Thu Apr 20 12:29:49 2017 -0400
----------------------------------------------------------------------
.../server/fs/VolumeChooserEnvironment.java | 22 +++++++++++++++++++-
.../apache/accumulo/tserver/log/DfsLogger.java | 1 +
2 files changed, 22 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/57e8f219/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java
index 119a06f..a096854 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java
@@ -21,6 +21,8 @@ import java.util.Optional;
public class VolumeChooserEnvironment {
private final Optional<String> tableId;
+ // scope is meant for non-table identifiers
+ private String scope;
public VolumeChooserEnvironment(Optional<String> tableId) {
this.tableId = tableId;
@@ -34,6 +36,18 @@ public class VolumeChooserEnvironment {
return tableId.get();
}
+ public void setScope(String scope) {
+ this.scope = scope;
+ }
+
+ public String getScope() {
+ return this.scope;
+ }
+
+ public boolean hasScope() {
+ return this.scope != null;
+ }
+
@Override
public boolean equals(Object obj) {
if (obj == this) {
@@ -52,11 +66,17 @@ public class VolumeChooserEnvironment {
if (!other.getTableId().equals(this.getTableId())) {
return false;
}
+ if (other.hasScope() != this.hasScope()) {
+ return false;
+ }
+ if (other.hasScope() && !other.getScope().equals(this.getScope())) {
+ return false;
+ }
return true;
}
@Override
public int hashCode() {
- return tableId.hashCode();
+ return tableId.hashCode() * 31 + (this.scope == null ? 17 : this.scope.hashCode());
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/57e8f219/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
index 926d964..9f0c7f7 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
@@ -440,6 +440,7 @@ public class DfsLogger implements Comparable<DfsLogger> {
VolumeManager fs = conf.getFileSystem();
VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.empty());
+ chooserEnv.setScope("logger");
logPath = fs.choose(chooserEnv, ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.WAL_DIR + Path.SEPARATOR + logger + Path.SEPARATOR
+ filename;
[2/2] accumulo git commit: ACCUMULO-4084 improve
VolumeChooserEnvironment
Posted by ct...@apache.org.
ACCUMULO-4084 improve VolumeChooserEnvironment
Update the VolumeManager interface to take a VolumeChooserEnvironment, for maintainability
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/21c27c44
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/21c27c44
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/21c27c44
Branch: refs/heads/master
Commit: 21c27c44e0c5e58922f8134d0f41c29b18924d0f
Parents: f70f507
Author: Christopher Tubbs <ct...@apache.org>
Authored: Wed Apr 19 14:33:22 2017 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Thu Apr 20 12:29:49 2017 -0400
----------------------------------------------------------------------
.../server/fs/VolumeChooserEnvironment.java | 25 ++++++++++++++++++++
.../accumulo/server/fs/VolumeManager.java | 3 +--
.../accumulo/server/fs/VolumeManagerImpl.java | 4 +---
.../apache/accumulo/server/fs/VolumeUtil.java | 5 ++--
.../apache/accumulo/server/init/Initialize.java | 17 +++++++------
.../apache/accumulo/server/util/FileUtil.java | 4 +++-
.../accumulo/server/util/MetadataTableUtil.java | 12 ++++++----
.../accumulo/server/util/RandomizeVolumes.java | 6 +++--
.../server/fs/VolumeManagerImplTest.java | 3 ++-
.../java/org/apache/accumulo/master/Master.java | 4 +++-
.../accumulo/master/TabletGroupWatcher.java | 6 +++--
.../accumulo/master/tableOps/ChooseDir.java | 6 +++--
.../master/tableOps/PopulateMetadataTable.java | 4 +++-
.../master/tableOps/ImportTableTest.java | 4 +++-
.../apache/accumulo/tserver/log/DfsLogger.java | 6 +++--
.../apache/accumulo/tserver/tablet/Tablet.java | 4 +++-
.../tserver/TabletServerSyncCheckTest.java | 4 ++--
17 files changed, 83 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java
index a944791..119a06f 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java
@@ -34,4 +34,29 @@ public class VolumeChooserEnvironment {
return tableId.get();
}
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof VolumeChooserEnvironment)) {
+ return false;
+ }
+ VolumeChooserEnvironment other = (VolumeChooserEnvironment) obj;
+ if (other.hasTableId() != this.hasTableId()) {
+ return false;
+ }
+ if (!other.getTableId().equals(this.getTableId())) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return tableId.hashCode();
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
index 06ece48..7b72829 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
@@ -18,7 +18,6 @@ package org.apache.accumulo.server.fs;
import java.io.IOException;
import java.util.Collection;
-import java.util.Optional;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.volume.Volume;
@@ -161,7 +160,7 @@ public interface VolumeManager {
ContentSummary getContentSummary(Path dir) throws IOException;
// decide on which of the given locations to create a new file
- String choose(Optional<String> tableId, String[] options);
+ String choose(VolumeChooserEnvironment env, String[] options);
/**
* Fetch the default Volume
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index 8a5f4cb..123b5c4 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -27,7 +27,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Optional;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.DefaultConfiguration;
@@ -468,8 +467,7 @@ public class VolumeManagerImpl implements VolumeManager {
private final VolumeChooser failsafeChooser = new RandomVolumeChooser();
@Override
- public String choose(Optional<String> tableId, String[] options) {
- final VolumeChooserEnvironment env = new VolumeChooserEnvironment(tableId);
+ public String choose(VolumeChooserEnvironment env, String[] options) {
final String choice = chooser.choose(env, options);
if (!(ArrayUtils.contains(options, choice))) {
log.error("The configured volume chooser, '" + chooser.getClass() + "', or one of its delegates returned a volume not in the set of options provided; "
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index 9414909..95146c2 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -262,8 +262,9 @@ public class VolumeUtil {
throw new IllegalArgumentException("Unexpected table dir " + dir);
}
- Path newDir = new Path(vm.choose(Optional.of(extent.getTableId()), ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR
- + Path.SEPARATOR + dir.getParent().getName() + Path.SEPARATOR + dir.getName());
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.of(extent.getTableId()));
+ Path newDir = new Path(vm.choose(chooserEnv, ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR
+ + dir.getParent().getName() + Path.SEPARATOR + dir.getName());
log.info("Updating directory for " + extent + " from " + dir + " to " + newDir);
if (extent.isRootTablet()) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index 347ac87..4a17e24 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@ -86,6 +86,7 @@ import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfigurationFactory;
import org.apache.accumulo.server.constraints.MetadataConstraints;
+import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.iterators.MetadataBulkLoadFilter;
@@ -321,8 +322,9 @@ public class Initialize implements KeywordExecutable {
UUID uuid = UUID.randomUUID();
// the actual disk locations of the root table and tablets
String[] configuredVolumes = VolumeConfiguration.getVolumeUris(SiteConfiguration.getInstance());
- final String rootTabletDir = new Path(fs.choose(Optional.<String> empty(), configuredVolumes) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR
- + RootTable.ID + RootTable.ROOT_TABLET_LOCATION).toString();
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.empty());
+ final String rootTabletDir = new Path(fs.choose(chooserEnv, configuredVolumes) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + RootTable.ID
+ + RootTable.ROOT_TABLET_LOCATION).toString();
try {
initZooKeeper(opts, uuid.toString(), instanceNamePath, rootTabletDir);
@@ -420,12 +422,13 @@ public class Initialize implements KeywordExecutable {
// initialize initial system tables config in zookeeper
initSystemTablesConfig();
- String tableMetadataTabletDir = fs.choose(Optional.<String> empty(), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR
- + MetadataTable.ID + TABLE_TABLETS_TABLET_DIR;
- String replicationTableDefaultTabletDir = fs.choose(Optional.<String> empty(), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.empty());
+ String tableMetadataTabletDir = fs.choose(chooserEnv, ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + MetadataTable.ID
+ + TABLE_TABLETS_TABLET_DIR;
+ String replicationTableDefaultTabletDir = fs.choose(chooserEnv, ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR
+ ReplicationTable.ID + Constants.DEFAULT_TABLET_LOCATION;
- String defaultMetadataTabletDir = fs.choose(Optional.<String> empty(), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR
- + MetadataTable.ID + Constants.DEFAULT_TABLET_LOCATION;
+ String defaultMetadataTabletDir = fs.choose(chooserEnv, ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + MetadataTable.ID
+ + Constants.DEFAULT_TABLET_LOCATION;
// create table and default tablets directories
createDirectories(fs, rootTabletDir, tableMetadataTabletDir, defaultMetadataTabletDir, replicationTableDefaultTabletDir);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
index 95d7d11..4e0528e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
@@ -49,6 +49,7 @@ import org.apache.accumulo.core.util.LocalityGroupUtil;
import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -81,7 +82,8 @@ public class FileUtil {
private static final Logger log = LoggerFactory.getLogger(FileUtil.class);
private static Path createTmpDir(AccumuloConfiguration acuConf, VolumeManager fs) throws IOException {
- String accumuloDir = fs.choose(Optional.<String> empty(), ServerConstants.getBaseUris());
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.empty());
+ String accumuloDir = fs.choose(chooserEnv, ServerConstants.getBaseUris());
Path result = null;
while (result == null) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 13d7b6a..a4290e8 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -84,6 +84,7 @@ import org.apache.accumulo.server.AccumuloServerContext;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.tablets.TabletTime;
@@ -874,8 +875,9 @@ public class MetadataTableUtil {
Key k = entry.getKey();
Mutation m = new Mutation(k.getRow());
m.putDelete(k.getColumnFamily(), k.getColumnQualifier());
- String dir = volumeManager.choose(Optional.of(tableId), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableId
- + Path.SEPARATOR + new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES));
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.of(tableId));
+ String dir = volumeManager.choose(chooserEnv, ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR
+ + new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES));
TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(dir.getBytes(UTF_8)));
bw.addMutation(m);
@@ -969,8 +971,10 @@ public class MetadataTableUtil {
* During an upgrade from 1.6 to 1.7, we need to add the replication table
*/
public static void createReplicationTable(ClientContext context) throws IOException {
- String dir = VolumeManagerImpl.get().choose(Optional.of(ReplicationTable.ID), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR
- + ReplicationTable.ID + Constants.DEFAULT_TABLET_LOCATION;
+
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.of(ReplicationTable.ID));
+ String dir = VolumeManagerImpl.get().choose(chooserEnv, ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + ReplicationTable.ID
+ + Constants.DEFAULT_TABLET_LOCATION;
Mutation m = new Mutation(new Text(KeyExtent.getMetadataEntry(ReplicationTable.ID, null)));
m.put(DIRECTORY_COLUMN.getColumnFamily(), DIRECTORY_COLUMN.getColumnQualifier(), 0, new Value(dir.getBytes(UTF_8)));
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index a92087c..f9063c8 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@ -42,6 +42,7 @@ import org.apache.accumulo.core.util.SimpleThreadPool;
import org.apache.accumulo.server.AccumuloServerContext;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.conf.ServerConfigurationFactory;
+import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.tables.TableManager;
@@ -112,8 +113,9 @@ public class RandomizeVolumes {
Key key = entry.getKey();
Mutation m = new Mutation(key.getRow());
- final String newLocation = vm.choose(Optional.of(tableId), ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR
- + tableId + Path.SEPARATOR + directory;
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.of(tableId));
+ final String newLocation = vm.choose(chooserEnv, ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId
+ + Path.SEPARATOR + directory;
m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(UTF_8)));
if (log.isTraceEnabled()) {
log.trace("Replacing " + oldLocation + " with " + newLocation);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
index 8a8d8bf..ffac2c0 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
@@ -112,7 +112,8 @@ public class VolumeManagerImplTest {
conf.set(Property.INSTANCE_VOLUMES, StringUtils.join(volumes, ","));
conf.set(Property.GENERAL_VOLUME_CHOOSER, WrongVolumeChooser.class.getName());
VolumeManager vm = VolumeManagerImpl.get(conf);
- String choice = vm.choose(Optional.of("sometable"), volumes.toArray(new String[0]));
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.of("sometable"));
+ String choice = vm.choose(chooserEnv, volumes.toArray(new String[0]));
Assert.assertTrue("shouldn't see invalid options from misbehaving chooser.", volumes.contains(choice));
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index 032d6bc..b5877bb 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -100,6 +100,7 @@ import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.ServerOpts;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfigurationFactory;
+import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManager.FileType;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
@@ -279,7 +280,8 @@ public class Master extends AccumuloServerContext implements LiveTServerSet.List
if (!zoo.exists(dirZPath)) {
Path oldPath = fs.getFullPath(FileType.TABLE, "/" + MetadataTable.ID + "/root_tablet");
if (fs.exists(oldPath)) {
- String newPath = fs.choose(Optional.of(RootTable.ID), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + RootTable.ID;
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.of(RootTable.ID));
+ String newPath = fs.choose(chooserEnv, ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + RootTable.ID;
fs.mkdirs(new Path(newPath));
if (!fs.rename(oldPath, new Path(newPath))) {
throw new IOException("Failed to move root tablet from " + oldPath + " to " + newPath);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
index aa235d6..34dc1e2 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
@@ -72,6 +72,7 @@ import org.apache.accumulo.master.state.TableStats;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
import org.apache.accumulo.server.fs.VolumeManager.FileType;
import org.apache.accumulo.server.log.WalStateManager;
import org.apache.accumulo.server.log.WalStateManager.WalMarkerException;
@@ -642,8 +643,9 @@ abstract class TabletGroupWatcher extends Daemon {
} else {
// Recreate the default tablet to hold the end of the table
Master.log.debug("Recreating the last tablet to point to " + extent.getPrevEndRow());
- String tdir = master.getFileSystem().choose(Optional.of(extent.getTableId()), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR
- + Path.SEPARATOR + extent.getTableId() + Constants.DEFAULT_TABLET_LOCATION;
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.of(extent.getTableId()));
+ String tdir = master.getFileSystem().choose(chooserEnv, ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR
+ + extent.getTableId() + Constants.DEFAULT_TABLET_LOCATION;
MetadataTableUtil.addTablet(new KeyExtent(extent.getTableId(), null, extent.getPrevEndRow()), tdir, master, timeType, this.master.masterLock);
}
} catch (RuntimeException | IOException | TableNotFoundException | AccumuloSecurityException ex) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/master/src/main/java/org/apache/accumulo/master/tableOps/ChooseDir.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ChooseDir.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ChooseDir.java
index ef3d0df..0a33bb3 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ChooseDir.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ChooseDir.java
@@ -22,6 +22,7 @@ import org.apache.accumulo.core.Constants;
import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.master.Master;
import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
import org.apache.hadoop.fs.Path;
class ChooseDir extends MasterRepo {
@@ -41,8 +42,9 @@ class ChooseDir extends MasterRepo {
@Override
public Repo<Master> call(long tid, Master master) throws Exception {
// Constants.DEFAULT_TABLET_LOCATION has a leading slash prepended to it so we don't need to add one here
- tableInfo.dir = master.getFileSystem().choose(Optional.of(tableInfo.tableId), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR
- + tableInfo.tableId + Constants.DEFAULT_TABLET_LOCATION;
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.of(tableInfo.tableId));
+ tableInfo.dir = master.getFileSystem().choose(chooserEnv, ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableInfo.tableId
+ + Constants.DEFAULT_TABLET_LOCATION;
return new CreateDir(tableInfo);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/master/src/main/java/org/apache/accumulo/master/tableOps/PopulateMetadataTable.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/PopulateMetadataTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/PopulateMetadataTable.java
index 0c251b1..ff3ee18 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/PopulateMetadataTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/PopulateMetadataTable.java
@@ -46,6 +46,7 @@ import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.master.Master;
import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.util.MetadataTableUtil;
import org.apache.hadoop.fs.Path;
@@ -206,7 +207,8 @@ class PopulateMetadataTable extends MasterRepo {
*/
protected String getClonedTabletDir(Master master, String[] tableDirs, String tabletDir) {
// We can try to spread out the tablet dirs across all volumes
- String tableDir = master.getFileSystem().choose(Optional.of(tableInfo.tableId), tableDirs);
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.of(tableInfo.tableId));
+ String tableDir = master.getFileSystem().choose(chooserEnv, tableDirs);
// Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
return tableDir + "/" + tableInfo.tableId + "/" + tabletDir;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/master/src/test/java/org/apache/accumulo/master/tableOps/ImportTableTest.java
----------------------------------------------------------------------
diff --git a/server/master/src/test/java/org/apache/accumulo/master/tableOps/ImportTableTest.java b/server/master/src/test/java/org/apache/accumulo/master/tableOps/ImportTableTest.java
index fc82f07..1d9c694 100644
--- a/server/master/src/test/java/org/apache/accumulo/master/tableOps/ImportTableTest.java
+++ b/server/master/src/test/java/org/apache/accumulo/master/tableOps/ImportTableTest.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.master.tableOps;
import java.util.Optional;
import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
import org.apache.accumulo.server.fs.VolumeManager;
import org.easymock.EasyMock;
import org.junit.Assert;
@@ -43,7 +44,8 @@ public class ImportTableTest {
EasyMock.expect(master.getFileSystem()).andReturn(volumeManager);
// Choose the 2nd element
- EasyMock.expect(volumeManager.choose(Optional.of(iti.tableId), tableDirs)).andReturn(tableDirs[1]);
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.of(iti.tableId));
+ EasyMock.expect(volumeManager.choose(EasyMock.eq(chooserEnv), EasyMock.eq(tableDirs))).andReturn(tableDirs[1]);
EasyMock.replay(master, volumeManager);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
index 4d2fe7e..926d964 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
@@ -56,6 +56,7 @@ import org.apache.accumulo.core.util.Daemon;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.fate.util.LoggingRunnable;
import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.tserver.TabletMutations;
import org.apache.accumulo.tserver.logger.LogFileKey;
@@ -438,8 +439,9 @@ public class DfsLogger implements Comparable<DfsLogger> {
log.debug("DfsLogger.open() begin");
VolumeManager fs = conf.getFileSystem();
- logPath = fs.choose(Optional.<String> empty(), ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.WAL_DIR + Path.SEPARATOR + logger
- + Path.SEPARATOR + filename;
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.empty());
+ logPath = fs.choose(chooserEnv, ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.WAL_DIR + Path.SEPARATOR + logger + Path.SEPARATOR
+ + filename;
metaReference = toString();
LoggerOperation op = null;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index d675cbc..da3888e 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -98,6 +98,7 @@ import org.apache.accumulo.core.util.ratelimit.RateLimiter;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManager.FileType;
import org.apache.accumulo.server.fs.VolumeUtil;
@@ -2620,7 +2621,8 @@ public class Tablet implements TabletCommitter {
String lowDirectory;
UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
- String volume = fs.choose(Optional.of(tableId), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR;
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.of(tableId));
+ String volume = fs.choose(chooserEnv, ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR;
while (true) {
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21c27c44/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
index dd0be57..d584581 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
@@ -19,12 +19,12 @@ package org.apache.accumulo.tserver;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
-import java.util.Optional;
import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.volume.VolumeImpl;
+import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
@@ -203,7 +203,7 @@ public class TabletServerSyncCheckTest {
}
@Override
- public String choose(Optional<String> tableID, String[] options) {
+ public String choose(VolumeChooserEnvironment env, String[] options) {
return null;
}