You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2019/02/04 23:00:07 UTC
[accumulo] branch master updated: Add endRow to
VolumeChooserEnvironment (#524)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new 050dec2 Add endRow to VolumeChooserEnvironment (#524)
050dec2 is described below
commit 050dec2003a786ea014c994a38e180a82b997c0d
Author: Adam Lerman <al...@gmail.com>
AuthorDate: Mon Feb 4 17:53:40 2019 -0500
Add endRow to VolumeChooserEnvironment (#524)
---
.../accumulo/server/fs/VolumeChooserEnvironment.java | 19 ++++++++++++++++++-
.../org/apache/accumulo/server/fs/VolumeUtil.java | 3 ++-
.../accumulo/server/util/MetadataTableUtil.java | 5 +++--
.../apache/accumulo/server/util/RandomizeVolumes.java | 5 ++++-
.../accumulo/server/fs/PerTableVolumeChooserTest.java | 3 ++-
.../server/fs/PreferredVolumeChooserTest.java | 3 ++-
.../accumulo/server/fs/VolumeManagerImplTest.java | 2 +-
.../main/java/org/apache/accumulo/master/Master.java | 3 ++-
.../apache/accumulo/master/TabletGroupWatcher.java | 3 ++-
.../accumulo/master/tableOps/create/ChooseDir.java | 2 +-
.../tableOps/tableImport/PopulateMetadataTable.java | 9 +++++----
.../master/tableOps/tableImport/ImportTableTest.java | 4 ++--
.../org/apache/accumulo/tserver/tablet/Tablet.java | 2 +-
13 files changed, 45 insertions(+), 18 deletions(-)
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java
index 985e29e..a1a4f3e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeChooserEnvironment.java
@@ -20,6 +20,7 @@ import java.util.Objects;
import org.apache.accumulo.core.clientImpl.Table;
import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.io.Text;
public class VolumeChooserEnvironment {
@@ -35,17 +36,33 @@ public class VolumeChooserEnvironment {
private final ServerContext context;
private final ChooserScope scope;
private final Table.ID tableId;
+ private final Text endRow;
public VolumeChooserEnvironment(ChooserScope scope, ServerContext context) {
this.context = context;
this.scope = Objects.requireNonNull(scope);
this.tableId = null;
+ this.endRow = null;
}
- public VolumeChooserEnvironment(Table.ID tableId, ServerContext context) {
+ public VolumeChooserEnvironment(Table.ID tableId, Text endRow, ServerContext context) {
this.context = context;
this.scope = ChooserScope.TABLE;
this.tableId = Objects.requireNonNull(tableId);
+ this.endRow = endRow;
+
+ }
+
+ /**
+ * The end row of the tablet for which a volume is being chosen. Only call this when the scope is
+ * TABLE
+ *
+ * @since 2.0.0
+ */
+ public Text getEndRow() {
+ if (scope != ChooserScope.TABLE)
+ throw new IllegalStateException("Can only request end row for tables, not for " + scope);
+ return endRow;
}
public Table.ID getTableId() {
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index f6c2dd0..0653134 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -273,7 +273,8 @@ public class VolumeUtil {
}
VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(extent.getTableId(),
- context);
+ extent.getEndRow(), context);
+
Path newDir = new Path(vm.choose(chooserEnv, ServerConstants.getBaseUris(context))
+ Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + dir.getParent().getName()
+ Path.SEPARATOR + dir.getName());
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index b5baa50..0e8beb2 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -914,7 +914,8 @@ public class MetadataTableUtil {
Key k = entry.getKey();
Mutation m = new Mutation(k.getRow());
m.putDelete(k.getColumnFamily(), k.getColumnQualifier());
- VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableId, context);
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableId,
+ new KeyExtent(k.getRow(), (Text) null).getEndRow(), context);
String dir = volumeManager.choose(chooserEnv, ServerConstants.getBaseUris(context))
+ Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + new String(
FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES));
@@ -1023,7 +1024,7 @@ public class MetadataTableUtil {
*/
public static void createReplicationTable(ServerContext context) {
- VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(ReplicationTable.ID,
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(ReplicationTable.ID, null,
context);
String dir = context.getVolumeManager().choose(chooserEnv, ServerConstants.getBaseUris(context))
+ Constants.HDFS_TABLES_DIR + Path.SEPARATOR + ReplicationTable.ID
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index 94fbde1..a16b2bf 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.clientImpl.Table;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.master.state.tables.TableState;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
@@ -43,6 +44,7 @@ import org.apache.accumulo.server.cli.ServerUtilOnRequiredTable;
import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -105,7 +107,8 @@ public class RandomizeVolumes {
Key key = entry.getKey();
Mutation m = new Mutation(key.getRow());
- VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableId, context);
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableId,
+ new KeyExtent(key.getRow(), (Text) null).getEndRow(), context);
final String newLocation = vm.choose(chooserEnv, ServerConstants.getBaseUris(context))
+ Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR
+ directory;
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/PerTableVolumeChooserTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/PerTableVolumeChooserTest.java
index 89e8f9e..eaec3cb 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/PerTableVolumeChooserTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/PerTableVolumeChooserTest.java
@@ -78,7 +78,8 @@ public class PerTableVolumeChooserTest {
}
private VolumeChooser getTableDelegate() {
- VolumeChooserEnvironment env = new VolumeChooserEnvironment(Table.ID.of("testTable"), null);
+ VolumeChooserEnvironment env = new VolumeChooserEnvironment(Table.ID.of("testTable"), null,
+ null);
return chooser.getDelegateChooser(env);
}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/PreferredVolumeChooserTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/PreferredVolumeChooserTest.java
index 076b29a..f1edb9c 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/PreferredVolumeChooserTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/PreferredVolumeChooserTest.java
@@ -74,7 +74,8 @@ public class PreferredVolumeChooserTest {
}
private String[] chooseForTable() {
- VolumeChooserEnvironment env = new VolumeChooserEnvironment(Table.ID.of("testTable"), null);
+ VolumeChooserEnvironment env = new VolumeChooserEnvironment(Table.ID.of("testTable"), null,
+ null);
return chooser.getPreferredVolumes(env, ALL_OPTIONS);
}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
index 8a3c7e7..a3d704d 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
@@ -138,7 +138,7 @@ public class VolumeManagerImplTest {
thrown.expect(RuntimeException.class);
VolumeManager vm = VolumeManagerImpl.get(conf, hadoopConf);
VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Table.ID.of("sometable"),
- null);
+ null, null);
String choice = vm.choose(chooserEnv, volumes.toArray(new String[0]));
assertTrue("shouldn't see invalid options from misbehaving chooser.", volumes.contains(choice));
}
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index f130e0c..8005af9 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -286,7 +286,8 @@ public class Master
if (!zoo.exists(dirZPath)) {
Path oldPath = fs.getFullPath(FileType.TABLE, "/" + MetadataTable.ID + "/root_tablet");
if (fs.exists(oldPath)) {
- VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(RootTable.ID, context);
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(RootTable.ID,
+ RootTable.EXTENT.getEndRow(), context);
String newPath = fs.choose(chooserEnv, ServerConstants.getBaseUris(context))
+ Constants.HDFS_TABLES_DIR + Path.SEPARATOR + RootTable.ID;
fs.mkdirs(new Path(newPath));
diff --git a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
index 4c4fe8a..0a1437d 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
@@ -660,7 +660,8 @@ abstract class TabletGroupWatcher extends Daemon {
// Recreate the default tablet to hold the end of the table
Master.log.debug("Recreating the last tablet to point to {}", extent.getPrevEndRow());
VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(extent.getTableId(),
- master.getContext());
+ extent.getEndRow(), master.getContext());
+
String tdir = master.getFileSystem().choose(chooserEnv,
ServerConstants.getBaseUris(master.getContext())) + Constants.HDFS_TABLES_DIR
+ Path.SEPARATOR + extent.getTableId() + Constants.DEFAULT_TABLET_LOCATION;
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java
index bf3e4e0..0c401d9 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java
@@ -54,7 +54,7 @@ class ChooseDir extends MasterRepo {
// Constants.DEFAULT_TABLET_LOCATION has a leading slash prepended to it so we don't need to add
// one here
- VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableInfo.getTableId(),
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableInfo.getTableId(), null,
master.getContext());
String baseDir = master.getFileSystem().choose(chooserEnv,
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/PopulateMetadataTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/PopulateMetadataTable.java
index ca2c968..156dd95 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/PopulateMetadataTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/PopulateMetadataTable.java
@@ -152,7 +152,7 @@ class PopulateMetadataTable extends MasterRepo {
UTF_8);
// Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
- String absolutePath = getClonedTabletDir(master, tableDirs, tabletDir);
+ String absolutePath = getClonedTabletDir(master, endRow, tableDirs, tabletDir);
m = new Mutation(metadataRow);
TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m,
@@ -170,7 +170,7 @@ class PopulateMetadataTable extends MasterRepo {
UTF_8);
// Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
- String absolutePath = getClonedTabletDir(master, tableDirs, tabletDir);
+ String absolutePath = getClonedTabletDir(master, endRow, tableDirs, tabletDir);
m = new Mutation(metadataRow);
TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m,
@@ -217,9 +217,10 @@ class PopulateMetadataTable extends MasterRepo {
*
* @return An absolute, unique path for the imported table
*/
- protected String getClonedTabletDir(Master master, String[] tableDirs, String tabletDir) {
+ protected String getClonedTabletDir(Master master, Text endRow, String[] tableDirs,
+ String tabletDir) {
// We can try to spread out the tablet dirs across all volumes
- VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableInfo.tableId,
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableInfo.tableId, endRow,
master.getContext());
String tableDir = master.getFileSystem().choose(chooserEnv, tableDirs);
diff --git a/server/master/src/test/java/org/apache/accumulo/master/tableOps/tableImport/ImportTableTest.java b/server/master/src/test/java/org/apache/accumulo/master/tableOps/tableImport/ImportTableTest.java
index e9577a7..313adda 100644
--- a/server/master/src/test/java/org/apache/accumulo/master/tableOps/tableImport/ImportTableTest.java
+++ b/server/master/src/test/java/org/apache/accumulo/master/tableOps/tableImport/ImportTableTest.java
@@ -43,7 +43,7 @@ public class ImportTableTest {
EasyMock.expect(master.getContext()).andReturn(null);
EasyMock.expect(master.getFileSystem()).andReturn(volumeManager);
// Choose the 2nd element
- VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(iti.tableId, null);
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(iti.tableId, null, null);
EasyMock.expect(volumeManager.choose(EasyMock.eq(chooserEnv), EasyMock.eq(tableDirs)))
.andReturn(tableDirs[1]);
@@ -51,7 +51,7 @@ public class ImportTableTest {
PopulateMetadataTable pmt = new PopulateMetadataTable(iti);
assertEquals(tableDirs[1] + "/" + iti.tableId + "/" + tabletDir,
- pmt.getClonedTabletDir(master, tableDirs, tabletDir));
+ pmt.getClonedTabletDir(master, null, tableDirs, tabletDir));
EasyMock.verify(master, volumeManager);
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 1720343..5730bed 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -2861,7 +2861,7 @@ public class Tablet {
String lowDirectory;
UniqueNameAllocator namer = context.getUniqueNameAllocator();
- VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableId, context);
+ VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableId, endRow, context);
String volume = fs.choose(chooserEnv, ServerConstants.getBaseUris(context))
+ Constants.HDFS_TABLES_DIR + Path.SEPARATOR;