You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2014/01/28 18:16:01 UTC
[1/3] git commit: ACCUMULO-2265 fixing copyright notice
Updated Branches:
refs/heads/master 1bcc62af3 -> 601110868
ACCUMULO-2265 fixing copyright notice
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3c3911dd
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3c3911dd
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3c3911dd
Branch: refs/heads/master
Commit: 3c3911dd59d4326a7c95e0f353186b208f1b25da
Parents: bff8668
Author: John Vines <vi...@apache.org>
Authored: Tue Jan 28 10:29:20 2014 -0500
Committer: John Vines <vi...@apache.org>
Committed: Tue Jan 28 10:29:47 2014 -0500
----------------------------------------------------------------------
.../src/main/java/org/apache/accumulo/tserver/TabletServer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c3911dd/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 5c1f6ce..d03f4b6 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -1,5 +1,5 @@
/*
-77 * Licensed to the Apache Software Foundation (ASF) under one or more
+ * Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
[3/3] git commit: Merge remote-tracking branch 'origin/1.6.0-SNAPSHOT'
Posted by kt...@apache.org.
Merge remote-tracking branch 'origin/1.6.0-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/60111086
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/60111086
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/60111086
Branch: refs/heads/master
Commit: 60111086809d76f04bb8d7a273d42b62412f2e34
Parents: 1bcc62a 783e895
Author: Keith Turner <kt...@apache.org>
Authored: Tue Jan 28 12:21:57 2014 -0500
Committer: Keith Turner <kt...@apache.org>
Committed: Tue Jan 28 12:21:57 2014 -0500
----------------------------------------------------------------------
.../accumulo/server/util/MetadataTableUtil.java | 20 +++
.../tserver/DirectoryDecommissioner.java | 169 +++++++++++++++++++
.../org/apache/accumulo/tserver/Tablet.java | 6 +-
.../apache/accumulo/tserver/TabletServer.java | 2 +-
.../tserver/DirectoryDecommissionerTest.java | 108 ++++++++++++
.../java/org/apache/accumulo/test/VolumeIT.java | 81 +++++++--
6 files changed, 366 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/60111086/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
----------------------------------------------------------------------
[2/3] git commit: ACCUMULO-1772 If at table load time volume is no
longer configured, then choose another
Posted by kt...@apache.org.
ACCUMULO-1772 If at table load time volume is no longer configured, then choose another
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/783e895f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/783e895f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/783e895f
Branch: refs/heads/master
Commit: 783e895fbab086059c0a3d64f8fba566973ffcc6
Parents: 3c3911d
Author: Keith Turner <kt...@apache.org>
Authored: Tue Jan 28 12:17:28 2014 -0500
Committer: Keith Turner <kt...@apache.org>
Committed: Tue Jan 28 12:17:28 2014 -0500
----------------------------------------------------------------------
.../accumulo/server/util/MetadataTableUtil.java | 20 +++
.../tserver/DirectoryDecommissioner.java | 169 +++++++++++++++++++
.../org/apache/accumulo/tserver/Tablet.java | 6 +-
.../tserver/DirectoryDecommissionerTest.java | 108 ++++++++++++
.../java/org/apache/accumulo/test/VolumeIT.java | 81 +++++++--
5 files changed, 365 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/783e895f/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 158157b..bc80b9e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -177,6 +177,12 @@ public class MetadataTableUtil {
update(credentials, zooLock, m, extent);
}
+ public static void updateTabletDir(KeyExtent extent, String newDir, Credentials creds, ZooLock lock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes()));
+ update(creds, lock, m, extent);
+ }
+
public static void addTablet(KeyExtent extent, String path, Credentials credentials, char timeType, ZooLock lock) {
Mutation m = extent.getPrevRowUpdateMutation();
@@ -418,6 +424,19 @@ public class MetadataTableUtil {
}
}
+ public static void setRootTabletDir(String dir) throws IOException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ String zpath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_PATH;
+ try {
+ zoo.putPersistentData(zpath, dir.getBytes(Constants.UTF8), -1, NodeExistsPolicy.OVERWRITE);
+ } catch (KeeperException e) {
+ throw new IOException(e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException(e);
+ }
+ }
+
public static String getRootTabletDir() throws IOException {
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
String zpath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_PATH;
@@ -426,6 +445,7 @@ public class MetadataTableUtil {
} catch (KeeperException e) {
throw new IOException(e);
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new IOException(e);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/783e895f/server/tserver/src/main/java/org/apache/accumulo/tserver/DirectoryDecommissioner.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/DirectoryDecommissioner.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/DirectoryDecommissioner.java
new file mode 100644
index 0000000..ea932ff
--- /dev/null
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/DirectoryDecommissioner.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.security.SecureRandom;
+import java.util.HashSet;
+
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * This class contains utility code for switching a tablets default directory, if that default directory is no longer configured.
+ */
+public class DirectoryDecommissioner {
+
+ private static final Logger log = Logger.getLogger(DirectoryDecommissioner.class);
+
+ public static boolean isActiveVolume(Path dir) {
+ for (String tableDir : ServerConstants.getTablesDirs()) {
+ // use Path to normalize tableDir
+ if (dir.toString().startsWith(new Path(tableDir).toString()))
+ return true;
+ }
+
+ return false;
+ }
+
+ public static Path checkTabletDirectory(TabletServer tserver, VolumeManager vm, KeyExtent extent, Path dir) throws IOException {
+ if (isActiveVolume(dir))
+ return dir;
+
+ if (!dir.getParent().getParent().getName().equals(ServerConstants.TABLE_DIR)) {
+ throw new IllegalArgumentException("Unexpected table dir " + dir);
+ }
+
+ Path newDir = new Path(vm.choose(ServerConstants.getTablesDirs()) + "/" + dir.getParent().getName() + "/" + dir.getName());
+
+ log.info("Updating directory for " + extent + " from " + dir + " to " + newDir);
+ if (extent.isRootTablet()) {
+ // the root tablet is special case, its files need to be copied if its dir is changed
+
+ // this code needs to be idempotent
+
+ FileSystem fs1 = vm.getFileSystemByPath(dir);
+ FileSystem fs2 = vm.getFileSystemByPath(newDir);
+
+ if (!same(fs1, dir, fs2, newDir)) {
+ if (fs2.exists(newDir)) {
+ Path newDirBackup = getBackupName(fs2, newDir);
+ // never delete anything because were dealing with the root tablet
+ // one reason this dir may exist is because this method failed previously
+ log.info("renaming " + newDir + " to " + newDirBackup);
+ if (!fs2.rename(newDir, newDirBackup)) {
+ throw new IOException("Failed to rename " + newDir + " to " + newDirBackup);
+ }
+ }
+
+ // do a lot of logging since this is the root tablet
+ log.info("copying " + dir + " to " + newDir);
+ if (!FileUtil.copy(fs1, dir, fs2, newDir, false, CachedConfiguration.getInstance())) {
+ throw new IOException("Failed to copy " + dir + " to " + newDir);
+ }
+
+ // only set the new location in zookeeper after a successful copy
+ log.info("setting root tablet location to " + newDir);
+ MetadataTableUtil.setRootTabletDir(newDir.toString());
+
+ // rename the old dir to avoid confusion when someone looks at filesystem... its ok if we fail here and this does not happen because the location in
+ // zookeeper is the authority
+ Path dirBackup = getBackupName(fs1, dir);
+ log.info("renaming " + dir + " to " + dirBackup);
+ fs1.rename(dir, dirBackup);
+ } else {
+ log.info("setting root tablet location to " + newDir);
+ MetadataTableUtil.setRootTabletDir(newDir.toString());
+ }
+
+ return dir;
+ } else {
+ MetadataTableUtil.updateTabletDir(extent, newDir.toString(), SystemCredentials.get(), tserver.getLock());
+ return newDir;
+ }
+ }
+
+ static boolean same(FileSystem fs1, Path dir, FileSystem fs2, Path newDir) throws FileNotFoundException, IOException {
+ // its possible that a user changes config in such a way that two uris point to the same thing. Like hdfs://foo/a/b and hdfs://1.2.3.4/a/b both reference
+ // the same thing because DNS resolves foo to 1.2.3.4. This method does not analyze uris to determine if equivalent, instead it inspects the contents of
+ // what the uris point to.
+
+ //this code is called infrequently and does not need to be optimized.
+
+ if (fs1.exists(dir) && fs2.exists(newDir)) {
+
+ if (!fs1.isDirectory(dir))
+ throw new IllegalArgumentException("expected " + dir + " to be a directory");
+
+
+ if (!fs2.isDirectory(newDir))
+ throw new IllegalArgumentException("expected " + newDir + " to be a directory");
+
+
+ HashSet<String> names1 = getFileNames(fs1.listStatus(dir));
+ HashSet<String> names2 = getFileNames(fs2.listStatus(newDir));
+
+ if (names1.equals(names2)) {
+ for (String name : names1)
+ if (!hash(fs1, dir, name).equals(hash(fs2, newDir, name)))
+ return false;
+ return true;
+ }
+
+ }
+ return false;
+ }
+
+ @SuppressWarnings("deprecation")
+ private static HashSet<String> getFileNames(FileStatus[] filesStatuses) {
+ HashSet<String> names = new HashSet<String>();
+ for (FileStatus fileStatus : filesStatuses)
+ if (fileStatus.isDir())
+ throw new IllegalArgumentException("expected " + fileStatus.getPath() + " to be a file");
+ else
+ names.add(fileStatus.getPath().getName());
+ return names;
+ }
+
+ private static String hash(FileSystem fs, Path dir, String name) throws IOException {
+ FSDataInputStream in = fs.open(new Path(dir, name));
+ try {
+ return DigestUtils.sha1Hex(in);
+ } finally {
+ in.close();
+ }
+
+ }
+
+ private static Path getBackupName(FileSystem fs, Path path) {
+ SecureRandom rand = new SecureRandom();
+ return new Path(path.getParent(), path.getName() + "_" + System.currentTimeMillis() + "_" + Math.abs(rand.nextInt()) + ".bak");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/783e895f/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
index e457ed5..d3f8993 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
@@ -1256,8 +1256,10 @@ public class Tablet {
} else {
locationPath = fs.getFullPath(FileType.TABLE, extent.getTableId().toString() + location.toString());
}
- FileSystem fsForPath = fs.getFileSystemByPath(locationPath);
- this.location = locationPath.makeQualified(fsForPath.getUri(), fsForPath.getWorkingDirectory());
+
+ locationPath = DirectoryDecommissioner.checkTabletDirectory(tabletServer, fs, extent, locationPath);
+
+ this.location = locationPath;
this.lastLocation = lastLocation;
this.tabletDirectory = location.toString();
this.conf = conf;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/783e895f/server/tserver/src/test/java/org/apache/accumulo/tserver/DirectoryDecommissionerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/DirectoryDecommissionerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/DirectoryDecommissionerTest.java
new file mode 100644
index 0000000..47cdab9
--- /dev/null
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/DirectoryDecommissionerTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ *
+ */
+public class DirectoryDecommissionerTest {
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ @Test
+ public void testSame() throws Exception {
+ FileSystem fs = FileSystem.getLocal(new Configuration());
+
+ Path subdir1 = new Path(tempFolder.newFolder().toURI());
+ Path subdir2 = new Path(tempFolder.newFolder().toURI());
+ Path subdir3 = new Path(tempFolder.newFolder().toURI());
+
+ Assert.assertFalse(DirectoryDecommissioner.same(fs, subdir1, fs, new Path(tempFolder.getRoot().toURI().toString(), "8854339269459287524098238497")));
+ Assert.assertFalse(DirectoryDecommissioner.same(fs, new Path(tempFolder.getRoot().toURI().toString(), "8854339269459287524098238497"), fs, subdir1));
+ Assert.assertTrue(DirectoryDecommissioner.same(fs, subdir1, fs, subdir1));
+
+ writeFile(fs, subdir1, "abc", "foo");
+ writeFile(fs, subdir2, "abc", "bar");
+ writeFile(fs, subdir3, "abc", "foo");
+
+ Assert.assertTrue(DirectoryDecommissioner.same(fs, subdir1, fs, subdir1));
+ Assert.assertFalse(DirectoryDecommissioner.same(fs, subdir1, fs, subdir2));
+ Assert.assertFalse(DirectoryDecommissioner.same(fs, subdir2, fs, subdir1));
+ Assert.assertTrue(DirectoryDecommissioner.same(fs, subdir1, fs, subdir3));
+ Assert.assertTrue(DirectoryDecommissioner.same(fs, subdir3, fs, subdir1));
+
+ writeFile(fs, subdir1, "def", "123456");
+ writeFile(fs, subdir2, "def", "123456");
+ writeFile(fs, subdir3, "def", "123456");
+
+ Assert.assertTrue(DirectoryDecommissioner.same(fs, subdir1, fs, subdir1));
+ Assert.assertFalse(DirectoryDecommissioner.same(fs, subdir1, fs, subdir2));
+ Assert.assertFalse(DirectoryDecommissioner.same(fs, subdir2, fs, subdir1));
+ Assert.assertTrue(DirectoryDecommissioner.same(fs, subdir1, fs, subdir3));
+ Assert.assertTrue(DirectoryDecommissioner.same(fs, subdir3, fs, subdir1));
+
+ writeFile(fs, subdir3, "ghi", "09876");
+
+ Assert.assertFalse(DirectoryDecommissioner.same(fs, subdir1, fs, subdir3));
+ Assert.assertFalse(DirectoryDecommissioner.same(fs, subdir3, fs, subdir1));
+
+ fs.mkdirs(new Path(subdir2, "dir1"));
+
+ try {
+ DirectoryDecommissioner.same(fs, subdir1, fs, subdir2);
+ Assert.fail();
+ } catch (IllegalArgumentException e) {}
+
+ try {
+ DirectoryDecommissioner.same(fs, subdir2, fs, subdir1);
+ Assert.fail();
+ } catch (IllegalArgumentException e) {}
+
+ try {
+ DirectoryDecommissioner.same(fs, subdir1, fs, new Path(subdir2, "def"));
+ Assert.fail();
+ } catch (IllegalArgumentException e) {}
+
+ try {
+ DirectoryDecommissioner.same(fs, new Path(subdir2, "def"), fs, subdir3);
+ Assert.fail();
+ } catch (IllegalArgumentException e) {}
+
+ }
+
+ private void writeFile(FileSystem fs, Path dir, String filename, String data) throws IOException {
+ FSDataOutputStream out = fs.create(new Path(dir, filename));
+ try {
+ out.writeUTF(data);
+ } finally {
+ out.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/783e895f/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index 2f64d58..ee38efd 100644
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@ -31,6 +31,7 @@ import java.util.Map.Entry;
import java.util.SortedSet;
import java.util.TreeSet;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
@@ -49,11 +50,13 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.init.Initialize;
@@ -65,6 +68,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.ZooKeeper;
import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -298,33 +302,40 @@ public class VolumeIT extends ConfigurableMacIT {
}
private void verifyVolumesUsed(String tableName, Path... paths) throws AccumuloException, AccumuloSecurityException, TableExistsException,
- TableNotFoundException,
- MutationsRejectedException {
- TreeSet<Text> splits = new TreeSet<Text>();
- for (int i = 0; i < 100; i++) {
- splits.add(new Text(String.format("%06d", i * 100)));
- }
+ TableNotFoundException, MutationsRejectedException {
Connector conn = cluster.getConnector("root", ROOT_PASSWORD);
- conn.tableOperations().create(tableName);
- conn.tableOperations().addSplits(tableName, splits);
List<String> expected = new ArrayList<String>();
-
- BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
for (int i = 0; i < 100; i++) {
String row = String.format("%06d", i * 100 + 3);
- Mutation m = new Mutation(row);
- m.put("cf1", "cq1", "1");
- bw.addMutation(m);
expected.add(row + ":cf1:cq1:1");
}
- bw.close();
+ if (!conn.tableOperations().exists(tableName)) {
- verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
+ TreeSet<Text> splits = new TreeSet<Text>();
+ for (int i = 0; i < 100; i++) {
+ splits.add(new Text(String.format("%06d", i * 100)));
+ }
- conn.tableOperations().flush(tableName, null, null, true);
+ conn.tableOperations().create(tableName);
+ conn.tableOperations().addSplits(tableName, splits);
+
+ BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ String row = String.format("%06d", i * 100 + 3);
+ Mutation m = new Mutation(row);
+ m.put("cf1", "cq1", "1");
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
+
+ conn.tableOperations().flush(tableName, null, null, true);
+ }
verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
@@ -335,13 +346,16 @@ public class VolumeIT extends ConfigurableMacIT {
int counts[] = new int[paths.length];
- for (Entry<Key,Value> entry : metaScanner) {
+ outer: for (Entry<Key,Value> entry : metaScanner) {
String cq = entry.getKey().getColumnQualifier().toString();
for (int i = 0; i < paths.length; i++) {
if (cq.startsWith(paths[i].toString())) {
counts[i]++;
+ continue outer;
}
}
+
+ Assert.fail("Unexpected volume " + cq);
}
// if a volume is chosen randomly for each tablet, then the probability that a volume will not be chosen for any tablet is ((num_volumes -
@@ -356,4 +370,37 @@ public class VolumeIT extends ConfigurableMacIT {
Assert.assertEquals(100, sum);
}
+ @Test
+ public void testRemoveVolumes() throws Exception {
+ String[] tableNames = getTableNames(1);
+
+ verifyVolumesUsed(tableNames[0], v1, v2);
+
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ cluster.stop();
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v2.toString());
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // start cluster and verify that volume was decommisioned
+ cluster.start();
+
+ Connector conn = cluster.getConnector("root", ROOT_PASSWORD);
+ conn.tableOperations().compact(tableNames[0], null, null, true, true);
+
+ verifyVolumesUsed(tableNames[0], v2);
+
+ // check that root tablet is not on volume 1
+ String zpath = ZooUtil.getRoot(new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers())) + RootTable.ZROOT_TABLET_PATH;
+ ZooKeeper zookeeper = new ZooKeeper(cluster.getZooKeepers(), 30000, null);
+ String rootTabletDir = new String(zookeeper.getData(zpath, false, null), Constants.UTF8);
+ Assert.assertTrue(rootTabletDir.startsWith(v2.toString()));
+ zookeeper.close();
+
+ }
}