You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2014/10/29 15:45:35 UTC
[1/3] git commit: ACCUMULO-3263 use volume chooser to select
directory entries
Repository: accumulo
Updated Branches:
refs/heads/master efc75d1c5 -> 04e4312ed
ACCUMULO-3263 use volume chooser to select directory entries
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/309d9eb4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/309d9eb4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/309d9eb4
Branch: refs/heads/master
Commit: 309d9eb444f1aed1d63e48cea9966677d573d5a8
Parents: ca083a5
Author: Eric C. Newton <er...@gmail.com>
Authored: Wed Oct 29 10:05:24 2014 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Wed Oct 29 10:05:24 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/server/util/Admin.java | 2 +-
.../accumulo/server/util/RandomizeVolumes.java | 75 ++++++++++----------
.../test/RewriteTabletDirectoriesIT.java | 21 ++++--
3 files changed, 56 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/309d9eb4/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
index 5e7cb3e..bab42b5 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
@@ -208,7 +208,7 @@ public class Admin {
} else if (cl.getParsedCommand().equals("volumes")) {
ListVolumesUsed.listVolumes(instance, principal, token);
} else if (cl.getParsedCommand().equals("randomizeVolumes")) {
- RandomizeVolumes.randomize(instance, new Credentials(principal, token), randomizeVolumesOpts.table);
+ rc = RandomizeVolumes.randomize(instance.getConnector(principal, token), randomizeVolumesOpts.table);
} else {
everything = cl.getParsedCommand().equals("stopAll");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/309d9eb4/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index 7b3b6cc..afc3902 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@ -16,44 +16,48 @@
*/
package org.apache.accumulo.server.util;
+import java.io.IOException;
import java.util.Map.Entry;
+import org.apache.accumulo.server.security.SystemCredentials;
+
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.master.state.tables.TableState;
import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.Credentials;
-import org.apache.accumulo.core.util.ColumnFQ;
import org.apache.accumulo.server.ServerConstants;
-import org.apache.accumulo.server.conf.ServerConfigurationFactory;
-import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.tables.TableManager;
+import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
public class RandomizeVolumes {
private static final Logger log = Logger.getLogger(RandomizeVolumes.class);
- public static void main(String[] args) {
+ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException {
ClientOnRequiredTable opts = new ClientOnRequiredTable();
opts.parseArgs(RandomizeVolumes.class.getName(), args);
- String principal = SystemCredentials.get().getPrincipal();
- AuthenticationToken token = SystemCredentials.get().getToken();
+ Connector c;
+ if (opts.getToken() == null) {
+ SystemCredentials creds = SystemCredentials.get();
+ c = opts.getInstance().getConnector(creds.getPrincipal(), creds.getToken());
+ } else {
+ c = opts.getConnector();
+ }
try {
- int status = randomize(opts.getInstance(), new Credentials(principal, token), opts.tableName);
+ int status = randomize(c, opts.tableName);
System.exit(status);
} catch (Exception ex) {
log.error(ex, ex);
@@ -61,42 +65,34 @@ public class RandomizeVolumes {
}
}
- public static int randomize(Instance instance, Credentials credentials, String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
- SiteConfiguration siteConfiguration = new ServerConfigurationFactory(instance).getSiteConfiguration();
- String volumesConfig = siteConfiguration.get(Property.INSTANCE_VOLUMES);
- if (null == volumesConfig || "".equals(volumesConfig)) {
- log.error(Property.INSTANCE_VOLUMES.getKey() + " is not set in accumulo-site.xml");
+ public static int randomize(Connector c, String tableName) throws IOException, AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ VolumeManager vm = VolumeManagerImpl.get();
+ if (vm.getVolumes().size() < 2) {
+ log.error("There are not enough volumes configured");
return 1;
}
- String[] volumes = volumesConfig.split(",");
- if (volumes.length < 2) {
- log.error("There are not enough volumes configured: " + volumesConfig);
- return 2;
- }
- Connector c = instance.getConnector(credentials.getPrincipal(), credentials.getToken());
String tableId = c.tableOperations().tableIdMap().get(tableName);
if (null == tableId) {
log.error("Could not determine the table ID for table " + tableName);
- return 3;
+ return 2;
}
TableState tableState = TableManager.getInstance().getTableState(tableId);
if (TableState.OFFLINE != tableState) {
- log.debug("Taking " + tableName + " offline");
+ log.info("Taking " + tableName + " offline");
c.tableOperations().offline(tableName, true);
- log.debug(tableName + " offline");
+ log.info(tableName + " offline");
}
- log.debug("Rewriting entries for " + tableName);
+ log.info("Rewriting entries for " + tableName);
Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
- final ColumnFQ DIRCOL = MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
- DIRCOL.fetch(scanner);
- scanner.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+ DIRECTORY_COLUMN.fetch(scanner);
+ scanner.setRange(TabletsSection.getRange(tableId));
BatchWriter writer = c.createBatchWriter(MetadataTable.NAME, null);
int count = 0;
for (Entry<Key,Value> entry : scanner) {
String oldLocation = entry.getValue().toString();
String directory;
if (oldLocation.contains(":")) {
- String[] parts = oldLocation.split("/");
+ String[] parts = oldLocation.split(Path.SEPARATOR);
String tableIdEntry = parts[parts.length - 2];
if (!tableIdEntry.equals(tableId)) {
log.error("Unexpected table id found: " + tableIdEntry + ", expected " + tableId + "; skipping");
@@ -104,15 +100,22 @@ public class RandomizeVolumes {
}
directory = parts[parts.length - 1];
} else {
- directory = oldLocation.substring(1);
+ directory = oldLocation.substring(Path.SEPARATOR.length());
}
- Mutation m = new Mutation(entry.getKey().getRow());
- String newLocation = volumes[count % volumes.length] + "/" + ServerConstants.TABLE_DIR + "/" + tableId + "/" + directory;
- m.put(DIRCOL.getColumnFamily(), DIRCOL.getColumnQualifier(), new Value(newLocation.getBytes()));
+ Key key = entry.getKey();
+ Mutation m = new Mutation(key.getRow());
+
+ String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + directory;
+ m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes()));
if (log.isTraceEnabled()) {
log.trace("Replacing " + oldLocation + " with " + newLocation);
}
writer.addMutation(m);
+ try {
+ vm.mkdirs(new Path(newLocation));
+ } catch (IOException ex) {
+ // nevermind
+ }
count++;
}
writer.close();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/309d9eb4/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java b/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
index 4cd4a3e..d989789 100644
--- a/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
@@ -25,7 +25,6 @@ import java.util.SortedSet;
import java.util.TreeSet;
import org.apache.accumulo.core.util.ColumnFQ;
-
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.security.TablePermission;
@@ -77,11 +76,16 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
Connector c = getConnector();
c.securityOperations().grantTablePermission(c.whoami(), MetadataTable.NAME, TablePermission.WRITE);
final String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ BatchWriter bw = c.createBatchWriter(tableName, null);
final SortedSet<Text> splits = new TreeSet<Text>();
for (String split : "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",")) {
splits.add(new Text(split));
+ Mutation m = new Mutation(new Text(split));
+ m.put(new byte[]{}, new byte[]{}, new byte[]{});
+ bw.addMutation(m);
}
- c.tableOperations().create(tableName);
+ bw.close();
c.tableOperations().addSplits(tableName, splits);
BatchScanner scanner = c.createBatchScanner(MetadataTable.NAME, Authorizations.EMPTY, 1);
@@ -90,7 +94,7 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
String tableId = c.tableOperations().tableIdMap().get(tableName);
scanner.setRanges(Collections.singletonList(MetadataSchema.TabletsSection.getRange(tableId)));
// verify the directory entries are all on v1, make a few entries relative
- BatchWriter bw = c.createBatchWriter(MetadataTable.NAME, null);
+ bw = c.createBatchWriter(MetadataTable.NAME, null);
int count = 0;
for (Entry<Key,Value> entry : scanner) {
assertTrue(entry.getValue().toString().contains(v1.toString()));
@@ -106,7 +110,7 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
assertEquals(splits.size() + 1, count);
// This should fail: only one volume
- assertEquals(2, cluster.exec(RandomizeVolumes.class, "-z", cluster.getZooKeepers(), "-i", c.getInstance().getInstanceName(), "-t", tableName).waitFor());
+ assertEquals(1, cluster.exec(RandomizeVolumes.class, "-z", cluster.getZooKeepers(), "-i", c.getInstance().getInstanceName(), "-t", tableName).waitFor());
cluster.stop();
@@ -138,6 +142,13 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
}
}
assertEquals(splits.size() + 1, v1Count + v2Count);
- assertTrue(Math.abs(v1Count - v2Count) < 2);
+ assertTrue(Math.abs(v1Count - v2Count) < 10);
+ // verify we can read the old data
+ count = 0;
+ for (Entry<Key,Value> entry : c.createScanner(tableName, Authorizations.EMPTY)) {
+ assertTrue(splits.contains(entry.getKey().getRow()));
+ count++;
+ }
+ assertEquals(splits.size(), count);
}
}
[3/3] git commit: ACCUMULO-3263 merge 1.6
Posted by ec...@apache.org.
ACCUMULO-3263 merge 1.6
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/04e4312e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/04e4312e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/04e4312e
Branch: refs/heads/master
Commit: 04e4312ed947af790aaf77592c2104ff87c67f36
Parents: efc75d1 1abb862
Author: Eric C. Newton <er...@gmail.com>
Authored: Wed Oct 29 10:44:51 2014 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Wed Oct 29 10:44:51 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/server/util/Admin.java | 2 +-
.../accumulo/server/util/RandomizeVolumes.java | 75 ++++++++++----------
.../test/RewriteTabletDirectoriesIT.java | 43 ++++++-----
3 files changed, 67 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/04e4312e/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/04e4312e/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index 197c46c,afc3902..751e28b
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@@ -46,14 -45,19 +45,19 @@@ import static org.apache.accumulo.core.
public class RandomizeVolumes {
private static final Logger log = Logger.getLogger(RandomizeVolumes.class);
-
+
- public static void main(String[] args) {
+ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException {
ClientOnRequiredTable opts = new ClientOnRequiredTable();
opts.parseArgs(RandomizeVolumes.class.getName(), args);
- String principal = SystemCredentials.get().getPrincipal();
- AuthenticationToken token = SystemCredentials.get().getToken();
+ Connector c;
+ if (opts.getToken() == null) {
+ SystemCredentials creds = SystemCredentials.get();
+ c = opts.getInstance().getConnector(creds.getPrincipal(), creds.getToken());
+ } else {
+ c = opts.getConnector();
+ }
try {
- int status = randomize(opts.getInstance(), new Credentials(principal, token), opts.getTableName());
- int status = randomize(c, opts.tableName);
++ int status = randomize(c, opts.getTableName());
System.exit(status);
} catch (Exception ex) {
log.error(ex, ex);
[2/3] git commit: ACCUMULO-3263 more cleanup
Posted by ec...@apache.org.
ACCUMULO-3263 more cleanup
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1abb8629
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1abb8629
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1abb8629
Branch: refs/heads/master
Commit: 1abb8629096a7d007339684d61d5dc504bf0461f
Parents: 309d9eb
Author: Eric C. Newton <er...@gmail.com>
Authored: Wed Oct 29 10:42:07 2014 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Wed Oct 29 10:42:07 2014 -0400
----------------------------------------------------------------------
.../test/RewriteTabletDirectoriesIT.java | 22 ++++++++++----------
1 file changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1abb8629/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java b/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
index d989789..f2efa11 100644
--- a/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
@@ -24,18 +24,17 @@ import java.util.Map.Entry;
import java.util.SortedSet;
import java.util.TreeSet;
-import org.apache.accumulo.core.util.ColumnFQ;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
import org.apache.accumulo.server.init.Initialize;
import org.apache.accumulo.server.util.Admin;
@@ -46,6 +45,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.Text;
import org.junit.Test;
+
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -89,10 +90,9 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
c.tableOperations().addSplits(tableName, splits);
BatchScanner scanner = c.createBatchScanner(MetadataTable.NAME, Authorizations.EMPTY, 1);
- ColumnFQ DC = MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
- DC.fetch(scanner);
+ DIRECTORY_COLUMN.fetch(scanner);
String tableId = c.tableOperations().tableIdMap().get(tableName);
- scanner.setRanges(Collections.singletonList(MetadataSchema.TabletsSection.getRange(tableId)));
+ scanner.setRanges(Collections.singletonList(TabletsSection.getRange(tableId)));
// verify the directory entries are all on v1, make a few entries relative
bw = c.createBatchWriter(MetadataTable.NAME, null);
int count = 0;
@@ -101,8 +101,9 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
count++;
if (count % 2 == 0) {
String parts[] = entry.getValue().toString().split("/");
- Mutation m = new Mutation(entry.getKey().getRow());
- m.put(DC.getColumnFamily(), DC.getColumnQualifier(), new Value(("/" + parts[parts.length - 1]).getBytes()));
+ Key key = entry.getKey();
+ Mutation m = new Mutation(key.getRow());
+ m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value((Path.SEPARATOR + parts[parts.length - 1]).getBytes()));
bw.addMutation(m);
}
}
@@ -111,7 +112,6 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
// This should fail: only one volume
assertEquals(1, cluster.exec(RandomizeVolumes.class, "-z", cluster.getZooKeepers(), "-i", c.getInstance().getInstanceName(), "-t", tableName).waitFor());
-
cluster.stop();