You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2013/11/01 03:00:10 UTC
[09/54] git commit: ACCUMULO-802 Many small fixes. Largest change was
adding CloneNamespace which clones all the tables in a namespace to a new
namespace.
ACCUMULO-802 Many small fixes. Largest change was adding CloneNamespace which clones all the tables in a namespace to a new namespace.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a9ede22e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a9ede22e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a9ede22e
Branch: refs/heads/ACCUMULO-802
Commit: a9ede22eedcd2a320144048617f167b460721786
Parents: e56fc7e
Author: Sean Hickey <ta...@gmail.com>
Authored: Thu Jul 11 15:05:43 2013 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Thu Oct 31 21:20:18 2013 -0400
----------------------------------------------------------------------
.../client/admin/TableNamespaceOperations.java | 31 +++
.../admin/TableNamespaceOperationsImpl.java | 174 ++++++++++++-----
.../core/client/admin/TableOperationsImpl.java | 29 ++-
.../mock/MockTableNamespaceOperations.java | 62 +++---
.../apache/accumulo/core/util/shell/Shell.java | 3 +-
.../shell/commands/CloneNamespaceCommand.java | 112 +++++++++++
.../master/balancer/TableLoadBalancer.java | 26 +--
.../accumulo/server/tables/TableManager.java | 17 +-
.../accumulo/server/util/NamespacePropUtil.java | 2 +-
.../accumulo/server/util/TablePropUtil.java | 2 +-
.../java/org/apache/accumulo/master/Master.java | 56 +++++-
.../master/tableOps/CloneTableNamespace.java | 194 +++++++++++++++++++
.../accumulo/master/tableOps/RenameTable.java | 1 +
.../accumulo/test/TableNamespacesTest.java | 128 ++++++------
14 files changed, 678 insertions(+), 159 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a9ede22e/core/src/main/java/org/apache/accumulo/core/client/admin/TableNamespaceOperations.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/TableNamespaceOperations.java b/core/src/main/java/org/apache/accumulo/core/client/admin/TableNamespaceOperations.java
index 24793f0..9658dc6 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableNamespaceOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableNamespaceOperations.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.core.client.admin;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.SortedSet;
import org.apache.accumulo.core.client.AccumuloException;
@@ -228,7 +229,37 @@ public interface TableNamespaceOperations {
*
* @return a list of disk usage objects containing linked table names and sizes
* @throws AccumuloException
+ * when there is a general accumulo error
* @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
*/
public List<DiskUsage> getDiskUsage(String namespace) throws AccumuloException, AccumuloSecurityException, TableNamespaceNotFoundException;
+
+
+ /**
+ * Clone a all the tables in a table namespace to a new table namespace. Optionally copy all their properties as well.
+ *
+ * @param srcName
+ * The table namespace to clone
+ * @param newName
+ * The new table namespace to clone to
+ * @param flush
+ * Whether to flush each table before cloning
+ * @param propertiesToSet
+ * Which table namespace properties to set
+ * @param propertiesToExclude
+ * Which table namespace properties to exclude
+ * @param copyTableProps
+ * Whether to copy each table's properties
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws TableNamespaceNotFoundException
+ * If the old table namespace doesn't exist
+ * @throws TableNamespaceExistsException
+ * If the new table namespace already exists
+ */
+ public void clone(String srcName, String newName, boolean flush, Map<String,String> propertiesToSet, Set<String> propertiesToExclude, Boolean copyTableProps)
+ throws AccumuloSecurityException, AccumuloException, TableNamespaceNotFoundException, TableNamespaceExistsException;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a9ede22e/core/src/main/java/org/apache/accumulo/core/client/admin/TableNamespaceOperationsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/TableNamespaceOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/client/admin/TableNamespaceOperationsImpl.java
index a80dcf6..7df73f9 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableNamespaceOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableNamespaceOperationsImpl.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.core.client.admin;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -33,6 +34,7 @@ import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNamespaceExistsException;
import org.apache.accumulo.core.client.TableNamespaceNotEmptyException;
import org.apache.accumulo.core.client.TableNamespaceNotFoundException;
@@ -50,7 +52,7 @@ import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException
import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.master.thrift.MasterClientService;
import org.apache.accumulo.core.master.thrift.TableOperation;
-import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.security.Credentials;
import org.apache.accumulo.core.util.ArgumentChecker;
import org.apache.accumulo.core.util.ByteBufferUtil;
import org.apache.accumulo.core.util.OpTimer;
@@ -67,22 +69,22 @@ import org.apache.thrift.transport.TTransportException;
*/
public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
private Instance instance;
- private TCredentials credentials;
-
+ private Credentials credentials;
+
private static final Logger log = Logger.getLogger(TableOperations.class);
-
+
/**
* @param instance
* the connection information for this instance
* @param credentials
* the username/password for this connection
*/
- public TableNamespaceOperationsImpl(Instance instance, TCredentials credentials) {
+ public TableNamespaceOperationsImpl(Instance instance, Credentials credentials) {
ArgumentChecker.notNull(instance, credentials);
this.instance = instance;
this.credentials = credentials;
}
-
+
/**
* Retrieve a list of table namespaces in Accumulo.
*
@@ -95,7 +97,7 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
opTimer.stop("Fetched " + namespaces.size() + " table namespaces in %DURATION%");
return namespaces;
}
-
+
/**
* A method to check if a table namespace exists in Accumulo.
*
@@ -106,13 +108,13 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
@Override
public boolean exists(String namespace) {
ArgumentChecker.notNull(namespace);
-
+
OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Checking if table namespace " + namespace + " exists...");
boolean exists = TableNamespaces.getNameToIdMap(instance).containsKey(namespace);
opTimer.stop("Checked existance of " + exists + " in %DURATION%");
return exists;
}
-
+
/**
* Create a table namespace with no special configuration
*
@@ -129,7 +131,7 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
public void create(String namespace) throws AccumuloException, AccumuloSecurityException, TableNamespaceExistsException {
create(namespace, true, TimeType.MILLIS);
}
-
+
/**
* @param namespace
* the name of the table namespace
@@ -140,7 +142,7 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
public void create(String namespace, boolean limitVersion) throws AccumuloException, AccumuloSecurityException, TableNamespaceExistsException {
create(namespace, limitVersion, TimeType.MILLIS);
}
-
+
/**
* @param namespace
* the name of the table namespace
@@ -153,11 +155,11 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
public void create(String namespace, boolean limitVersion, TimeType timeType) throws AccumuloException, AccumuloSecurityException,
TableNamespaceExistsException {
ArgumentChecker.notNull(namespace, timeType);
-
+
List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(namespace.getBytes()), ByteBuffer.wrap(timeType.name().getBytes()));
-
+
Map<String,String> opts = IteratorUtil.generateInitialTableProperties(limitVersion);
-
+
try {
doTableNamespaceOperation(TableOperation.CREATE, args, opts);
} catch (TableNamespaceNotFoundException e1) {
@@ -165,13 +167,13 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
throw new RuntimeException(e1);
}
}
-
+
private long beginTableNamespaceOperation() throws ThriftSecurityException, TException {
while (true) {
MasterClientService.Iface client = null;
try {
client = MasterClient.getConnectionWithRetry(instance);
- return client.beginTableNamespaceOperation(Tracer.traceInfo(), credentials);
+ return client.beginTableNamespaceOperation(Tracer.traceInfo(), credentials.toThrift(instance));
} catch (TTransportException tte) {
log.debug("Failed to call beginTableOperation(), retrying ... ", tte);
UtilWaitThread.sleep(100);
@@ -180,14 +182,14 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
}
}
}
-
+
private void executeTableNamespaceOperation(long opid, TableOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean autoCleanUp)
throws ThriftSecurityException, TException, ThriftTableOperationException {
while (true) {
MasterClientService.Iface client = null;
try {
client = MasterClient.getConnectionWithRetry(instance);
- client.executeTableNamespaceOperation(Tracer.traceInfo(), credentials, opid, op, args, opts, autoCleanUp);
+ client.executeTableNamespaceOperation(Tracer.traceInfo(), credentials.toThrift(instance), opid, op, args, opts, autoCleanUp);
break;
} catch (TTransportException tte) {
log.debug("Failed to call executeTableOperation(), retrying ... ", tte);
@@ -197,13 +199,13 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
}
}
}
-
+
private String waitForTableNamespaceOperation(long opid) throws ThriftSecurityException, TException, ThriftTableOperationException {
while (true) {
MasterClientService.Iface client = null;
try {
client = MasterClient.getConnectionWithRetry(instance);
- return client.waitForTableNamespaceOperation(Tracer.traceInfo(), credentials, opid);
+ return client.waitForTableNamespaceOperation(Tracer.traceInfo(), credentials.toThrift(instance), opid);
} catch (TTransportException tte) {
log.debug("Failed to call waitForTableOperation(), retrying ... ", tte);
UtilWaitThread.sleep(100);
@@ -212,13 +214,13 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
}
}
}
-
+
private void finishTableNamespaceOperation(long opid) throws ThriftSecurityException, TException {
while (true) {
MasterClientService.Iface client = null;
try {
client = MasterClient.getConnectionWithRetry(instance);
- client.finishTableNamespaceOperation(Tracer.traceInfo(), credentials, opid);
+ client.finishTableNamespaceOperation(Tracer.traceInfo(), credentials.toThrift(instance), opid);
break;
} catch (TTransportException tte) {
log.debug("Failed to call finishTableOperation(), retrying ... ", tte);
@@ -228,16 +230,16 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
}
}
}
-
+
private String doTableNamespaceOperation(TableOperation op, List<ByteBuffer> args, Map<String,String> opts) throws AccumuloSecurityException,
TableNamespaceExistsException, TableNamespaceNotFoundException, AccumuloException {
return doTableNamespaceOperation(op, args, opts, true);
}
-
+
private String doTableNamespaceOperation(TableOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean wait) throws AccumuloSecurityException,
TableNamespaceExistsException, TableNamespaceNotFoundException, AccumuloException {
Long opid = null;
-
+
try {
opid = beginTableNamespaceOperation();
executeTableNamespaceOperation(opid, op, args, opts, !wait);
@@ -276,7 +278,7 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
}
}
}
-
+
/**
* Delete a table namespace if empty
*
@@ -288,14 +290,14 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
* if the user does not have permission
* @throws TableNamespaceNotFoundException
* if the table namespace does not exist
- * @throws TableNamespaceNotEmptyException
+ * @throws TableNamespaceNotEmptyException
* if the table namespaces still contains tables
*/
@Override
public void delete(String namespace) throws AccumuloException, AccumuloSecurityException, TableNamespaceNotFoundException, TableNamespaceNotEmptyException {
delete(namespace, false);
}
-
+
/**
* Delete a table namespace
*
@@ -309,18 +311,19 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
* if the user does not have permission
* @throws TableNamespaceNotFoundException
* if the table namespace does not exist
- * @throws TableNamespaceNotEmptyException
+ * @throws TableNamespaceNotEmptyException
*/
@Override
- public void delete(String namespace, boolean deleteTables) throws AccumuloException, AccumuloSecurityException, TableNamespaceNotFoundException, TableNamespaceNotEmptyException {
+ public void delete(String namespace, boolean deleteTables) throws AccumuloException, AccumuloSecurityException, TableNamespaceNotFoundException,
+ TableNamespaceNotEmptyException {
ArgumentChecker.notNull(namespace);
String namespaceId = TableNamespaces.getNamespaceId(instance, namespace);
-
+
if (namespaceId.equals(Constants.SYSTEM_TABLE_NAMESPACE_ID) || namespaceId.equals(Constants.DEFAULT_TABLE_NAMESPACE_ID)) {
String why = "Can't delete the system or default table namespace";
throw new RuntimeException(why);
}
-
+
if (TableNamespaces.getTableIds(instance, namespaceId).size() > 0) {
if (!deleteTables) {
throw new TableNamespaceNotEmptyException(namespaceId, namespace, null);
@@ -333,19 +336,84 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
}
}
}
-
+
List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(namespace.getBytes()));
Map<String,String> opts = new HashMap<String,String>();
-
+
try {
doTableNamespaceOperation(TableOperation.DELETE, args, opts);
} catch (TableNamespaceExistsException e) {
// should not happen
throw new RuntimeException(e);
}
-
+
+ }
+
+ /**
+ * Clone a all the tables in a table namespace to a new table namespace. Optionally copy all their properties as well.
+ *
+ * @param srcName
+ * The table namespace to clone
+ * @param newName
+ * The new table namespace to clone to
+ * @param flush
+ * Whether to flush each table before cloning
+ * @param propertiesToSet
+ * Which table namespace properties to set
+ * @param propertiesToExclude
+ * Which table namespace properties to exclude
+ * @param copyTableProps
+ * Whether to copy each table's properties
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws TableNamespaceNotFoundException
+ * If the old table namespace doesn't exist
+ * @throws TableNamespaceExistsException
+ * If the new table namespace already exists
+ */
+ @Override
+ public void clone(String srcName, String newName, boolean flush, Map<String,String> propertiesToSet, Set<String> propertiesToExclude, Boolean copyTableProps)
+ throws AccumuloSecurityException, AccumuloException, TableNamespaceNotFoundException, TableNamespaceExistsException {
+
+ ArgumentChecker.notNull(srcName, newName);
+
+ String namespaceId = TableNamespaces.getNamespaceId(instance, srcName);
+
+ if (propertiesToExclude == null)
+ propertiesToExclude = Collections.emptySet();
+
+ if (propertiesToSet == null)
+ propertiesToSet = Collections.emptyMap();
+
+ if (!Collections.disjoint(propertiesToExclude, propertiesToSet.keySet()))
+ throw new IllegalArgumentException("propertiesToSet and propertiesToExclude not disjoint");
+
+ String srcNamespaceId = TableNamespaces.getNamespaceId(instance, srcName);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(srcNamespaceId.getBytes()), ByteBuffer.wrap(newName.getBytes()));
+ Map<String,String> opts = new HashMap<String,String>();
+ opts.putAll(propertiesToSet);
+ for (String prop : propertiesToExclude)
+ opts.put(prop, null);
+ doTableNamespaceOperation(TableOperation.CLONE, args, opts);
+
+ for (String tableId : TableNamespaces.getTableIds(instance, namespaceId)) {
+ try {
+ String tableName = Tables.getTableName(instance, tableId);
+
+ String newTableName = newName + "." + Tables.extractTableName(tableName);
+ getTableOperations().clone(tableName, newTableName, flush, null, null);
+ } catch (TableNotFoundException e) {
+ String why = "Table (" + tableId + ") dissappeared while cloning namespace (" + srcName + ")";
+ throw new IllegalStateException(why);
+ } catch (TableExistsException e) {
+ String why = "Table somehow already existed in the newly created namespace (" + newName + ")";
+ throw new IllegalStateException(why);
+ }
+ }
}
-
+
/**
* Rename a table namespace
*
@@ -365,12 +433,12 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
@Override
public void rename(String oldNamespaceName, String newNamespaceName) throws AccumuloSecurityException, TableNamespaceNotFoundException, AccumuloException,
TableNamespaceExistsException {
-
+
List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(oldNamespaceName.getBytes()), ByteBuffer.wrap(newNamespaceName.getBytes()));
Map<String,String> opts = new HashMap<String,String>();
doTableNamespaceOperation(TableOperation.RENAME, args, opts);
}
-
+
/**
* Sets a property on a table namespace
*
@@ -388,15 +456,15 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
@Override
public void setProperty(final String namespace, final String property, final String value) throws AccumuloException, AccumuloSecurityException {
ArgumentChecker.notNull(namespace, property, value);
-
+
MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
@Override
public void execute(MasterClientService.Client client) throws Exception {
- client.setTableNamespaceProperty(Tracer.traceInfo(), credentials, namespace, property, value);
+ client.setTableNamespaceProperty(Tracer.traceInfo(), credentials.toThrift(instance), namespace, property, value);
}
});
}
-
+
/**
* Removes a property from a table namespace
*
@@ -412,15 +480,15 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
@Override
public void removeProperty(final String namespace, final String property) throws AccumuloException, AccumuloSecurityException {
ArgumentChecker.notNull(namespace, property);
-
+
MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
@Override
public void execute(MasterClientService.Client client) throws Exception {
- client.removeTableNamespaceProperty(Tracer.traceInfo(), credentials, namespace, property);
+ client.removeTableNamespaceProperty(Tracer.traceInfo(), credentials.toThrift(instance), namespace, property);
}
});
}
-
+
/**
* Gets properties of a table namespace
*
@@ -437,7 +505,7 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
return ServerClient.executeRaw(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
@Override
public Map<String,String> execute(ClientService.Client client) throws Exception {
- return client.getTableNamespaceConfiguration(Tracer.traceInfo(), credentials, namespace);
+ return client.getTableNamespaceConfiguration(Tracer.traceInfo(), credentials.toThrift(instance), namespace);
}
}).entrySet();
} catch (ThriftTableOperationException e) {
@@ -453,9 +521,9 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
} catch (Exception e) {
throw new AccumuloException(e);
}
-
+
}
-
+
/**
*
* @param namespace
@@ -468,7 +536,7 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
*/
@Override
public void offline(String namespace) throws AccumuloSecurityException, AccumuloException, TableNamespaceNotFoundException {
-
+
ArgumentChecker.notNull(namespace);
String namespaceId = TableNamespaces.getNamespaceId(instance, namespace);
try {
@@ -479,7 +547,7 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
Log.error("Table namespace (" + namespaceId + ") contains reference to table that doesn't exist");
}
}
-
+
/**
*
* @param namespace
@@ -502,7 +570,7 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
Log.warn("Table namespace (" + namespaceId + ") contains a reference to a table that doesn't exist");
}
}
-
+
/**
* Get a mapping of table namespace name to internal table namespace id.
*
@@ -512,7 +580,7 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
public Map<String,String> namespaceIdMap() {
return TableNamespaces.getNameToIdMap(instance);
}
-
+
@Override
public List<DiskUsage> getDiskUsage(String namespace) throws AccumuloException, AccumuloSecurityException, TableNamespaceNotFoundException {
Set<String> tables = new HashSet<String>();
@@ -526,9 +594,9 @@ public class TableNamespaceOperationsImpl implements TableNamespaceOperations {
}
return du;
}
-
+
private TableOperations getTableOperations() throws AccumuloException, AccumuloSecurityException {
return new TableOperationsImpl(instance, credentials);
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a9ede22e/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
index a3cf190..6fa4174 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
@@ -693,11 +693,17 @@ public class TableOperationsImpl extends TableOperationsHelper {
_flush(srcTableId, null, null, true);
if (propertiesToExclude == null)
- propertiesToExclude = Collections.emptySet();
+ propertiesToExclude = new HashSet<String>();
if (propertiesToSet == null)
propertiesToSet = Collections.emptyMap();
+
+ Set<String> nProps = getUniqueNamespaceProperties(namespace, srcTableName);
+ for (String p : nProps) {
+ propertiesToExclude.add(p);
+ }
+
if (!Collections.disjoint(propertiesToExclude, propertiesToSet.keySet()))
throw new IllegalArgumentException("propertiesToSet and propertiesToExclude not disjoint");
@@ -716,6 +722,27 @@ public class TableOperationsImpl extends TableOperationsHelper {
doTableOperation(TableOperation.CLONE, args, opts);
}
+ private Set<String> getUniqueNamespaceProperties(String namespace, String table) throws TableNotFoundException, AccumuloException {
+ Set<String> props = new HashSet<String>();
+ /*try {
+ Iterable<Entry<String,String>> n = new TableNamespaceOperationsImpl(instance, credentials).getProperties(namespace);
+ Iterable<Entry<String,String>> t = getProperties(table);
+ Map<String,String> tmap = new HashMap<String,String>();
+ for (Entry<String,String> e : t) {
+ tmap.put(e.getKey(), e.getValue());
+ }
+ for (Entry<String,String> e : n) {
+ String val = tmap.get(e.getKey());
+ if (e.getValue().equals(val)) {
+ props.add(e.getKey());
+ }
+ }
+ } catch (TableNamespaceNotFoundException e) {
+ throw new IllegalStateException(new TableNamespaceNotFoundException(null, namespace, null));
+ }*/
+ return props;
+ }
+
/**
* Rename a table
*
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a9ede22e/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableNamespaceOperations.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableNamespaceOperations.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableNamespaceOperations.java
index c2b8a39..358a980 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableNamespaceOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableNamespaceOperations.java
@@ -21,6 +21,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
@@ -35,63 +36,65 @@ import org.apache.accumulo.core.client.admin.DiskUsage;
import org.apache.accumulo.core.client.admin.TableNamespaceOperations;
import org.apache.accumulo.core.client.admin.TimeType;
import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.commons.lang.NotImplementedException;
public class MockTableNamespaceOperations implements TableNamespaceOperations {
-
+
final private MockAccumulo acu;
final private String username;
-
+
MockTableNamespaceOperations(MockAccumulo acu, String username) {
this.acu = acu;
this.username = username;
}
-
+
@Override
public SortedSet<String> list() {
return new TreeSet<String>(acu.namespaces.keySet());
}
-
+
@Override
public boolean exists(String tableName) {
return acu.namespaces.containsKey(tableName);
}
-
+
@Override
public void create(String tableName) throws AccumuloException, AccumuloSecurityException, TableNamespaceExistsException {
create(tableName, true, TimeType.MILLIS);
}
-
+
@Override
public void create(String tableName, boolean versioningIter) throws AccumuloException, AccumuloSecurityException, TableNamespaceExistsException {
create(tableName, versioningIter, TimeType.MILLIS);
}
-
+
@Override
public void create(String namespace, boolean versioningIter, TimeType timeType) throws AccumuloException, AccumuloSecurityException,
TableNamespaceExistsException {
if (!namespace.matches(Constants.VALID_TABLE_NAMESPACE_REGEX)) {
throw new IllegalArgumentException();
}
-
+
if (exists(namespace))
throw new TableNamespaceExistsException(namespace, namespace, "");
-
+
if (!exists(namespace)) {
acu.createNamespace(username, namespace);
}
acu.createTable(username, namespace, versioningIter, timeType);
}
-
+
@Override
public void delete(String namespace) throws AccumuloException, AccumuloSecurityException, TableNamespaceNotFoundException, TableNamespaceNotEmptyException {
delete(namespace, false);
}
-
+
@Override
- public void delete(String namespace, boolean deleteTables) throws AccumuloException, AccumuloSecurityException, TableNamespaceNotFoundException, TableNamespaceNotEmptyException {
+ public void delete(String namespace, boolean deleteTables) throws AccumuloException, AccumuloSecurityException, TableNamespaceNotFoundException,
+ TableNamespaceNotEmptyException {
if (!exists(namespace))
throw new TableNamespaceNotFoundException(namespace, namespace, "");
-
+
MockTableNamespace n = acu.namespaces.get(namespace);
if (!deleteTables) {
if (n.getTables(acu).size() > 0) {
@@ -100,7 +103,7 @@ public class MockTableNamespaceOperations implements TableNamespaceOperations {
} else {
for (String t : n.getTables(acu)) {
try {
- new MockConnector(username, acu, null).tableOperations().delete(t);
+ new MockTableOperations(acu, username).delete(t);
} catch (TableNotFoundException e) {
System.err.println("Table (" + e.getTableName() + ") not found while deleting namespace (" + namespace + ")");
}
@@ -108,7 +111,7 @@ public class MockTableNamespaceOperations implements TableNamespaceOperations {
}
acu.namespaces.remove(namespace);
}
-
+
@Override
public void rename(String oldNamespaceName, String newNamespaceName) throws AccumuloSecurityException, TableNamespaceNotFoundException, AccumuloException,
TableNamespaceExistsException {
@@ -116,7 +119,7 @@ public class MockTableNamespaceOperations implements TableNamespaceOperations {
throw new TableNamespaceNotFoundException(oldNamespaceName, oldNamespaceName, "");
if (exists(newNamespaceName))
throw new TableNamespaceExistsException(newNamespaceName, newNamespaceName, "");
-
+
MockTableNamespace n = acu.namespaces.get(oldNamespaceName);
for (String t : n.getTables(acu)) {
String tt = newNamespaceName + "." + Tables.extractTableName(t);
@@ -124,38 +127,38 @@ public class MockTableNamespaceOperations implements TableNamespaceOperations {
}
acu.namespaces.put(newNamespaceName, acu.namespaces.remove(oldNamespaceName));
}
-
+
@Override
public void setProperty(String namespace, String property, String value) throws AccumuloException, AccumuloSecurityException {
acu.namespaces.get(namespace).settings.put(property, value);
}
-
+
@Override
public void removeProperty(String namespace, String property) throws AccumuloException, AccumuloSecurityException {
acu.namespaces.get(namespace).settings.remove(property);
}
-
+
@Override
public Iterable<Entry<String,String>> getProperties(String namespace) throws TableNamespaceNotFoundException {
if (!exists(namespace)) {
throw new TableNamespaceNotFoundException(namespace, namespace, "");
}
-
+
return acu.namespaces.get(namespace).settings.entrySet();
}
-
+
@Override
public void offline(String namespace) throws AccumuloSecurityException, AccumuloException, TableNamespaceNotFoundException {
if (!exists(namespace))
throw new TableNamespaceNotFoundException(namespace, namespace, "");
}
-
+
@Override
public void online(String namespace) throws AccumuloSecurityException, AccumuloException, TableNamespaceNotFoundException {
if (!exists(namespace))
throw new TableNamespaceNotFoundException(namespace, namespace, "");
}
-
+
@Override
public Map<String,String> namespaceIdMap() {
Map<String,String> result = new HashMap<String,String>();
@@ -164,13 +167,20 @@ public class MockTableNamespaceOperations implements TableNamespaceOperations {
}
return result;
}
-
+
@Override
public List<DiskUsage> getDiskUsage(String namespace) throws AccumuloException, AccumuloSecurityException {
-
+
List<DiskUsage> diskUsages = new ArrayList<DiskUsage>();
diskUsages.add(new DiskUsage(new TreeSet<String>(acu.namespaces.get(namespace).getTables(acu)), 0l));
-
+
return diskUsages;
}
+
+ @Override
+ public void clone(String srcName, String newName, boolean flush, Map<String,String> propertiesToSet, Set<String> propertiesToExclude, Boolean copyTableProps)
+ throws AccumuloSecurityException, AccumuloException, TableNamespaceNotFoundException, TableNamespaceExistsException {
+ // TODO Implement clone in Mock
+ throw new NotImplementedException();
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a9ede22e/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
index cdb86f1..74a2dc9 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
@@ -68,6 +68,7 @@ import org.apache.accumulo.core.util.shell.commands.AuthenticateCommand;
import org.apache.accumulo.core.util.shell.commands.ByeCommand;
import org.apache.accumulo.core.util.shell.commands.ClasspathCommand;
import org.apache.accumulo.core.util.shell.commands.ClearCommand;
+import org.apache.accumulo.core.util.shell.commands.CloneNamespaceCommand;
import org.apache.accumulo.core.util.shell.commands.CloneTableCommand;
import org.apache.accumulo.core.util.shell.commands.ClsCommand;
import org.apache.accumulo.core.util.shell.commands.CompactCommand;
@@ -353,7 +354,7 @@ public class Shell extends ShellOptions {
new TableCommand(), new UserCommand(), new WhoAmICommand()};
Command[] tableCommands = {new CloneTableCommand(), new ConfigCommand(), new CreateTableCommand(), new DeleteTableCommand(), new DropTableCommand(),
new DUCommand(), new ExportTableCommand(), new ImportTableCommand(), new OfflineCommand(), new OnlineCommand(), new RenameTableCommand(),
- new TablesCommand(), new NamespacesCommand(), new CreateNamespaceCommand(), new DeleteNamespaceCommand(), new RenameNamespaceCommand()};
+ new TablesCommand(), new NamespacesCommand(), new CreateNamespaceCommand(), new DeleteNamespaceCommand(), new RenameNamespaceCommand(), new CloneNamespaceCommand()};
Command[] tableControlCommands = {new AddSplitsCommand(), new CompactCommand(), new ConstraintCommand(), new FlushCommand(), new GetGroupsCommand(),
new GetSplitsCommand(), new MergeCommand(), new SetGroupsCommand()};
Command[] userCommands = {new AddAuthsCommand(), new CreateUserCommand(), new DeleteUserCommand(), new DropUserCommand(), new GetAuthsCommand(),
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a9ede22e/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CloneNamespaceCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CloneNamespaceCommand.java b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CloneNamespaceCommand.java
new file mode 100644
index 0000000..21f5a9f
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CloneNamespaceCommand.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util.shell.commands;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNamespaceExistsException;
+import org.apache.accumulo.core.client.TableNamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.util.shell.Shell;
+import org.apache.accumulo.core.util.shell.Shell.Command;
+import org.apache.accumulo.core.util.shell.Token;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+
+public class CloneNamespaceCommand extends Command {
+
+ private Option setPropsOption;
+ private Option excludePropsOption;
+ private Option noFlushOption;
+ private Option copyTablePropsOption;
+
+ @Override
+ public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
+ TableExistsException, TableNamespaceNotFoundException, TableNamespaceExistsException {
+
+ final HashMap<String,String> props = new HashMap<String,String>();
+ final HashSet<String> exclude = new HashSet<String>();
+ boolean flush = true;
+ boolean copyTableProps = false;
+
+ if (cl.hasOption(setPropsOption.getOpt())) {
+ String[] keyVals = cl.getOptionValue(setPropsOption.getOpt()).split(",");
+ for (String keyVal : keyVals) {
+ String[] sa = keyVal.split("=");
+ props.put(sa[0], sa[1]);
+ }
+ }
+
+ if (cl.hasOption(excludePropsOption.getOpt())) {
+ String[] keys = cl.getOptionValue(excludePropsOption.getOpt()).split(",");
+ for (String key : keys) {
+ exclude.add(key);
+ }
+ }
+
+ if (cl.hasOption(noFlushOption.getOpt())) {
+ flush = false;
+ }
+
+ if (cl.hasOption(noFlushOption.getOpt())) {
+ copyTableProps = true;
+ }
+
+ shellState.getConnector().tableNamespaceOperations().clone(cl.getArgs()[0], cl.getArgs()[1], flush, props, exclude, copyTableProps);
+ return 0;
+ }
+
+ @Override
+ public String usage() {
+ return getName() + " <current name> <new name>";
+ }
+
+ @Override
+ public String description() {
+ return "clones a table namespace";
+ }
+
+ public void registerCompletion(final Token root, final Map<Command.CompletionSet,Set<String>> completionSet) {
+ registerCompletionForTableNamespaces(root, completionSet);
+ }
+
+ @Override
+ public Options getOptions() {
+ final Options o = new Options();
+ setPropsOption = new Option("s", "set", true, "set initial properties. Expects <prop>=<value>{,<prop>=<value>}");
+ o.addOption(setPropsOption);
+ excludePropsOption = new Option("e", "exclude", true, "exclude properties that should not be copied from source. Expects <prop>{,<prop>}");
+ o.addOption(excludePropsOption);
+ noFlushOption = new Option("nf", "noFlush", false, "do not flush table data in memory before cloning.");
+ o.addOption(noFlushOption);
+ copyTablePropsOption = new Option("tp", "copyTableProps", false, "copy each table's properties to the cloned table in the new namespace.");
+ o.addOption(copyTablePropsOption);
+ return o;
+ }
+
+ @Override
+ public int numArgs() {
+ return 2;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a9ede22e/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
index 9de6ec2..17b2cf9 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
@@ -35,33 +35,33 @@ import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.master.state.TabletMigration;
import org.apache.accumulo.server.security.SystemCredentials;
-import org.apache.accumulo.server.master.state.tables.TableManager;
+import org.apache.accumulo.server.tables.TableManager;
import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.log4j.Logger;
public class TableLoadBalancer extends TabletBalancer {
-
+
private static final Logger log = Logger.getLogger(TableLoadBalancer.class);
-
+
Map<String,TabletBalancer> perTableBalancers = new HashMap<String,TabletBalancer>();
-
+
private TabletBalancer constructNewBalancerForTable(String clazzName, String table) throws Exception {
Class<? extends TabletBalancer> clazz = AccumuloVFSClassLoader.loadClass(clazzName, TabletBalancer.class);
Constructor<? extends TabletBalancer> constructor = clazz.getConstructor(String.class);
return constructor.newInstance(table);
}
-
+
protected String getLoadBalancerClassNameForTable(String table) {
if (TableManager.getInstance().getTableState(table).equals(TableState.ONLINE))
return configuration.getTableConfiguration(table).get(Property.TABLE_LOAD_BALANCER);
return null;
}
-
+
protected TabletBalancer getBalancerForTable(String table) {
TabletBalancer balancer = perTableBalancers.get(table);
-
+
String clazzName = getLoadBalancerClassNameForTable(table);
-
+
if (clazzName == null)
clazzName = DefaultLoadBalancer.class.getName();
if (balancer != null) {
@@ -87,7 +87,7 @@ public class TableLoadBalancer extends TabletBalancer {
} catch (Exception e) {
log.warn("Failed to load table balancer class " + clazzName + " for table " + table, e);
}
-
+
if (balancer == null) {
log.info("Using balancer " + DefaultLoadBalancer.class.getName() + " for table " + table);
balancer = new DefaultLoadBalancer(table);
@@ -97,7 +97,7 @@ public class TableLoadBalancer extends TabletBalancer {
}
return balancer;
}
-
+
@Override
public void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current, Map<KeyExtent,TServerInstance> unassigned,
Map<KeyExtent,TServerInstance> assignments) {
@@ -117,9 +117,9 @@ public class TableLoadBalancer extends TabletBalancer {
assignments.putAll(newAssignments);
}
}
-
+
private TableOperations tops = null;
-
+
protected TableOperations getTableOperations() {
if (tops == null)
try {
@@ -131,7 +131,7 @@ public class TableLoadBalancer extends TabletBalancer {
}
return tops;
}
-
+
@Override
public long balance(SortedMap<TServerInstance,TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
long minBalanceTime = 5 * 1000;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a9ede22e/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
index aed6321..911459c 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
@@ -34,6 +34,7 @@ import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.util.NamespacePropUtil;
import org.apache.accumulo.server.util.TablePropUtil;
import org.apache.accumulo.server.zookeeper.ZooCache;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
@@ -297,7 +298,7 @@ public class TableManager {
String zPath = Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZNAMESPACES + "/" + namespaceId;
IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
-
+
zoo.putPersistentData(zPath, new byte[0], existsPolicy);
zoo.putPersistentData(zPath + Constants.ZNAMESPACE_NAME, namespace.getBytes(Constants.UTF8), existsPolicy);
zoo.putPersistentData(zPath + Constants.ZNAMESPACE_CONF, new byte[0], existsPolicy);
@@ -318,6 +319,20 @@ public class TableManager {
ZooReaderWriter.getRetryingInstance().putPersistentData(zPath, Constants.DEFAULT_TABLE_NAMESPACE.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
}
+ public void cloneNamespace(String srcId, String newId, String namespaceName, Map<String,String> propertiesToSet, Set<String> propertiesToExclude,
+ NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException {
+ String srcPath = Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZNAMESPACES + "/" + srcId + Constants.ZNAMESPACE_CONF;
+ String newPath = Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZNAMESPACES + "/" + newId + Constants.ZNAMESPACE_CONF;
+ ZooReaderWriter.getRetryingInstance().recursiveCopyPersistent(srcPath, newPath, NodeExistsPolicy.OVERWRITE);
+
+ for (Entry<String,String> entry : propertiesToSet.entrySet())
+ NamespacePropUtil.setNamespaceProperty(newId, entry.getKey(), entry.getValue());
+
+ for (String prop : propertiesToExclude)
+ ZooReaderWriter.getRetryingInstance().recursiveDelete(
+ Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZNAMESPACES + "/" + newId + Constants.ZNAMESPACE_CONF + "/" + prop,
+ NodeMissingPolicy.SKIP);
+ }
/*
* private static boolean verifyTabletAssignments(String tableId) { log.info( "Sending message to load balancer to verify assignment of tablets with tableId="
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a9ede22e/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
index a3ff33a..61ba133 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
@@ -36,7 +36,7 @@ public class NamespacePropUtil {
// create the zk node for this property and set it's data to the specified value
String zPath = zkNamespacePath + "/" + property;
- ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(), NodeExistsPolicy.OVERWRITE);
+ ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
return true;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a9ede22e/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
index cdee8fb..bcaf9b0 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
@@ -36,7 +36,7 @@ public class TablePropUtil {
// create the zk node for this property and set it's data to the specified value
String zPath = zkTablePath + "/" + property;
- ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(), NodeExistsPolicy.OVERWRITE);
+ ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
return true;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a9ede22e/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index e9910d5..3325778 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -97,6 +97,7 @@ import org.apache.accumulo.master.tableOps.BulkImport;
import org.apache.accumulo.master.tableOps.CancelCompactions;
import org.apache.accumulo.master.tableOps.ChangeTableState;
import org.apache.accumulo.master.tableOps.CloneTable;
+import org.apache.accumulo.master.tableOps.CloneTableNamespace;
import org.apache.accumulo.master.tableOps.CompactRange;
import org.apache.accumulo.master.tableOps.CreateTable;
import org.apache.accumulo.master.tableOps.CreateTableNamespace;
@@ -1116,8 +1117,8 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt
case CREATE: {
String namespace = ByteBufferUtil.toString(arguments.get(0));
// TODO security check once namespace permissions exist (ACCUMULO-1479)
-
- checkTableNamespaceName(namespace);
+ checkNotSystemNamespace(namespace, TableOperation.CREATE);
+ checkTableNamespaceName(namespace, TableOperation.CREATE);
fate.seedTransaction(opid, new TraceRepo<Master>(new CreateTableNamespace(c.getPrincipal(), namespace, options)), autoCleanup);
break;
}
@@ -1127,31 +1128,74 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt
String newName = ByteBufferUtil.toString(arguments.get(1));
// TODO security check (ACCUMULO-1479)
String namespaceId = checkNamespaceId(oldName, TableOperation.RENAME);
- checkTableNamespaceName(newName);
+ checkNotSystemNamespace(oldName, TableOperation.RENAME);
+ checkNotSystemNamespace(newName, TableOperation.RENAME);
+ checkTableNamespaceName(newName, TableOperation.RENAME);
fate.seedTransaction(opid, new TraceRepo<Master>(new RenameTableNamespace(namespaceId, oldName, newName)), autoCleanup);
break;
}
case DELETE: {
String namespace = ByteBufferUtil.toString(arguments.get(0));
+ checkNotSystemNamespace(namespace, TableOperation.DELETE);
String namespaceId = checkNamespaceId(namespace, TableOperation.DELETE);
// TODO security check (ACCUMULO-1479)
fate.seedTransaction(opid, new TraceRepo<Master>(new DeleteTableNamespace(namespaceId)), autoCleanup);
break;
}
+ case CLONE: {
+ String namespaceId = ByteBufferUtil.toString(arguments.get(0));
+ String namespace = ByteBufferUtil.toString(arguments.get(1));
+ checkNotSystemNamespace(namespace, TableOperation.CLONE);
+ checkTableNamespaceName(namespace, TableOperation.CLONE);
+ // TODO security check (ACCUMULO-1479)
+
+ Map<String,String> propertiesToSet = new HashMap<String,String>();
+ Set<String> propertiesToExclude = new HashSet<String>();
+
+ for (Entry<String,String> entry : options.entrySet()) {
+ if (entry.getValue() == null) {
+ propertiesToExclude.add(entry.getKey());
+ continue;
+ }
+ if (!TablePropUtil.isPropertyValid(entry.getKey(), entry.getValue())) {
+ throw new ThriftTableOperationException(null, namespace, TableOperation.CLONE, TableOperationExceptionType.OTHER, "Property or value not valid "
+ + entry.getKey() + "=" + entry.getValue());
+ }
+ propertiesToSet.put(entry.getKey(), entry.getValue());
+ }
+
+ fate.seedTransaction(opid, new TraceRepo<Master>(new CloneTableNamespace(c.getPrincipal(), namespaceId, namespace, propertiesToSet,
+ propertiesToExclude)), autoCleanup);
+
+ break;
+ }
default:
throw new UnsupportedOperationException();
}
}
- protected void checkTableNamespaceName(String namespace) throws ThriftTableOperationException {
+ private void checkNotSystemNamespace(String namespace, TableOperation operation) throws ThriftTableOperationException {
+ if (Constants.SYSTEM_TABLE_NAMESPACE.equals(namespace)) {
+ String why = "Table namespaces cannot be == " + Constants.SYSTEM_TABLE_NAMESPACE;
+ log.warn(why);
+ throw new ThriftTableOperationException(null, namespace, operation, TableOperationExceptionType.OTHER, why);
+ }
+ }
+
+ private void checkTableNamespaceName(String namespace, TableOperation operation) throws ThriftTableOperationException {
+ if (!namespace.matches(Constants.VALID_TABLE_NAMESPACE_REGEX)) {
+ String why = "Table namespaces must only contain word characters (letters, digits, and underscores): " + namespace;
+ log.warn(why);
+ throw new ThriftTableOperationException(null, namespace, operation, TableOperationExceptionType.OTHER, why);
+ }
if (TableNamespaces.getNameToIdMap(instance).containsKey(namespace)) {
String why = "Table namespace already exists: " + namespace;
- throw new ThriftTableOperationException(null, namespace, TableOperation.CREATE, TableOperationExceptionType.EXISTS, why);
+ throw new ThriftTableOperationException(null, namespace, operation, TableOperationExceptionType.EXISTS, why);
}
}
- protected String checkNamespaceId(String namespace, TableOperation operation) throws ThriftTableOperationException {
+ private String checkNamespaceId(String namespace, TableOperation operation) throws ThriftTableOperationException {
final String namespaceId = TableNamespaces.getNameToIdMap(getConfiguration().getInstance()).get(namespace);
if (namespaceId == null)
throw new ThriftTableOperationException(null, namespace, operation, TableOperationExceptionType.NOTFOUND, null);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a9ede22e/server/master/src/main/java/org/apache/accumulo/master/tableOps/CloneTableNamespace.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CloneTableNamespace.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CloneTableNamespace.java
new file mode 100644
index 0000000..9b661f2
--- /dev/null
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CloneTableNamespace.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tableOps;
+
+import java.io.Serializable;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.tables.TableManager;
+import org.apache.log4j.Logger;
+
+class CloneNamespaceInfo implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ String srcId;
+ String namespace;
+ String newId;
+ Map<String,String> propertiesToSet;
+ Set<String> propertiesToExclude;
+
+ public String user;
+}
+
+class FinishCloneTableNamespace extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+ private CloneNamespaceInfo cloneInfo;
+
+ public FinishCloneTableNamespace(CloneNamespaceInfo cloneInfo) {
+ this.cloneInfo = cloneInfo;
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return 0;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master environment) throws Exception {
+ Utils.unreserveTableNamespace(cloneInfo.srcId, tid, false);
+ Utils.unreserveTableNamespace(cloneInfo.newId, tid, true);
+
+ environment.getEventCoordinator().event("Cloned table namespace %s from %s", cloneInfo.namespace, cloneInfo.srcId);
+
+ Logger.getLogger(FinishCloneTableNamespace.class).debug("Cloned table namespace " + cloneInfo.srcId + " " + cloneInfo.newId + " " + cloneInfo.namespace);
+
+ return null;
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {}
+}
+
+class CloneNamespaceZookeeper extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private CloneNamespaceInfo cloneInfo;
+
+ public CloneNamespaceZookeeper(CloneNamespaceInfo cloneInfo) {
+ this.cloneInfo = cloneInfo;
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return Utils.reserveTableNamespace(cloneInfo.newId, tid, true, false, TableOperation.CLONE);
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master environment) throws Exception {
+ Utils.tableNameLock.lock();
+ try {
+ // write namespace to zookeeper
+ Instance instance = HdfsZooInstance.getInstance();
+
+ Utils.checkTableNamespaceDoesNotExist(instance, cloneInfo.namespace, cloneInfo.newId, TableOperation.CLONE);
+
+ TableManager.getInstance().addNamespace(cloneInfo.newId, cloneInfo.namespace, NodeExistsPolicy.FAIL);
+ TableManager.getInstance().cloneNamespace(cloneInfo.srcId, cloneInfo.newId, cloneInfo.namespace, cloneInfo.propertiesToSet,
+ cloneInfo.propertiesToExclude, NodeExistsPolicy.OVERWRITE);
+ Tables.clearCache(instance);
+
+ return new FinishCloneTableNamespace(cloneInfo);
+ } finally {
+ Utils.tableNameLock.unlock();
+ }
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {
+ Instance instance = HdfsZooInstance.getInstance();
+ TableManager.getInstance().removeNamespace(cloneInfo.newId);
+ Utils.unreserveTableNamespace(cloneInfo.newId, tid, true);
+ Tables.clearCache(instance);
+ }
+}
+
+class CloneNamespacePermissions extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private CloneNamespaceInfo cloneInfo;
+
+ public CloneNamespacePermissions(CloneNamespaceInfo cloneInfo) {
+ this.cloneInfo = cloneInfo;
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return 0;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master environment) throws Exception {
+ // TODO add table namespace permissions (ACCUMULO-1479)
+ // give all table permissions to the creator
+ /*
+ * for (TablePermission permission : TablePermission.values()) { try {
+ * AuditedSecurityOperation.getInstance().grantTablePermission(SecurityConstants.getSystemCredentials(), cloneInfo.user, cloneInfo.newId, permission); }
+ * catch (ThriftSecurityException e) { Logger.getLogger(FinishCloneTableNamespace.class).error(e.getMessage(), e); throw e; } }
+ */
+
+ // setup permissions in zookeeper before table info in zookeeper
+ // this way concurrent users will not get a spurious pemission denied
+ // error
+ return new CloneNamespaceZookeeper(cloneInfo);
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {
+
+ }
+}
+
+public class CloneTableNamespace extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+ private CloneNamespaceInfo cloneInfo;
+
+ public CloneTableNamespace(String user, String srcId, String namespace, Map<String,String> propertiesToSet, Set<String> propertiesToExclude) {
+ cloneInfo = new CloneNamespaceInfo();
+ cloneInfo.user = user;
+ cloneInfo.srcId = srcId;
+ cloneInfo.namespace = namespace;
+ cloneInfo.propertiesToExclude = propertiesToExclude;
+ cloneInfo.propertiesToSet = propertiesToSet;
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return Utils.reserveTableNamespace(cloneInfo.srcId, tid, false, true, TableOperation.CLONE);
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master environment) throws Exception {
+
+ Utils.idLock.lock();
+ try {
+ Instance instance = HdfsZooInstance.getInstance();
+ cloneInfo.newId = Utils.getNextTableId(cloneInfo.namespace, instance);
+ return new CloneNamespacePermissions(cloneInfo);
+ } finally {
+ Utils.idLock.unlock();
+ }
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {
+ Utils.unreserveTableNamespace(cloneInfo.srcId, tid, false);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a9ede22e/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
index 900d9ea..aa0886b 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
@@ -63,6 +63,7 @@ public class RenameTable extends MasterRepo {
if (!namespaceId.equals(oldNamespaceId)) {
TableManager tm = TableManager.getInstance();
tm.addNamespaceToTable(tableId, namespaceId);
+ // TODO change parent of table's configuration to new namespace...somehow...
}
newTableName = Tables.extractTableName(newTableName);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a9ede22e/test/src/test/java/org/apache/accumulo/test/TableNamespacesTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/TableNamespacesTest.java b/test/src/test/java/org/apache/accumulo/test/TableNamespacesTest.java
index 8c1a54a..6527b13 100644
--- a/test/src/test/java/org/apache/accumulo/test/TableNamespacesTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/TableNamespacesTest.java
@@ -20,13 +20,18 @@ package org.apache.accumulo.test;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.TableNamespaceNotEmptyException;
+import org.apache.accumulo.core.client.TableNamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.impl.TableNamespaces;
import org.apache.accumulo.core.client.impl.Tables;
import org.apache.accumulo.core.conf.Property;
@@ -144,37 +149,15 @@ public class TableNamespacesTest {
c.tableNamespaceOperations().setProperty(namespace, propKey, propVal);
// check the namespace has the property
- boolean itWorked = false;
- for (Entry<String,String> prop : c.tableNamespaceOperations().getProperties(namespace)) {
- if (prop.getKey().equals(propKey) && prop.getValue().equals(propVal)) {
- itWorked = true;
- break;
- }
- }
-
- assertTrue(itWorked);
+ assertTrue(checkTableNamespaceHasProp(c, namespace, propKey, propVal));
// check that the table gets it from the namespace
- itWorked = false;
- for (Entry<String,String> prop : c.tableOperations().getProperties(tableName1)) {
- if (prop.getKey().equals(propKey) && prop.getValue().equals(propVal)) {
- itWorked = true;
- break;
- }
- }
- assertTrue(itWorked);
+ assertTrue(checkTableHasProp(c, tableName1, propKey, propVal));
// test a second table to be sure the first wasn't magical
// (also, changed the order, the namespace already exists with the property)
- itWorked = false;
c.tableOperations().create(tableName2);
- for (Entry<String,String> prop : c.tableOperations().getProperties(tableName2)) {
- if (prop.getKey().equals(propKey) && prop.getValue().equals(propVal)) {
- itWorked = true;
- break;
- }
- }
- assertTrue(itWorked);
+ assertTrue(checkTableHasProp(c, tableName2, propKey, propVal));
// test that table properties override namespace properties
String propKey2 = Property.TABLE_FILE_MAX.getKey();
@@ -184,30 +167,15 @@ public class TableNamespacesTest {
c.tableOperations().setProperty(tableName2, propKey2, tablePropVal);
c.tableNamespaceOperations().setProperty("propchange", propKey2, propVal2);
- itWorked = false;
- for (Entry<String,String> prop : c.tableOperations().getProperties(tableName2)) {
- if (prop.getKey().equals(propKey2) && prop.getValue().equals(tablePropVal)) {
- itWorked = true;
- break;
- }
- }
- assertTrue(itWorked);
+ assertTrue(checkTableHasProp(c, tableName2, propKey2, tablePropVal));
// now check that you can change the default namespace's properties
propVal = "13K";
- propVal2 = "44";
String tableName = "some_table";
c.tableOperations().create(tableName);
c.tableNamespaceOperations().setProperty(Constants.DEFAULT_TABLE_NAMESPACE, propKey, propVal);
- itWorked = false;
- for (Entry<String,String> prop : c.tableOperations().getProperties(tableName)) {
- if (prop.getKey().equals(propKey) && prop.getValue().equals(propVal)) {
- itWorked = true;
- break;
- }
- }
- assertTrue(itWorked);
+ assertTrue(checkTableHasProp(c, tableName, propKey, propVal));
}
/**
@@ -301,27 +269,75 @@ public class TableNamespacesTest {
c.tableOperations().removeProperty(t1, Property.TABLE_FILE_MAX.getKey());
c.tableNamespaceOperations().setProperty(n1, propKey, propVal1);
- boolean itWorked = false;
- for (Entry<String,String> prop : c.tableOperations().getProperties(t1)) {
- if (prop.getKey().equals(propKey) && prop.getValue().equals(propVal1)) {
- itWorked = true;
- break;
- }
- }
- assertTrue(itWorked);
+ assertTrue(checkTableHasProp(c, t1, propKey, propVal1));
c.tableNamespaceOperations().create(n2);
c.tableNamespaceOperations().setProperty(n2, propKey, propVal2);
c.tableOperations().clone(t1, t2, true, null, null);
c.tableOperations().removeProperty(t2, propKey);
- itWorked = false;
- for (Entry<String,String> prop : c.tableOperations().getProperties(t2)) {
- if (prop.getKey().equals(propKey) && prop.getValue().equals(propVal2)) {
- itWorked = true;
- break;
+ assertTrue(checkTableHasProp(c, t2, propKey, propVal2));
+
+ c.tableNamespaceOperations().delete(n1, true);
+ c.tableNamespaceOperations().delete(n2, true);
+ }
+
+ /**
+ * This test clones namespaces. First checks to see that the properties were correctly copied over, then checks to see that the correct properties were set
+ * when given that option and that table properties copy successfully.
+ */
+ @Test
+ public void testCloneNamespace() throws Exception {
+ String n1 = "nspace1";
+ String n2 = "nspace2";
+ String n3 = "nspace3";
+ String t = ".table";
+
+ String propKey1 = Property.TABLE_FILE_MAX.getKey();
+ String propKey2 = Property.TABLE_SCAN_MAXMEM.getKey();
+ String propVal1 = "55";
+ String propVal2 = "66";
+ String propVal3 = "77K";
+
+ Connector c = accumulo.getConnector("root", secret);
+ c.tableNamespaceOperations().create(n1);
+ c.tableOperations().create(n1 + t);
+
+ c.tableNamespaceOperations().setProperty(n1, propKey1, propVal1);
+ c.tableOperations().setProperty(n1 + t, propKey1, propVal2);
+ c.tableNamespaceOperations().setProperty(n1, propKey2, propVal3);
+
+ c.tableNamespaceOperations().clone(n1, n2, false, null, null, false);
+ assertTrue(c.tableNamespaceOperations().exists(n2));
+ assertTrue(checkTableNamespaceHasProp(c, n2, propKey1, propVal1));
+ assertTrue(checkTableHasProp(c, n2 + t, propKey1, propVal2));
+ assertTrue(checkTableNamespaceHasProp(c, n2, propKey2, propVal3));
+
+ Map<String,String> propsToSet = new HashMap<String,String>();
+ propsToSet.put(propKey1, propVal1);
+ c.tableNamespaceOperations().clone(n1, n3, true, propsToSet, null, true);
+
+ assertTrue(checkTableNamespaceHasProp(c, n3, propKey1, propVal1));
+ assertTrue(checkTableHasProp(c, n3 + t, propKey1, propVal2));
+ assertTrue(checkTableNamespaceHasProp(c, n3, propKey2, propVal3));
+ assertTrue(!checkTableHasProp(c, n3 + t, propKey2, propVal3));
+ }
+
+ private boolean checkTableHasProp(Connector c, String t, String propKey, String propVal) throws AccumuloException, TableNotFoundException {
+ for (Entry<String,String> e : c.tableOperations().getProperties(t)) {
+ if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean checkTableNamespaceHasProp(Connector c, String n, String propKey, String propVal) throws AccumuloException, TableNamespaceNotFoundException {
+ for (Entry<String,String> e : c.tableNamespaceOperations().getProperties(n)) {
+ if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) {
+ return true;
}
}
- assertTrue(itWorked);
+ return false;
}
}