You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2015/01/09 03:44:23 UTC
[19/66] [abbrv] accumulo git commit: ACCUMULO-3451 Format master
branch (1.7.0-SNAPSHOT)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
index 29e4939..42d1313 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Logger;
/**
- *
+ *
*/
public class SecurityUtil {
private static final Logger log = Logger.getLogger(SecurityUtil.class);
@@ -39,13 +39,13 @@ public class SecurityUtil {
String keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB);
if (keyTab == null || keyTab.length() == 0)
return;
-
+
usingKerberos = true;
-
+
String principalConfig = acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL);
if (principalConfig == null || principalConfig.length() == 0)
return;
-
+
if (login(principalConfig, keyTab)) {
try {
// This spawns a thread to periodically renew the logged in (accumulo) user
@@ -58,10 +58,10 @@ public class SecurityUtil {
throw new RuntimeException("Failed to perform Kerberos login for " + principalConfig + " using " + keyTab);
}
-
+
/**
* This will log in the given user in kerberos.
- *
+ *
* @param principalConfig
* This is the principals name in the format NAME/HOST@REALM. {@link org.apache.hadoop.security.SecurityUtil#HOSTNAME_PATTERN} will automatically be
* replaced by the systems host name.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java b/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
index a59d57c..79201b1 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.io.Writable;
/**
* Credentials for the system services.
- *
+ *
* @since 1.6.0
*/
public final class SystemCredentials extends Credentials {
@@ -77,7 +77,7 @@ public final class SystemCredentials extends Credentials {
/**
* An {@link AuthenticationToken} type for Accumulo servers for inter-server communication.
- *
+ *
* @since 1.6.0
*/
public static final class SystemToken extends PasswordToken {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/security/handler/Authenticator.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/handler/Authenticator.java b/server/base/src/main/java/org/apache/accumulo/server/security/handler/Authenticator.java
index 5f0da82..092b92d 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/Authenticator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/Authenticator.java
@@ -29,27 +29,27 @@ import org.apache.accumulo.core.security.thrift.TCredentials;
*/
public interface Authenticator {
-
+
void initialize(String instanceId, boolean initialize);
-
+
boolean validSecurityHandlers(Authorizor auth, PermissionHandler pm);
-
+
void initializeSecurity(TCredentials credentials, String principal, byte[] token) throws AccumuloSecurityException, ThriftSecurityException;
-
+
boolean authenticateUser(String principal, AuthenticationToken token) throws AccumuloSecurityException;
-
+
Set<String> listUsers() throws AccumuloSecurityException;
-
+
void createUser(String principal, AuthenticationToken token) throws AccumuloSecurityException;
-
+
void dropUser(String user) throws AccumuloSecurityException;
-
+
void changePassword(String principal, AuthenticationToken token) throws AccumuloSecurityException;
-
+
boolean userExists(String user) throws AccumuloSecurityException;
-
+
Set<Class<? extends AuthenticationToken>> getSupportedTokenTypes();
-
+
/**
* Returns true if the given token is appropriate for this Authenticator
*/
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/security/handler/Authorizor.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/handler/Authorizor.java b/server/base/src/main/java/org/apache/accumulo/server/security/handler/Authorizor.java
index 5131bd3..abb579d 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/Authorizor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/Authorizor.java
@@ -29,27 +29,27 @@ import org.apache.accumulo.core.security.thrift.TCredentials;
* Accumulo, it should throw an AccumuloSecurityException with the error code UNSUPPORTED_OPERATION
*/
public interface Authorizor {
-
+
/**
* Sets up the authorizor for a new instance of Accumulo
*/
void initialize(String instanceId, boolean initialize);
-
+
/**
* Used to validate that the Authorizor, Authenticator, and permission handler can coexist
*/
boolean validSecurityHandlers(Authenticator auth, PermissionHandler pm);
-
+
/**
* Used to initialize security for the root user
*/
void initializeSecurity(TCredentials credentials, String rootuser) throws AccumuloSecurityException, ThriftSecurityException;
-
+
/**
* Used to change the authorizations for the user
*/
void changeAuthorizations(String user, Authorizations authorizations) throws AccumuloSecurityException;
-
+
/**
* Used to get the authorizations for the user
*/
@@ -59,12 +59,12 @@ public interface Authorizor {
* Used to check if a user has valid auths.
*/
boolean isValidAuthorizations(String user, List<ByteBuffer> list) throws AccumuloSecurityException;
-
+
/**
* Initializes a new user
*/
void initUser(String user) throws AccumuloSecurityException;
-
+
/**
* Deletes a user
*/
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/security/handler/InsecureAuthenticator.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/handler/InsecureAuthenticator.java b/server/base/src/main/java/org/apache/accumulo/server/security/handler/InsecureAuthenticator.java
index 5b374d9..a57608c 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/InsecureAuthenticator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/InsecureAuthenticator.java
@@ -30,47 +30,42 @@ import org.apache.accumulo.core.security.thrift.TCredentials;
* primarily for testing, but can also be used for any system where user space management is not a concern.
*/
public class InsecureAuthenticator implements Authenticator {
-
+
@Override
- public void initialize(String instanceId, boolean initialize) {
- }
-
+ public void initialize(String instanceId, boolean initialize) {}
+
@Override
public boolean validSecurityHandlers(Authorizor auth, PermissionHandler pm) {
return true;
}
-
+
@Override
- public void initializeSecurity(TCredentials credentials, String principal, byte[] token) throws AccumuloSecurityException {
- }
-
+ public void initializeSecurity(TCredentials credentials, String principal, byte[] token) throws AccumuloSecurityException {}
+
@Override
public boolean authenticateUser(String principal, AuthenticationToken token) {
return token instanceof NullToken;
}
-
+
@Override
public Set<String> listUsers() throws AccumuloSecurityException {
return Collections.emptySet();
}
-
+
@Override
- public void createUser(String principal, AuthenticationToken token) throws AccumuloSecurityException {
- }
-
+ public void createUser(String principal, AuthenticationToken token) throws AccumuloSecurityException {}
+
@Override
- public void dropUser(String user) throws AccumuloSecurityException {
- }
-
+ public void dropUser(String user) throws AccumuloSecurityException {}
+
@Override
- public void changePassword(String user, AuthenticationToken token) throws AccumuloSecurityException {
- }
-
+ public void changePassword(String user, AuthenticationToken token) throws AccumuloSecurityException {}
+
@Override
public boolean userExists(String user) {
return true;
}
-
+
@Override
public boolean validTokenClass(String tokenClass) {
return tokenClass.equals(NullToken.class.getName());
@@ -82,5 +77,5 @@ public class InsecureAuthenticator implements Authenticator {
cs.add(NullToken.class);
return cs;
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java b/server/base/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
index dfe3c88..b4e5e25 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
@@ -19,8 +19,8 @@ package org.apache.accumulo.server.security.handler;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.NamespaceNotFoundException;
import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.security.thrift.TCredentials;
@@ -28,68 +28,59 @@ import org.apache.accumulo.core.security.thrift.TCredentials;
* This is a Permission Handler implementation that doesn't actually do any security. Use at your own risk.
*/
public class InsecurePermHandler implements PermissionHandler {
-
+
@Override
- public void initialize(String instanceId, boolean initialize) {
- }
-
+ public void initialize(String instanceId, boolean initialize) {}
+
@Override
public boolean validSecurityHandlers(Authenticator authent, Authorizor author) {
return true;
}
-
+
@Override
- public void initializeSecurity(TCredentials token, String rootuser) throws AccumuloSecurityException {
- }
-
+ public void initializeSecurity(TCredentials token, String rootuser) throws AccumuloSecurityException {}
+
@Override
public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
return true;
}
-
+
@Override
public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
return true;
}
-
+
@Override
public boolean hasTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
return true;
}
-
+
@Override
public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
return true;
}
-
+
@Override
- public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
- }
-
+ public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {}
+
@Override
- public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
- }
-
+ public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {}
+
@Override
- public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
- }
-
+ public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {}
+
@Override
- public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
- }
-
+ public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {}
+
@Override
- public void cleanTablePermissions(String table) throws AccumuloSecurityException, TableNotFoundException {
- }
-
+ public void cleanTablePermissions(String table) throws AccumuloSecurityException, TableNotFoundException {}
+
@Override
- public void initUser(String user) throws AccumuloSecurityException {
- }
-
+ public void initUser(String user) throws AccumuloSecurityException {}
+
@Override
- public void cleanUser(String user) throws AccumuloSecurityException {
- }
-
+ public void cleanUser(String user) throws AccumuloSecurityException {}
+
@Override
public void initTable(String table) throws AccumuloSecurityException {}
@@ -107,16 +98,13 @@ public class InsecurePermHandler implements PermissionHandler {
@Override
public void grantNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
- NamespaceNotFoundException {
- }
+ NamespaceNotFoundException {}
@Override
public void revokeNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
- NamespaceNotFoundException {
- }
+ NamespaceNotFoundException {}
@Override
- public void cleanNamespacePermissions(String namespace) throws AccumuloSecurityException, NamespaceNotFoundException {
- }
-
+ public void cleanNamespacePermissions(String namespace) throws AccumuloSecurityException, NamespaceNotFoundException {}
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/security/handler/PermissionHandler.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/handler/PermissionHandler.java b/server/base/src/main/java/org/apache/accumulo/server/security/handler/PermissionHandler.java
index 914bab3..117aa77 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/PermissionHandler.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/PermissionHandler.java
@@ -70,8 +70,7 @@ public interface PermissionHandler {
/**
* Used to get the namespace permission of a user for a namespace
*/
- boolean hasNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
- NamespaceNotFoundException;
+ boolean hasNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException, NamespaceNotFoundException;
/**
* Used to get the namespace permission of a user for a namespace, with caching. This method is for high frequency operations
@@ -102,14 +101,12 @@ public interface PermissionHandler {
/**
* Gives the user the given namespace permission
*/
- void grantNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
- NamespaceNotFoundException;
+ void grantNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException, NamespaceNotFoundException;
/**
* Denies the user the given namespace permission.
*/
- void revokeNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
- NamespaceNotFoundException;
+ void revokeNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException, NamespaceNotFoundException;
/**
* Cleans up the permissions for a table. Used when a table gets deleted.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
index c03c40b..f032aa2 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
@@ -217,8 +217,7 @@ public class ZKPermHandler implements PermissionHandler {
synchronized (zooCache) {
zooCache.clear(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
- ZKSecurityTool.convertTablePermissions(tablePerms),
- NodeExistsPolicy.OVERWRITE);
+ ZKSecurityTool.convertTablePermissions(tablePerms), NodeExistsPolicy.OVERWRITE);
}
}
} catch (KeeperException e) {
@@ -244,8 +243,7 @@ public class ZKPermHandler implements PermissionHandler {
synchronized (zooCache) {
zooCache.clear(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace,
- ZKSecurityTool.convertNamespacePermissions(namespacePerms),
- NodeExistsPolicy.OVERWRITE);
+ ZKSecurityTool.convertNamespacePermissions(namespacePerms), NodeExistsPolicy.OVERWRITE);
}
}
} catch (KeeperException e) {
@@ -435,8 +433,8 @@ public class ZKPermHandler implements PermissionHandler {
private void createTablePerm(String user, String table, Set<TablePermission> perms) throws KeeperException, InterruptedException {
synchronized (zooCache) {
zooCache.clear();
- ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
- ZKSecurityTool.convertTablePermissions(perms), NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(perms),
+ NodeExistsPolicy.FAIL);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKSecurityTool.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKSecurityTool.java b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKSecurityTool.java
index 2ed430c..aec3078 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKSecurityTool.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKSecurityTool.java
@@ -30,8 +30,8 @@ import java.util.Set;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.log4j.Logger;
@@ -43,7 +43,7 @@ import org.apache.log4j.Logger;
class ZKSecurityTool {
private static final Logger log = Logger.getLogger(ZKSecurityTool.class);
private static final int SALT_LENGTH = 8;
-
+
// Generates a byte array salt of length SALT_LENGTH
private static byte[] generateSalt() {
final SecureRandom random = new SecureRandom();
@@ -51,17 +51,17 @@ class ZKSecurityTool {
random.nextBytes(salt);
return salt;
}
-
+
private static byte[] hash(byte[] raw) throws NoSuchAlgorithmException {
MessageDigest md = MessageDigest.getInstance(Constants.PW_HASH_ALGORITHM);
md.update(raw);
return md.digest();
}
-
+
public static boolean checkPass(byte[] password, byte[] zkData) {
if (zkData == null)
return false;
-
+
byte[] salt = new byte[SALT_LENGTH];
System.arraycopy(zkData, 0, salt, 0, SALT_LENGTH);
byte[] passwordToCheck;
@@ -73,7 +73,7 @@ class ZKSecurityTool {
}
return java.util.Arrays.equals(passwordToCheck, zkData);
}
-
+
public static byte[] createPass(byte[] password) throws AccumuloException {
byte[] salt = generateSalt();
try {
@@ -83,7 +83,7 @@ class ZKSecurityTool {
throw new AccumuloException("Count not create hashed password", e);
}
}
-
+
private static byte[] convertPass(byte[] password, byte[] salt) throws NoSuchAlgorithmException {
byte[] plainSalt = new byte[password.length + SALT_LENGTH];
System.arraycopy(password, 0, plainSalt, 0, password.length);
@@ -94,15 +94,15 @@ class ZKSecurityTool {
System.arraycopy(hashed, 0, saltedHash, SALT_LENGTH, hashed.length);
return saltedHash; // contains salt+hash(password+salt)
}
-
+
public static Authorizations convertAuthorizations(byte[] authorizations) {
return new Authorizations(authorizations);
}
-
+
public static byte[] convertAuthorizations(Authorizations authorizations) {
return authorizations.getAuthorizationsArray();
}
-
+
public static byte[] convertSystemPermissions(Set<SystemPermission> systempermissions) {
ByteArrayOutputStream bytes = new ByteArrayOutputStream(systempermissions.size());
DataOutputStream out = new DataOutputStream(bytes);
@@ -115,7 +115,7 @@ class ZKSecurityTool {
}
return bytes.toByteArray();
}
-
+
public static Set<SystemPermission> convertSystemPermissions(byte[] systempermissions) {
ByteArrayInputStream bytes = new ByteArrayInputStream(systempermissions);
DataInputStream in = new DataInputStream(bytes);
@@ -129,7 +129,7 @@ class ZKSecurityTool {
}
return toReturn;
}
-
+
public static byte[] convertTablePermissions(Set<TablePermission> tablepermissions) {
ByteArrayOutputStream bytes = new ByteArrayOutputStream(tablepermissions.size());
DataOutputStream out = new DataOutputStream(bytes);
@@ -142,14 +142,14 @@ class ZKSecurityTool {
}
return bytes.toByteArray();
}
-
+
public static Set<TablePermission> convertTablePermissions(byte[] tablepermissions) {
Set<TablePermission> toReturn = new HashSet<TablePermission>();
for (byte b : tablepermissions)
toReturn.add(TablePermission.getPermissionById(b));
return toReturn;
}
-
+
public static byte[] convertNamespacePermissions(Set<NamespacePermission> namespacepermissions) {
ByteArrayOutputStream bytes = new ByteArrayOutputStream(namespacepermissions.size());
DataOutputStream out = new DataOutputStream(bytes);
@@ -162,14 +162,14 @@ class ZKSecurityTool {
}
return bytes.toByteArray();
}
-
+
public static Set<NamespacePermission> convertNamespacePermissions(byte[] namespacepermissions) {
Set<NamespacePermission> toReturn = new HashSet<NamespacePermission>();
for (byte b : namespacepermissions)
toReturn.add(NamespacePermission.getPermissionById(b));
return toReturn;
}
-
+
public static String getInstancePath(String instanceId) {
return Constants.ZROOT + "/" + instanceId;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
index 886c28f..7e5f54d 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
@@ -54,7 +54,7 @@ public class TableManager {
private static final Set<TableObserver> observers = Collections.synchronizedSet(new HashSet<TableObserver>());
private static final Map<String,TableState> tableStateCache = Collections.synchronizedMap(new HashMap<String,TableState>());
private static final byte[] ZERO_BYTE = new byte[] {'0'};
-
+
private static TableManager tableManager = null;
private final Instance instance;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/tables/TableObserver.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/tables/TableObserver.java b/server/base/src/main/java/org/apache/accumulo/server/tables/TableObserver.java
index 80aec3a..de88b96 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tables/TableObserver.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tables/TableObserver.java
@@ -22,8 +22,8 @@ import org.apache.accumulo.core.master.state.tables.TableState;
public interface TableObserver {
void initialize(Map<String,TableState> tableIdToStateMap);
-
+
void stateChanged(String tableId, TableState tState);
-
+
void sessionExpired();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/tablets/TabletTime.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/tablets/TabletTime.java b/server/base/src/main/java/org/apache/accumulo/server/tablets/TabletTime.java
index e3fd8f3..3dd9e7b 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tablets/TabletTime.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tablets/TabletTime.java
@@ -27,7 +27,7 @@ import org.apache.accumulo.server.util.time.RelativeTime;
public abstract class TabletTime {
public static final char LOGICAL_TIME_ID = 'L';
public static final char MILLIS_TIME_ID = 'M';
-
+
public static char getTimeID(TimeType timeType) {
switch (timeType) {
case LOGICAL:
@@ -35,115 +35,115 @@ public abstract class TabletTime {
case MILLIS:
return MILLIS_TIME_ID;
}
-
+
throw new IllegalArgumentException("Unknown time type " + timeType);
}
-
+
public abstract void useMaxTimeFromWALog(long time);
-
+
public abstract String getMetadataValue(long time);
-
+
public abstract String getMetadataValue();
-
+
public abstract long setUpdateTimes(List<Mutation> mutations);
-
+
public abstract long getTime();
-
+
public abstract long getAndUpdateTime();
-
+
protected void setSystemTimes(Mutation mutation, long lastCommitTime) {
ServerMutation m = (ServerMutation) mutation;
m.setSystemTimestamp(lastCommitTime);
}
-
+
public static TabletTime getInstance(String metadataValue) {
if (metadataValue.charAt(0) == LOGICAL_TIME_ID) {
return new LogicalTime(Long.parseLong(metadataValue.substring(1)));
} else if (metadataValue.charAt(0) == MILLIS_TIME_ID) {
return new MillisTime(Long.parseLong(metadataValue.substring(1)));
}
-
+
throw new IllegalArgumentException("Time type unknown : " + metadataValue);
-
+
}
-
+
public static String maxMetadataTime(String mv1, String mv2) {
if (mv1 == null && mv2 == null) {
return null;
}
-
+
if (mv1 == null) {
checkType(mv2);
return mv2;
}
-
+
if (mv2 == null) {
checkType(mv1);
return mv1;
}
-
+
if (mv1.charAt(0) != mv2.charAt(0))
throw new IllegalArgumentException("Time types differ " + mv1 + " " + mv2);
checkType(mv1);
-
+
long t1 = Long.parseLong(mv1.substring(1));
long t2 = Long.parseLong(mv2.substring(1));
-
+
if (t1 < t2)
return mv2;
else
return mv1;
-
+
}
-
+
private static void checkType(String mv1) {
if (mv1.charAt(0) != LOGICAL_TIME_ID && mv1.charAt(0) != MILLIS_TIME_ID)
throw new IllegalArgumentException("Invalid time type " + mv1);
}
-
+
static class MillisTime extends TabletTime {
-
+
private long lastTime;
private long lastUpdateTime = 0;
-
+
public MillisTime(long time) {
this.lastTime = time;
}
-
+
@Override
public String getMetadataValue(long time) {
return MILLIS_TIME_ID + "" + time;
}
-
+
@Override
public String getMetadataValue() {
return getMetadataValue(lastTime);
}
-
+
@Override
public void useMaxTimeFromWALog(long time) {
if (time > lastTime)
lastTime = time;
}
-
+
@Override
public long setUpdateTimes(List<Mutation> mutations) {
-
+
long currTime = RelativeTime.currentTimeMillis();
-
+
synchronized (this) {
if (mutations.size() == 0)
return lastTime;
-
+
currTime = updateTime(currTime);
}
-
+
for (Mutation mutation : mutations)
setSystemTimes(mutation, currTime);
-
+
return currTime;
}
-
+
private long updateTime(long currTime) {
if (currTime < lastTime) {
if (currTime - lastUpdateTime > 0) {
@@ -151,81 +151,81 @@ public abstract class TabletTime {
// to this method so move ahead slowly
lastTime++;
}
-
+
lastUpdateTime = currTime;
-
+
currTime = lastTime;
} else {
lastTime = currTime;
}
return currTime;
}
-
+
@Override
public long getTime() {
return lastTime;
}
-
+
@Override
public long getAndUpdateTime() {
long currTime = RelativeTime.currentTimeMillis();
-
+
synchronized (this) {
currTime = updateTime(currTime);
}
-
+
return currTime;
}
-
+
}
-
+
static class LogicalTime extends TabletTime {
AtomicLong nextTime;
-
+
private LogicalTime(Long time) {
this.nextTime = new AtomicLong(time.longValue() + 1);
}
-
+
@Override
public void useMaxTimeFromWALog(long time) {
time++;
-
+
if (this.nextTime.get() < time) {
this.nextTime.set(time);
}
}
-
+
@Override
public String getMetadataValue() {
return getMetadataValue(getTime());
}
-
+
@Override
public String getMetadataValue(long time) {
return LOGICAL_TIME_ID + "" + time;
}
-
+
@Override
public long setUpdateTimes(List<Mutation> mutations) {
if (mutations.size() == 0)
return getTime();
-
+
long time = nextTime.getAndAdd(mutations.size());
for (Mutation mutation : mutations)
setSystemTimes(mutation, time++);
-
+
return time - 1;
}
-
+
@Override
public long getTime() {
return nextTime.get() - 1;
}
-
+
@Override
public long getAndUpdateTime() {
return nextTime.getAndIncrement();
}
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
index 8bf4dc8..c2eac02 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
@@ -28,9 +28,9 @@ import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
/**
* Allocates unique names for an accumulo instance. The names are unique for the lifetime of the instance.
- *
+ *
* This is useful for filenames because it makes caching easy.
- *
+ *
*/
public class UniqueNameAllocator {
@@ -38,17 +38,17 @@ public class UniqueNameAllocator {
private long maxAllocated = 0;
private String nextNamePath;
private Random rand;
-
+
private UniqueNameAllocator() {
nextNamePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZNEXT_FILE;
rand = new Random();
}
-
+
public synchronized String getNextName() {
-
+
while (next >= maxAllocated) {
final int allocate = 100 + rand.nextInt(100);
-
+
try {
byte[] max = ZooReaderWriter.getInstance().mutate(nextNamePath, null, ZooUtil.PRIVATE, new ZooReaderWriter.Mutator() {
public byte[] mutate(byte[] currentValue) throws Exception {
@@ -57,25 +57,25 @@ public class UniqueNameAllocator {
return Long.toString(l, Character.MAX_RADIX).getBytes(UTF_8);
}
});
-
+
maxAllocated = Long.parseLong(new String(max, UTF_8), Character.MAX_RADIX);
next = maxAllocated - allocate;
-
+
} catch (Exception e) {
throw new RuntimeException(e);
}
}
-
+
return new String(FastFormat.toZeroPaddedString(next++, 7, Character.MAX_RADIX, new byte[0]), UTF_8);
}
-
+
private static UniqueNameAllocator instance = null;
-
+
public static synchronized UniqueNameAllocator getInstance() {
if (instance == null)
instance = new UniqueNameAllocator();
-
+
return instance;
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
index 41a1edb..5557f03 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
@@ -32,16 +32,15 @@ import org.apache.log4j.Logger;
* The LargestFirstMemoryManager attempts to keep memory between 80% and 90% full. It adapts over time the point at which it should start a compaction based on
* how full memory gets between successive calls. It will also flush idle tablets based on a per-table configurable idle time. It will only attempt to flush
* tablets up to 20% of all memory. And, as the name of the class would suggest, it flushes the tablet with the highest memory footprint. However, it actually
- * chooses the tablet as a function of its size doubled for every 15 minutes of idle time.
+ * chooses the tablet as a function of its size doubled for every 15 minutes of idle time.
*/
public class LargestFirstMemoryManager implements MemoryManager {
-
+
private static final Logger log = Logger.getLogger(LargestFirstMemoryManager.class);
private static final long ZERO_TIME = System.currentTimeMillis();
private static final int TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER = 2;
private static final double MAX_FLUSH_AT_ONCE_PERCENT = 0.20;
-
private long maxMemory = -1;
private int maxConcurrentMincs;
private int numWaitingMultiplier;
@@ -51,13 +50,13 @@ public class LargestFirstMemoryManager implements MemoryManager {
private long maxObserved;
private final HashMap<Text,Long> mincIdleThresholds = new HashMap<Text,Long>();
private ServerConfiguration config = null;
-
+
private static class TabletInfo {
final KeyExtent extent;
final long memTableSize;
final long idleTime;
final long load;
-
+
public TabletInfo(KeyExtent extent, long memTableSize, long idleTime, long load) {
this.extent = extent;
this.memTableSize = memTableSize;
@@ -65,16 +64,16 @@ public class LargestFirstMemoryManager implements MemoryManager {
this.load = load;
}
}
-
+
// A little map that will hold the "largest" N tablets, where largest is a result of the timeMemoryLoad function
private static class LargestMap {
final int max;
- final TreeMap<Long, List<TabletInfo>> map = new TreeMap<Long, List<TabletInfo>>();
-
+ final TreeMap<Long,List<TabletInfo>> map = new TreeMap<Long,List<TabletInfo>>();
+
LargestMap(int n) {
max = n;
}
-
+
public boolean put(Long key, TabletInfo value) {
if (map.size() == max) {
if (key.compareTo(map.firstKey()) < 0)
@@ -114,14 +113,14 @@ public class LargestFirstMemoryManager implements MemoryManager {
map.remove(key);
}
}
-
+
LargestFirstMemoryManager(long maxMemory, int maxConcurrentMincs, int numWaitingMultiplier) {
this();
this.maxMemory = maxMemory;
this.maxConcurrentMincs = maxConcurrentMincs;
this.numWaitingMultiplier = numWaitingMultiplier;
}
-
+
@Override
public void init(ServerConfiguration conf) {
this.config = conf;
@@ -129,39 +128,39 @@ public class LargestFirstMemoryManager implements MemoryManager {
maxConcurrentMincs = conf.getConfiguration().getCount(Property.TSERV_MINC_MAXCONCURRENT);
numWaitingMultiplier = TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER;
}
-
+
public LargestFirstMemoryManager() {
prevIngestMemory = 0;
compactionThreshold = 0.5;
maxObserved = 0;
}
-
+
protected long getMinCIdleThreshold(KeyExtent extent) {
Text tableId = extent.getTableId();
if (!mincIdleThresholds.containsKey(tableId))
mincIdleThresholds.put(tableId, config.getTableConfiguration(tableId.toString()).getTimeInMillis(Property.TABLE_MINC_COMPACT_IDLETIME));
return mincIdleThresholds.get(tableId);
}
-
+
@Override
public MemoryManagementActions getMemoryManagementActions(List<TabletState> tablets) {
if (maxMemory < 0)
throw new IllegalStateException("need to initialize " + LargestFirstMemoryManager.class.getName());
-
+
final int maxMinCs = maxConcurrentMincs * numWaitingMultiplier;
-
+
mincIdleThresholds.clear();
final MemoryManagementActions result = new MemoryManagementActions();
result.tabletsToMinorCompact = new ArrayList<KeyExtent>();
-
+
LargestMap largestMemTablets = new LargestMap(maxMinCs);
final LargestMap largestIdleMemTablets = new LargestMap(maxConcurrentMincs);
final long now = currentTimeMillis();
-
+
long ingestMemory = 0;
long compactionMemory = 0;
int numWaitingMincs = 0;
-
+
// find the largest and most idle tablets
for (TabletState ts : tablets) {
final long memTabletSize = ts.getMemTableSize();
@@ -176,21 +175,21 @@ public class LargestFirstMemoryManager implements MemoryManager {
largestIdleMemTablets.put(timeMemoryLoad, tabletInfo);
}
}
-
+
compactionMemory += minorCompactingSize;
if (minorCompactingSize > 0)
numWaitingMincs++;
}
-
+
if (ingestMemory + compactionMemory > maxObserved) {
maxObserved = ingestMemory + compactionMemory;
}
-
+
final long memoryChange = ingestMemory - prevIngestMemory;
prevIngestMemory = ingestMemory;
-
+
boolean startMinC = false;
-
+
if (numWaitingMincs < maxMinCs) {
// based on previous ingest memory increase, if we think that the next increase will
// take us over the threshold for non-compacting memory, then start a minor compaction
@@ -204,23 +203,22 @@ public class LargestFirstMemoryManager implements MemoryManager {
log.debug("IDLE minor compaction chosen");
}
}
-
+
if (startMinC) {
long toBeCompacted = compactionMemory;
- outer:
- for (int i = numWaitingMincs; i < maxMinCs && !largestMemTablets.isEmpty(); /* empty */) {
- Entry<Long,List<TabletInfo>> lastEntry = largestMemTablets.lastEntry();
- for (TabletInfo largest : lastEntry.getValue()) {
- toBeCompacted += largest.memTableSize;
- result.tabletsToMinorCompact.add(largest.extent);
- log.debug(String.format("COMPACTING %s total = %,d ingestMemory = %,d", largest.extent.toString(), (ingestMemory + compactionMemory), ingestMemory));
- log.debug(String.format("chosenMem = %,d chosenIT = %.2f load %,d", largest.memTableSize, largest.idleTime / 1000.0, largest.load));
- if (toBeCompacted > ingestMemory * MAX_FLUSH_AT_ONCE_PERCENT)
- break outer;
- i++;
- }
- largestMemTablets.remove(lastEntry.getKey());
+ outer: for (int i = numWaitingMincs; i < maxMinCs && !largestMemTablets.isEmpty(); /* empty */) {
+ Entry<Long,List<TabletInfo>> lastEntry = largestMemTablets.lastEntry();
+ for (TabletInfo largest : lastEntry.getValue()) {
+ toBeCompacted += largest.memTableSize;
+ result.tabletsToMinorCompact.add(largest.extent);
+ log.debug(String.format("COMPACTING %s total = %,d ingestMemory = %,d", largest.extent.toString(), (ingestMemory + compactionMemory), ingestMemory));
+ log.debug(String.format("chosenMem = %,d chosenIT = %.2f load %,d", largest.memTableSize, largest.idleTime / 1000.0, largest.load));
+ if (toBeCompacted > ingestMemory * MAX_FLUSH_AT_ONCE_PERCENT)
+ break outer;
+ i++;
}
+ largestMemTablets.remove(lastEntry.getKey());
+ }
} else if (memoryChange < 0) {
// before idle mincs, starting a minor compaction meant that memoryChange >= 0.
// we thought we might want to remove the "else" if that changed,
@@ -228,12 +226,12 @@ public class LargestFirstMemoryManager implements MemoryManager {
// change more often, so it is staying for now.
// also, now we have the case where memoryChange < 0 due to an idle compaction, yet
// we are still adjusting the threshold. should this be tracked and prevented?
-
+
// memory change < 0 means a minor compaction occurred
// we want to see how full the memory got during the compaction
// (the goal is for it to have between 80% and 90% memory utilization)
// and adjust the compactionThreshold accordingly
-
+
log.debug(String.format("BEFORE compactionThreshold = %.3f maxObserved = %,d", compactionThreshold, maxObserved));
if (compactionThreshold < 0.82 && maxObserved < 0.8 * maxMemory) {
// 0.82 * 1.1 is about 0.9, which is our desired max threshold
@@ -243,24 +241,24 @@ public class LargestFirstMemoryManager implements MemoryManager {
compactionThreshold *= 0.9;
}
maxObserved = 0;
-
+
log.debug(String.format("AFTER compactionThreshold = %.3f", compactionThreshold));
}
-
+
return result;
}
-
+
protected long currentTimeMillis() {
return System.currentTimeMillis();
}
@Override
public void tabletClosed(KeyExtent extent) {}
-
+
// The load function: memory times the idle time, doubling every 15 mins
static long timeMemoryLoad(long mem, long time) {
double minutesIdle = time / 60000.0;
-
+
return (long) (mem * Math.pow(2, minutesIdle / 15.0));
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/tabletserver/MemoryManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/MemoryManager.java b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/MemoryManager.java
index 5e6fecc..11327f6 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/MemoryManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/MemoryManager.java
@@ -24,13 +24,13 @@ import org.apache.accumulo.server.conf.ServerConfiguration;
/**
* A MemoryManager in accumulo currently determines when minor compactions should occur and when ingest should be put on hold. The goal of a memory manager
* implementation is to maximize ingest throughput and minimize the number of minor compactions.
- *
- *
- *
+ *
+ *
+ *
*/
public interface MemoryManager {
-
+
/**
* Initialize the memory manager.
*/
@@ -38,14 +38,14 @@ public interface MemoryManager {
/**
* An implementation of this function will be called periodically by accumulo and should return a list of tablets to minor compact.
- *
+ *
* Instructing a tablet that is already minor compacting (this can be inferred from the TabletState) to minor compact has no effect.
- *
+ *
* Holding all ingest does not affect metadata tablets.
*/
-
+
MemoryManagementActions getMemoryManagementActions(List<TabletState> tablets);
-
+
/**
* This method is called when a tablet is closed. A memory manger can clean up any per tablet state it is keeping when this is called.
*/
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/tabletserver/TabletState.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/TabletState.java b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/TabletState.java
index aeacb8d..78301e8 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/TabletState.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/TabletState.java
@@ -20,10 +20,10 @@ import org.apache.accumulo.core.data.KeyExtent;
public interface TabletState {
KeyExtent getExtent();
-
+
long getLastCommitTime();
-
+
long getMemTableSize();
-
+
long getMinorCompactingMemTableSize();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/ActionStatsUpdator.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ActionStatsUpdator.java b/server/base/src/main/java/org/apache/accumulo/server/util/ActionStatsUpdator.java
index dd4573b..de28ad9 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ActionStatsUpdator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ActionStatsUpdator.java
@@ -19,10 +19,10 @@ package org.apache.accumulo.server.util;
import org.apache.accumulo.core.tabletserver.thrift.ActionStats;
/**
- *
+ *
*/
public class ActionStatsUpdator {
-
+
public static void update(ActionStats summary, ActionStats td) {
summary.status += td.status;
summary.elapsed += td.elapsed;
@@ -33,5 +33,5 @@ public class ActionStatsUpdator {
summary.queueSumDev += td.queueSumDev;
summary.fail += td.fail;
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
index ae36f1f..7d247f7 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
@@ -129,10 +129,10 @@ public class Admin {
@Parameter(names = {"-u", "--users"}, description = "print users and their authorizations and permissions")
boolean users = false;
}
-
+
@Parameters(commandDescription = "redistribute tablet directories across the current volume list")
static class RandomizeVolumesCommand {
- @Parameter(names={"-t"}, description = "table to update", required=true)
+ @Parameter(names = {"-t"}, description = "table to update", required = true)
String table = null;
}
@@ -164,10 +164,10 @@ public class Admin {
cl.addCommand("stopAll", stopAllOpts);
StopMasterCommand stopMasterOpts = new StopMasterCommand();
cl.addCommand("stopMaster", stopMasterOpts);
-
+
RandomizeVolumesCommand randomizeVolumesOpts = new RandomizeVolumesCommand();
cl.addCommand("randomizeVolumes", randomizeVolumesOpts);
-
+
cl.parse(args);
if (opts.help || cl.getParsedCommand() == null) {
@@ -269,7 +269,7 @@ public class Admin {
/**
* flushing during shutdown is a performance optimization, its not required. The method will make an attempt to initiate flushes of all tables and give up if
* it takes too long.
- *
+ *
*/
private static void flushAll(final ClientContext context) throws AccumuloException, AccumuloSecurityException {
@@ -385,21 +385,21 @@ public class Admin {
}
if (opts.allConfiguration) {
- //print accumulo site
+ // print accumulo site
printSystemConfiguration(connector, outputDirectory);
- //print namespaces
+ // print namespaces
for (String namespace : connector.namespaceOperations().list()) {
printNameSpaceConfiguration(connector, namespace, outputDirectory);
}
- //print tables
+ // print tables
SortedSet<String> tableNames = connector.tableOperations().list();
for (String tableName : tableNames) {
printTableConfiguration(connector, tableName, outputDirectory);
}
- //print users
+ // print users
for (String user : localUsers) {
- printUserConfiguration(connector, user, outputDirectory);
- }
+ printUserConfiguration(connector, user, outputDirectory);
+ }
} else {
if (opts.systemConfiguration) {
printSystemConfiguration(connector, outputDirectory);
@@ -436,56 +436,57 @@ public class Admin {
return defaultValue;
}
- private static void printNameSpaceConfiguration(Connector connector, String namespace, File outputDirectory) throws IOException, AccumuloException, AccumuloSecurityException, NamespaceNotFoundException {
- File namespaceScript = new File(outputDirectory, namespace + NS_FILE_SUFFIX);
- FileWriter nsWriter = new FileWriter(namespaceScript);
- nsWriter.write(createNsFormat.format(new String[] {namespace}));
- TreeMap<String,String> props = new TreeMap<String,String>();
- for (Entry<String,String> p : connector.namespaceOperations().getProperties(namespace)) {
- props.put(p.getKey(), p.getValue());
- }
- for (Entry<String,String> entry : props.entrySet()) {
- String defaultValue = getDefaultConfigValue(entry.getKey());
- if (defaultValue == null || !defaultValue.equals(entry.getValue())) {
- if (!entry.getValue().equals(siteConfig.get(entry.getKey())) && !entry.getValue().equals(systemConfig.get(entry.getKey()))) {
- nsWriter.write(nsConfigFormat.format(new String[] {namespace, entry.getKey()+"="+entry.getValue()}));
- }
+ private static void printNameSpaceConfiguration(Connector connector, String namespace, File outputDirectory) throws IOException, AccumuloException,
+ AccumuloSecurityException, NamespaceNotFoundException {
+ File namespaceScript = new File(outputDirectory, namespace + NS_FILE_SUFFIX);
+ FileWriter nsWriter = new FileWriter(namespaceScript);
+ nsWriter.write(createNsFormat.format(new String[] {namespace}));
+ TreeMap<String,String> props = new TreeMap<String,String>();
+ for (Entry<String,String> p : connector.namespaceOperations().getProperties(namespace)) {
+ props.put(p.getKey(), p.getValue());
+ }
+ for (Entry<String,String> entry : props.entrySet()) {
+ String defaultValue = getDefaultConfigValue(entry.getKey());
+ if (defaultValue == null || !defaultValue.equals(entry.getValue())) {
+ if (!entry.getValue().equals(siteConfig.get(entry.getKey())) && !entry.getValue().equals(systemConfig.get(entry.getKey()))) {
+ nsWriter.write(nsConfigFormat.format(new String[] {namespace, entry.getKey() + "=" + entry.getValue()}));
}
- }
- nsWriter.close();
+ }
+ }
+ nsWriter.close();
}
- private static void printUserConfiguration(Connector connector, String user, File outputDirectory) throws IOException, AccumuloException, AccumuloSecurityException {
- File userScript = new File(outputDirectory, user + USER_FILE_SUFFIX);
- FileWriter userWriter = new FileWriter(userScript);
- userWriter.write(createUserFormat.format(new String[] {user}));
- Authorizations auths = connector.securityOperations().getUserAuthorizations(user);
- userWriter.write(userAuthsFormat.format(new String[] {user, auths.toString()}));
- for (SystemPermission sp : SystemPermission.values()) {
- if (connector.securityOperations().hasSystemPermission(user, sp)) {
- userWriter.write(sysPermFormat.format(new String[] {sp.name(), user}));
- }
+ private static void printUserConfiguration(Connector connector, String user, File outputDirectory) throws IOException, AccumuloException,
+ AccumuloSecurityException {
+ File userScript = new File(outputDirectory, user + USER_FILE_SUFFIX);
+ FileWriter userWriter = new FileWriter(userScript);
+ userWriter.write(createUserFormat.format(new String[] {user}));
+ Authorizations auths = connector.securityOperations().getUserAuthorizations(user);
+ userWriter.write(userAuthsFormat.format(new String[] {user, auths.toString()}));
+ for (SystemPermission sp : SystemPermission.values()) {
+ if (connector.securityOperations().hasSystemPermission(user, sp)) {
+ userWriter.write(sysPermFormat.format(new String[] {sp.name(), user}));
}
- for (String namespace : connector.namespaceOperations().list()) {
- for (NamespacePermission np : NamespacePermission.values()) {
- if (connector.securityOperations().hasNamespacePermission(user, namespace, np)) {
- userWriter.write(nsPermFormat.format(new String[] {np.name(), namespace, user}));
- }
+ }
+ for (String namespace : connector.namespaceOperations().list()) {
+ for (NamespacePermission np : NamespacePermission.values()) {
+ if (connector.securityOperations().hasNamespacePermission(user, namespace, np)) {
+ userWriter.write(nsPermFormat.format(new String[] {np.name(), namespace, user}));
}
}
- for (String tableName : connector.tableOperations().list()) {
- for (TablePermission perm : TablePermission.values()) {
- if (connector.securityOperations().hasTablePermission(user, tableName, perm)) {
- userWriter.write(tablePermFormat.format(new String[] {perm.name(), tableName, user}));
- }
+ }
+ for (String tableName : connector.tableOperations().list()) {
+ for (TablePermission perm : TablePermission.values()) {
+ if (connector.securityOperations().hasTablePermission(user, tableName, perm)) {
+ userWriter.write(tablePermFormat.format(new String[] {perm.name(), tableName, user}));
}
}
+ }
- userWriter.close();
+ userWriter.close();
}
- private static void printSystemConfiguration(Connector connector, File outputDirectory) throws IOException, AccumuloException,
- AccumuloSecurityException {
+ private static void printSystemConfiguration(Connector connector, File outputDirectory) throws IOException, AccumuloException, AccumuloSecurityException {
Configuration conf = new Configuration(false);
TreeMap<String,String> site = new TreeMap<String,String>(siteConfig);
for (Entry<String,String> prop : site.entrySet()) {
@@ -510,8 +511,8 @@ public class Admin {
}
}
- private static void printTableConfiguration(Connector connector, String tableName, File outputDirectory) throws AccumuloException,
- TableNotFoundException, IOException, AccumuloSecurityException {
+ private static void printTableConfiguration(Connector connector, String tableName, File outputDirectory) throws AccumuloException, TableNotFoundException,
+ IOException, AccumuloSecurityException {
File tableBackup = new File(outputDirectory, tableName + ".cfg");
FileWriter writer = new FileWriter(tableBackup);
writer.write(createTableFormat.format(new String[] {tableName}));
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index c93f680..18ca881 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@ -44,14 +44,14 @@ import org.apache.zookeeper.data.Stat;
import com.beust.jcommander.Parameter;
public class ChangeSecret {
-
+
static class Opts extends ClientOpts {
- @Parameter(names="--old", description="old zookeeper password", password=true, hidden=true)
+ @Parameter(names = "--old", description = "old zookeeper password", password = true, hidden = true)
String oldPass;
- @Parameter(names="--new", description="new zookeeper password", password=true, hidden=true)
+ @Parameter(names = "--new", description = "new zookeeper password", password = true, hidden = true)
String newPass;
}
-
+
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
List<String> argsList = new ArrayList<String>(args.length + 2);
@@ -71,11 +71,11 @@ public class ChangeSecret {
System.out.println("New instance id is " + instanceId);
System.out.println("Be sure to put your new secret in accumulo-site.xml");
}
-
+
interface Visitor {
void visit(ZooReader zoo, String path) throws Exception;
}
-
+
private static void recurse(ZooReader zoo, String root, Visitor v) {
try {
v.visit(zoo, root);
@@ -86,7 +86,7 @@ public class ChangeSecret {
throw new RuntimeException(ex);
}
}
-
+
private static boolean verifyAccumuloIsDown(Instance inst, String oldPassword) {
ZooReader zooReader = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), oldPassword);
String root = ZooUtil.getRoot(inst);
@@ -101,14 +101,14 @@ public class ChangeSecret {
if (ephemerals.size() == 0) {
return true;
}
-
+
System.err.println("The following ephemeral nodes exist, something is still running:");
for (String path : ephemerals) {
System.err.println(path);
}
return false;
}
-
+
private static String rewriteZooKeeperInstance(final Instance inst, String oldPass, String newPass) throws Exception {
final ZooReaderWriter orig = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), oldPass);
final IZooReaderWriter new_ = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), newPass);
@@ -143,7 +143,7 @@ public class ChangeSecret {
new_.putPersistentData(path, newInstanceId.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
return newInstanceId;
}
-
+
private static void updateHdfs(VolumeManager fs, Instance inst, String newInstanceId) throws IOException {
// Need to recreate the instanceId on all of them to keep consistency
for (Volume v : fs.getVolumes()) {
@@ -159,7 +159,7 @@ public class ChangeSecret {
v.getFileSystem().create(new Path(instanceId, newInstanceId)).close();
}
}
-
+
private static void deleteInstance(Instance origInstance, String oldPass) throws Exception {
IZooReaderWriter orig = new ZooReaderWriter(origInstance.getZooKeepers(), origInstance.getZooKeepersSessionTimeOut(), oldPass);
orig.recursiveDelete("/accumulo/" + origInstance.getInstanceID(), NodeMissingPolicy.SKIP);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java b/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
index 0652cf1..3cfb1a7 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.server.util;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.IOException;
+
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
@@ -31,16 +32,16 @@ import org.apache.zookeeper.KeeperException;
import com.beust.jcommander.Parameter;
public class CleanZookeeper {
-
+
private static final Logger log = Logger.getLogger(CleanZookeeper.class);
-
+
static class Opts extends Help {
- @Parameter(names={"-z", "--keepers"}, description="comma separated list of zookeeper hosts")
+ @Parameter(names = {"-z", "--keepers"}, description = "comma separated list of zookeeper hosts")
String keepers = "localhost:2181";
- @Parameter(names={"--password"}, description="the system secret", password=true)
+ @Parameter(names = {"--password"}, description = "the system secret", password = true)
String auth;
}
-
+
/**
* @param args
* must contain one element: the address of a zookeeper node a second parameter provides an additional authentication value
@@ -50,13 +51,13 @@ public class CleanZookeeper {
public static void main(String[] args) throws IOException {
Opts opts = new Opts();
opts.parseArgs(CleanZookeeper.class.getName(), args);
-
+
String root = Constants.ZROOT;
IZooReaderWriter zk = ZooReaderWriter.getInstance();
if (opts.auth != null) {
- zk.getZooKeeper().addAuthInfo("digest", ("accumulo:"+opts.auth).getBytes(UTF_8));
+ zk.getZooKeeper().addAuthInfo("digest", ("accumulo:" + opts.auth).getBytes(UTF_8));
}
-
+
try {
for (String child : zk.getChildren(root)) {
if (Constants.ZINSTANCES.equals("/" + child)) {
@@ -84,5 +85,5 @@ public class CleanZookeeper {
System.out.println("Error Occurred: " + ex);
}
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/DefaultMap.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/DefaultMap.java b/server/base/src/main/java/org/apache/accumulo/server/util/DefaultMap.java
index 7038be7..66c5d8e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DefaultMap.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DefaultMap.java
@@ -20,17 +20,17 @@ import java.util.HashMap;
/**
* A HashMap that returns a default value if the key is not stored in the map.
- *
+ *
* A zero-argument constructor of the default object's class is used, otherwise the default object is used.
*/
public class DefaultMap<K,V> extends HashMap<K,V> {
private static final long serialVersionUID = 1L;
V dfault;
-
+
public DefaultMap(V dfault) {
this.dfault = dfault;
}
-
+
@SuppressWarnings("unchecked")
@Override
public V get(Object key) {
@@ -44,7 +44,7 @@ public class DefaultMap<K,V> extends HashMap<K,V> {
}
return result;
}
-
+
@SuppressWarnings("unchecked")
private V construct() {
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java b/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
index 0793c6b..ba27733 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
@@ -31,14 +31,14 @@ import org.apache.zookeeper.KeeperException;
import com.beust.jcommander.Parameter;
public class DeleteZooInstance {
-
+
static class Opts extends Help {
- @Parameter(names={"-i", "--instance"}, description="the instance name or id to delete")
+ @Parameter(names = {"-i", "--instance"}, description = "the instance name or id to delete")
String instance;
}
-
+
static void deleteRetry(IZooReaderWriter zk, String path) throws Exception {
- for (int i = 0; i < 10; i++){
+ for (int i = 0; i < 10; i++) {
try {
zk.recursiveDelete(path, NodeMissingPolicy.SKIP);
return;
@@ -49,7 +49,7 @@ public class DeleteZooInstance {
}
}
}
-
+
/**
* @param args
* : the name or UUID of the instance to be deleted
@@ -57,7 +57,7 @@ public class DeleteZooInstance {
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
opts.parseArgs(DeleteZooInstance.class.getName(), args);
-
+
IZooReaderWriter zk = ZooReaderWriter.getInstance();
// try instance name:
Set<String> instances = new HashSet<String>(zk.getChildren(Constants.ZROOT + Constants.ZINSTANCES));
@@ -79,5 +79,5 @@ public class DeleteZooInstance {
deleteRetry(zk, Constants.ZROOT + "/" + opts.instance);
}
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java b/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
index 478b3b5..e900202 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
@@ -20,6 +20,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
+
import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.core.util.Base64;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
@@ -32,35 +33,35 @@ import org.apache.zookeeper.data.Stat;
import com.beust.jcommander.Parameter;
public class DumpZookeeper {
-
+
static IZooReaderWriter zk = null;
-
+
private static final Logger log = Logger.getLogger(DumpZookeeper.class);
-
+
private static class Encoded {
public String encoding;
public String value;
-
+
Encoded(String e, String v) {
encoding = e;
value = v;
}
}
-
+
static class Opts extends Help {
@Parameter(names = "--root", description = "the root of the znode tree to dump")
String root = "/";
}
-
+
public static void main(String[] args) {
Opts opts = new Opts();
opts.parseArgs(DumpZookeeper.class.getName(), args);
-
+
Logger.getRootLogger().setLevel(Level.WARN);
PrintStream out = System.out;
try {
zk = ZooReaderWriter.getInstance();
-
+
write(out, 0, "<dump root='%s'>", opts.root);
for (String child : zk.getChildren(opts.root, null))
if (!child.equals("zookeeper"))
@@ -70,7 +71,7 @@ public class DumpZookeeper {
log.error(ex, ex);
}
}
-
+
private static void dump(PrintStream out, String root, String child, int indent) throws KeeperException, InterruptedException, UnsupportedEncodingException {
String path = root + "/" + child;
if (root.endsWith("/"))
@@ -102,7 +103,7 @@ public class DumpZookeeper {
write(out, indent, "</node>");
}
}
-
+
private static Encoded value(String path) throws KeeperException, InterruptedException, UnsupportedEncodingException {
byte[] data = zk.getData(path, null);
for (int i = 0; i < data.length; i++) {
@@ -112,7 +113,7 @@ public class DumpZookeeper {
}
return new Encoded(UTF_8.name(), new String(data, UTF_8));
}
-
+
private static void write(PrintStream out, int indent, String fmt, Object... args) {
for (int i = 0; i < indent; i++)
out.print(" ");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/FileSystemMonitor.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FileSystemMonitor.java b/server/base/src/main/java/org/apache/accumulo/server/util/FileSystemMonitor.java
index 665bf25..94a80d9 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/FileSystemMonitor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/FileSystemMonitor.java
@@ -38,43 +38,43 @@ import org.apache.log4j.Logger;
public class FileSystemMonitor {
private static final String PROC_MOUNTS = "/proc/mounts";
private static final Logger log = Logger.getLogger(FileSystemMonitor.class);
-
+
private static class Mount {
String mountPoint;
Set<String> options;
-
+
Mount(String line) {
String tokens[] = line.split("\\s+");
-
+
mountPoint = tokens[1];
-
+
options = new HashSet<String>(Arrays.asList(tokens[3].split(",")));
}
}
-
+
static List<Mount> parse(String procFile) throws IOException {
-
+
List<Mount> mounts = new ArrayList<Mount>();
-
+
FileReader fr = new FileReader(procFile);
BufferedReader br = new BufferedReader(fr);
-
+
String line;
try {
- while ((line = br.readLine()) != null)
- mounts.add(new Mount(line));
+ while ((line = br.readLine()) != null)
+ mounts.add(new Mount(line));
} finally {
br.close();
}
-
+
return mounts;
}
-
+
private Map<String,Boolean> readWriteFilesystems = new HashMap<String,Boolean>();
-
+
public FileSystemMonitor(final String procFile, long period) throws IOException {
List<Mount> mounts = parse(procFile);
-
+
for (Mount mount : mounts) {
if (mount.options.contains("rw"))
readWriteFilesystems.put(mount.mountPoint, true);
@@ -83,7 +83,7 @@ public class FileSystemMonitor {
else
throw new IOException("Filesystem " + mount + " does not have ro or rw option");
}
-
+
TimerTask tt = new TimerTask() {
@Override
public void run() {
@@ -98,16 +98,16 @@ public class FileSystemMonitor {
}
}
};
-
+
// use a new Timer object instead of a shared one.
// trying to avoid the case where one the timers other
// task gets stuck because a FS went read only, and this task
// does not execute
Timer timer = new Timer("filesystem monitor timer", true);
timer.schedule(tt, period, period);
-
+
}
-
+
protected void logAsync(final Level level, final String msg, final Exception e) {
Runnable r = new Runnable() {
@Override
@@ -115,13 +115,13 @@ public class FileSystemMonitor {
log.log(level, msg, e);
}
};
-
+
new Thread(r).start();
}
-
+
protected void checkMounts(String procFile) throws Exception {
List<Mount> mounts = parse(procFile);
-
+
for (Mount mount : mounts) {
if (!readWriteFilesystems.containsKey(mount.mountPoint))
if (mount.options.contains("rw"))
@@ -134,7 +134,7 @@ public class FileSystemMonitor {
throw new Exception("Filesystem " + mount.mountPoint + " switched to read only");
}
}
-
+
public static void start(AccumuloConfiguration conf, Property prop) {
if (conf.getBoolean(prop)) {
if (new File(PROC_MOUNTS).exists()) {