You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2014/04/09 19:58:25 UTC
[54/64] [abbrv] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
index cdcdfba,0000000..98d3c73
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
@@@ -1,276 -1,0 +1,272 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metrics;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.lang.management.ManagementFactory;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.commons.lang.time.DateUtils;
+
+public abstract class AbstractMetricsImpl {
+
+ public class Metric {
+
+ private long count = 0;
+ private long avg = 0;
+ private long min = 0;
+ private long max = 0;
+
+ public long getCount() {
+ return count;
+ }
+
+ public long getAvg() {
+ return avg;
+ }
+
+ public long getMin() {
+ return min;
+ }
+
+ public long getMax() {
+ return max;
+ }
+
+ public void incCount() {
+ count++;
+ }
+
+ public void addAvg(long a) {
+ if (a < 0)
+ return;
+ avg = (long) ((avg * .8) + (a * .2));
+ }
+
+ public void addMin(long a) {
+ if (a < 0)
+ return;
+ min = Math.min(min, a);
+ }
+
+ public void addMax(long a) {
+ if (a < 0)
+ return;
+ max = Math.max(max, a);
+ }
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this).append("count", count).append("average", avg).append("minimum", min).append("maximum", max).toString();
+ }
+
+ }
+
+ static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(AbstractMetricsImpl.class);
+
+ private static ConcurrentHashMap<String,Metric> registry = new ConcurrentHashMap<String,Metric>();
+
+ private boolean currentlyLogging = false;
+
+ private File logDir = null;
+
+ private String metricsPrefix = null;
+
+ private Date today = new Date();
+
+ private File logFile = null;
+
+ private Writer logWriter = null;
+
+ private SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
+
+ private SimpleDateFormat logFormatter = new SimpleDateFormat("yyyyMMddhhmmssz");
+
+ private MetricsConfiguration config = null;
+
+ public AbstractMetricsImpl() {
+ this.metricsPrefix = getMetricsPrefix();
+ config = new MetricsConfiguration(metricsPrefix);
+ }
+
+ /**
+ * Registers a StandardMBean with the MBean Server
- *
- * @throws Exception
+ */
+ public void register(StandardMBean mbean) throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(mbean, getObjectName());
+
+ setupLogging();
+ }
+
+ /**
+ * Registers this MBean with the MBean Server
- *
- * @throws Exception
+ */
+ public void register() throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(this, getObjectName());
+ setupLogging();
+ }
+
+ public void createMetric(String name) {
+ registry.put(name, new Metric());
+ }
+
+ public Metric getMetric(String name) {
+ return registry.get(name);
+ }
+
+ public long getMetricCount(String name) {
+ return registry.get(name).getCount();
+ }
+
+ public long getMetricAvg(String name) {
+ return registry.get(name).getAvg();
+ }
+
+ public long getMetricMin(String name) {
+ return registry.get(name).getMin();
+ }
+
+ public long getMetricMax(String name) {
+ return registry.get(name).getMax();
+ }
+
+ private void setupLogging() throws IOException {
+ if (null == config.getMetricsConfiguration())
+ return;
+ // If we are already logging, then return
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // Check to see if directory exists, else make it
+ String mDir = config.getMetricsConfiguration().getString("logging.dir");
+ if (null != mDir) {
+ File dir = new File(mDir);
+ if (!dir.isDirectory())
+ if (!dir.mkdir())
+ log.warn("Could not create log directory: " + dir);
+ logDir = dir;
+ // Create new log file
+ startNewLog();
+ }
+ currentlyLogging = true;
+ }
+ }
+
+ private void startNewLog() throws IOException {
+ if (null != logWriter) {
+ logWriter.flush();
+ logWriter.close();
+ }
+ logFile = new File(logDir, metricsPrefix + "-" + formatter.format(today) + ".log");
+ if (!logFile.exists()) {
+ if (!logFile.createNewFile()) {
+ log.error("Unable to create new log file");
+ currentlyLogging = false;
+ return;
+ }
+ }
+ logWriter = new OutputStreamWriter(new FileOutputStream(logFile, true), Constants.UTF8);
+ }
+
+ private void writeToLog(String name) throws IOException {
+ if (null == logWriter)
+ return;
+ // Increment the date if we have to
+ Date now = new Date();
+ if (!DateUtils.isSameDay(today, now)) {
+ today = now;
+ startNewLog();
+ }
+ logWriter.append(logFormatter.format(now)).append(" Metric: ").append(name).append(": ").append(registry.get(name).toString()).append("\n");
+ }
+
+ public void add(String name, long time) {
+ if (isEnabled()) {
+ registry.get(name).incCount();
+ registry.get(name).addAvg(time);
+ registry.get(name).addMin(time);
+ registry.get(name).addMax(time);
+ // If we are not currently logging and should be, then initialize
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ try {
+ setupLogging();
+ } catch (IOException ioe) {
+ log.error("Error setting up log", ioe);
+ }
+ } else if (currentlyLogging && !config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // if we are currently logging and shouldn't be, then close logs
+ try {
+ logWriter.flush();
+ logWriter.close();
+ logWriter = null;
+ logFile = null;
+ } catch (Exception e) {
+ log.error("Error stopping metrics logging", e);
+ }
+ currentlyLogging = false;
+ }
+ if (currentlyLogging) {
+ try {
+ writeToLog(name);
+ } catch (IOException ioe) {
+ log.error("Error writing to metrics log", ioe);
+ }
+ }
+ }
+ }
+
+ public boolean isEnabled() {
+ return config.isEnabled();
+ }
+
+ protected abstract ObjectName getObjectName();
+
+ protected abstract String getMetricsPrefix();
+
+ @Override
+ protected void finalize() {
+ if (null != logWriter) {
+ try {
+ logWriter.close();
+ } catch (Exception e) {
+ // do nothing
+ } finally {
+ logWriter = null;
+ }
+ }
+ logFile = null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
index 34d43f2,0000000..bbaf592
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
@@@ -1,175 -1,0 +1,171 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+public class ZKAuthorizor implements Authorizor {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static Authorizor zkAuthorizorInstance = null;
+
+ private final String ZKUserAuths = "/Authorizations";
+
+ private String ZKUserPath;
+ private final ZooCache zooCache;
+
+ public static synchronized Authorizor getInstance() {
+ if (zkAuthorizorInstance == null)
+ zkAuthorizorInstance = new ZKAuthorizor();
+ return zkAuthorizorInstance;
+ }
+
+ public ZKAuthorizor() {
+ zooCache = new ZooCache();
+ }
+
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ }
+
+ @Override
+ public Authorizations getCachedUserAuthorizations(String user) {
+ byte[] authsBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserAuths);
+ if (authsBytes != null)
+ return ZKSecurityTool.convertAuthorizations(authsBytes);
+ return Authorizations.EMPTY;
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator auth, PermissionHandler pm) {
+ return true;
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the metadata tables
+ tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+ tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
+ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserAuths, ZKSecurityTool.convertAuthorizations(Authorizations.EMPTY), NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
- /**
- * @param user
- * @throws AccumuloSecurityException
- */
+ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void dropUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserAuths, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public void changeAuthorizations(String user, Authorizations authorizations) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserAuths, ZKSecurityTool.convertAuthorizations(authorizations),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean isValidAuthorizations(String user, List<ByteBuffer> auths) throws AccumuloSecurityException {
+ Collection<ByteBuffer> userauths = getCachedUserAuthorizations(user).getAuthorizationsBB();
+ for (ByteBuffer auth : auths)
+ if (!userauths.contains(auth))
+ return false;
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
index 6319653,0000000..1b7e7d3
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
@@@ -1,517 -1,0 +1,513 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.Code;
+
+/**
+ *
+ */
+public class ZKPermHandler implements PermissionHandler {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static PermissionHandler zkPermHandlerInstance = null;
+
+ private String ZKUserPath;
+ private String ZKTablePath;
+ private String ZKNamespacePath;
+ private final ZooCache zooCache;
+ private final String ZKUserSysPerms = "/System";
+ private final String ZKUserTablePerms = "/Tables";
+ private final String ZKUserNamespacePerms = "/Namespaces";
+
+ public static synchronized PermissionHandler getInstance() {
+ if (zkPermHandlerInstance == null)
+ zkPermHandlerInstance = new ZKPermHandler();
+ return zkPermHandlerInstance;
+ }
+
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ ZKTablePath = ZKSecurityTool.getInstancePath(instanceId) + "/tables";
+ ZKNamespacePath = ZKSecurityTool.getInstancePath(instanceId) + "/namespaces";
+ }
+
+ public ZKPermHandler() {
+ zooCache = new ZooCache();
+ }
+
+ @Override
+ public boolean hasTablePermission(String user, String table, TablePermission permission) throws TableNotFoundException {
+ byte[] serializedPerms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table;
+ ZooReaderWriter.getRetryingInstance().sync(path);
+ serializedPerms = ZooReaderWriter.getRetryingInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ // maybe the table was just deleted?
+ try {
+ // check for existence:
+ ZooReaderWriter.getRetryingInstance().getData(ZKTablePath + "/" + table, null);
+ // it's there, you don't have permission
+ return false;
+ } catch (InterruptedException ex) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ } catch (KeeperException ex) {
+ // not there, throw an informative exception
+ if (e.code() == Code.NONODE) {
+ throw new TableNotFoundException(null, table, "while checking permissions");
+ }
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ }
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasNamespacePermission(String user, String namespace, NamespacePermission permission) throws NamespaceNotFoundException {
+ byte[] serializedPerms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace;
+ ZooReaderWriter.getRetryingInstance().sync(path);
+ serializedPerms = ZooReaderWriter.getRetryingInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ // maybe the namespace was just deleted?
+ try {
+ // check for existence:
+ ZooReaderWriter.getRetryingInstance().getData(ZKNamespacePath + "/" + namespace, null);
+ // it's there, you don't have permission
+ return false;
+ } catch (InterruptedException ex) {
+ log.warn("Unhandled InterruptedException, failing closed for namespace permission check", e);
+ return false;
+ } catch (KeeperException ex) {
+ // not there, throw an informative exception
+ if (e.code() == Code.NONODE) {
+ throw new NamespaceNotFoundException(null, namespace, "while checking permissions");
+ }
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ }
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasCachedNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
+ NamespaceNotFoundException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ try {
+ byte[] permBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ Set<SystemPermission> perms;
+ if (permBytes == null) {
+ perms = new TreeSet<SystemPermission>();
+ } else {
+ perms = ZKSecurityTool.convertSystemPermissions(permBytes);
+ }
+
+ if (perms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(perms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ Set<TablePermission> tablePerms;
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null)
+ tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ else
+ tablePerms = new TreeSet<TablePermission>();
+
+ try {
+ if (tablePerms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void grantNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
+ Set<NamespacePermission> namespacePerms;
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+ if (serializedPerms != null)
+ namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
+ else
+ namespacePerms = new TreeSet<NamespacePermission>();
+
+ try {
+ if (namespacePerms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, ZKSecurityTool.convertNamespacePermissions(namespacePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] sysPermBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+
+ // User had no system permission, nothing to revoke.
+ if (sysPermBytes == null)
+ return;
+
+ Set<SystemPermission> sysPerms = ZKSecurityTool.convertSystemPermissions(sysPermBytes);
+
+ try {
+ if (sysPerms.remove(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(sysPerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+
+ // User had no table permission, nothing to revoke.
+ if (serializedPerms == null)
+ return;
+
+ Set<TablePermission> tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ try {
+ if (tablePerms.remove(permission)) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ if (tablePerms.size() == 0)
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ else
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+
+ // User had no namespace permission, nothing to revoke.
+ if (serializedPerms == null)
+ return;
+
+ Set<NamespacePermission> namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
+ try {
+ if (namespacePerms.remove(permission)) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ if (namespacePerms.size() == 0)
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
+ else
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, ZKSecurityTool.convertNamespacePermissions(namespacePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanTablePermissions(String table) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ for (String user : zooCache.getChildren(ZKUserPath))
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanNamespacePermissions(String namespace) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ for (String user : zooCache.getChildren(ZKUserPath))
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the system tables
+ tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+ tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+ // essentially the same but on the system namespace, the ALTER_TABLE permission is now redundant
+ Map<String,Set<NamespacePermission>> namespacePerms = new HashMap<String,Set<NamespacePermission>>();
+ namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_NAMESPACE));
+ namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
+ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(rootPerms), NodeExistsPolicy.FAIL);
+ for (Entry<String,Set<TablePermission>> entry : tablePerms.entrySet())
+ createTablePerm(rootuser, entry.getKey(), entry.getValue());
+ for (Entry<String,Set<NamespacePermission>> entry : namespacePerms.entrySet())
+ createNamespacePerm(rootuser, entry.getKey(), entry.getValue());
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
- /**
- * @param user
- * @throws AccumuloSecurityException
- */
+ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms, new byte[0], NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Sets up a new table configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
+ */
+ private void createTablePerm(String user, String table, Set<TablePermission> perms) throws KeeperException, InterruptedException {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
+ ZKSecurityTool.convertTablePermissions(perms), NodeExistsPolicy.FAIL);
+ }
+ }
+
+ /**
+ * Sets up a new namespace configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
+ */
+ private void createNamespacePerm(String user, String namespace, Set<NamespacePermission> perms) throws KeeperException, InterruptedException {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace,
+ ZKSecurityTool.convertNamespacePermissions(perms), NodeExistsPolicy.FAIL);
+ }
+ }
+
+ @Override
+ public void cleanUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserSysPerms, NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms, NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserSysPerms;
+ ZooReaderWriter.getRetryingInstance().sync(path);
+ perms = ZooReaderWriter.getRetryingInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator authent, Authorizor author) {
+ return true;
+ }
+
+ @Override
+ public void initTable(String table) throws AccumuloSecurityException {
+ // All proper housekeeping is done on delete and permission granting, no work needs to be done here
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
index e16bd06,0000000..cf1a065
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
@@@ -1,62 -1,0 +1,59 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.TokenProperty;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.security.handler.Authenticator;
+import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+
+/**
+ *
+ */
+public class LoginProperties {
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ AccumuloConfiguration config = ServerConfiguration.getSystemConfiguration(HdfsZooInstance.getInstance());
+ Authenticator authenticator = AccumuloClassLoader.getClassLoader().loadClass(config.get(Property.INSTANCE_SECURITY_AUTHENTICATOR))
+ .asSubclass(Authenticator.class).newInstance();
+
+ List<Set<TokenProperty>> tokenProps = new ArrayList<Set<TokenProperty>>();
+
+ for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) {
+ tokenProps.add(tokenType.newInstance().getProperties());
+ }
+
+ System.out.println("Supported token types for " + authenticator.getClass().getName() + " are : ");
+ for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) {
+ System.out.println("\t" + tokenType.getName() + ", which accepts the following properties : ");
+
+ for (TokenProperty tokenProperty : tokenType.newInstance().getProperties()) {
+ System.out.println("\t\t" + tokenProperty);
+ }
+
+ System.out.println();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
index 6e5607e,0000000..37ef5f1
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
@@@ -1,128 -1,0 +1,124 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Stack;
+
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import com.beust.jcommander.Parameter;
+
+public class RestoreZookeeper {
+
+ private static class Restore extends DefaultHandler {
+ IZooReaderWriter zk = null;
+ Stack<String> cwd = new Stack<String>();
+ boolean overwrite = false;
+
+ Restore(IZooReaderWriter zk, boolean overwrite) {
+ this.zk = zk;
+ this.overwrite = overwrite;
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException {
+ if ("node".equals(name)) {
+ String child = attributes.getValue("name");
+ if (child == null)
+ throw new RuntimeException("name attribute not set");
+ String encoding = attributes.getValue("encoding");
+ String value = attributes.getValue("value");
+ if (value == null)
+ value = "";
+ String path = cwd.lastElement() + "/" + child;
+ create(path, value, encoding);
+ cwd.push(path);
+ } else if ("dump".equals(name)) {
+ String root = attributes.getValue("root");
+ if (root.equals("/"))
+ cwd.push("");
+ else
+ cwd.push(root);
+ create(root, "", Constants.UTF8.name());
+ }
+ }
+
+ @Override
+ public void endElement(String uri, String localName, String name) throws SAXException {
+ cwd.pop();
+ }
+
+ // assume UTF-8 if not "base64"
+ private void create(String path, String value, String encoding) {
+ byte[] data = value.getBytes(Constants.UTF8);
+ if ("base64".equals(encoding))
+ data = Base64.decodeBase64(data);
+ try {
+ try {
+ zk.putPersistentData(path, data, overwrite ? NodeExistsPolicy.OVERWRITE : NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ if (e.code().equals(KeeperException.Code.NODEEXISTS))
+ throw new RuntimeException(path + " exists. Remove it first.");
+ throw e;
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names = {"-z", "--keepers"})
+ String keepers = "localhost:2181";
+ @Parameter(names = "--overwrite")
+ boolean overwrite = false;
+ @Parameter(names = "--file")
+ String file;
+ }
+
- /**
- * @param args
- * @throws Exception
- */
+ public static void main(String[] args) throws Exception {
+ Logger.getRootLogger().setLevel(Level.WARN);
+ Opts opts = new Opts();
+ opts.parseArgs(RestoreZookeeper.class.getName(), args);
+
+ InputStream in = System.in;
+ if (opts.file != null) {
+ in = new FileInputStream(opts.file);
+ }
+
+ SAXParserFactory factory = SAXParserFactory.newInstance();
+ SAXParser parser = factory.newSAXParser();
+ parser.parse(in, new Restore(ZooReaderWriter.getInstance(), opts.overwrite));
+ in.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
index cb932d7,0000000..9a54927
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
@@@ -1,290 -1,0 +1,287 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.NumUtil;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+public class TableDiskUsage {
+
+ private static final Logger log = Logger.getLogger(Logger.class);
+ private int nextInternalId = 0;
+ private Map<String,Integer> internalIds = new HashMap<String,Integer>();
+ private Map<Integer,String> externalIds = new HashMap<Integer,String>();
+ private Map<String,Integer[]> tableFiles = new HashMap<String,Integer[]>();
+ private Map<String,Long> fileSizes = new HashMap<String,Long>();
+
+ void addTable(String tableId) {
+ if (internalIds.containsKey(tableId))
+ throw new IllegalArgumentException("Already added table " + tableId);
+
+ int iid = nextInternalId++;
+
+ internalIds.put(tableId, iid);
+ externalIds.put(iid, tableId);
+ }
+
+ void linkFileAndTable(String tableId, String file) {
+ int internalId = internalIds.get(tableId);
+
+ Integer[] tables = tableFiles.get(file);
+ if (tables == null) {
+ tables = new Integer[internalIds.size()];
+ for (int i = 0; i < tables.length; i++)
+ tables[i] = 0;
+ tableFiles.put(file, tables);
+ }
+
+ tables[internalId] = 1;
+ }
+
+ void addFileSize(String file, long size) {
+ fileSizes.put(file, size);
+ }
+
+ Map<List<String>,Long> calculateUsage() {
+
+ Map<List<Integer>,Long> usage = new HashMap<List<Integer>,Long>();
+
+ for (Entry<String,Integer[]> entry : tableFiles.entrySet()) {
+ log.info("fileSizes " + fileSizes + " key " + Arrays.asList(entry.getKey()));
+ List<Integer> key = Arrays.asList(entry.getValue());
+ Long size = fileSizes.get(entry.getKey());
+
+ Long tablesUsage = usage.get(key);
+ if (tablesUsage == null)
+ tablesUsage = 0l;
+
+ tablesUsage += size;
+
+ usage.put(key, tablesUsage);
+
+ }
+
+ Map<List<String>,Long> externalUsage = new HashMap<List<String>,Long>();
+
+ for (Entry<List<Integer>,Long> entry : usage.entrySet()) {
+ List<String> externalKey = new ArrayList<String>();
+ List<Integer> key = entry.getKey();
+ for (int i = 0; i < key.size(); i++)
+ if (key.get(i) != 0)
+ externalKey.add(externalIds.get(i));
+
+ externalUsage.put(externalKey, entry.getValue());
+ }
+
+ return externalUsage;
+ }
+
+ public interface Printer {
+ void print(String line);
+ }
+
+ public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String> tables, VolumeManager fs, Connector conn, boolean humanReadable)
+ throws TableNotFoundException, IOException {
+ printDiskUsage(acuConf, tables, fs, conn, new Printer() {
+ @Override
+ public void print(String line) {
+ System.out.println(line);
+ }
+ }, humanReadable);
+ }
+
+ public static Map<TreeSet<String>,Long> getDiskUsage(AccumuloConfiguration acuConf, Set<String> tableIds, VolumeManager fs, Connector conn)
+ throws IOException {
+ TableDiskUsage tdu = new TableDiskUsage();
+
+ for (String tableId : tableIds)
+ tdu.addTable(tableId);
+
+ HashSet<String> tablesReferenced = new HashSet<String>(tableIds);
+ HashSet<String> emptyTableIds = new HashSet<String>();
+ HashSet<String> nameSpacesReferenced = new HashSet<String>();
+
+ for (String tableId : tableIds) {
+ Scanner mdScanner = null;
+ try {
+ mdScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ } catch (TableNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ mdScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+
+ if (!mdScanner.iterator().hasNext()) {
+ emptyTableIds.add(tableId);
+ }
+
+ for (Entry<Key,Value> entry : mdScanner) {
+ String file = entry.getKey().getColumnQualifier().toString();
+ String parts[] = file.split("/");
+ String uniqueName = parts[parts.length - 1];
+ if (file.contains(":") || file.startsWith("../")) {
+ String ref = parts[parts.length - 3];
+ if (!ref.equals(tableId)) {
+ tablesReferenced.add(ref);
+ }
+ if (file.contains(":") && parts.length > 3) {
+ List<String> base = Arrays.asList(Arrays.copyOf(parts, parts.length - 3));
+ nameSpacesReferenced.add(StringUtil.join(base, "/"));
+ }
+ }
+
+ tdu.linkFileAndTable(tableId, uniqueName);
+ }
+ }
+
+ for (String tableId : tablesReferenced) {
+ for (String tableDir : nameSpacesReferenced) {
+ FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*"));
+ if (files != null) {
+ for (FileStatus fileStatus : files) {
+ // Assumes that all filenames are unique
+ String name = fileStatus.getPath().getName();
+ tdu.addFileSize(name, fileStatus.getLen());
+ }
+ }
+ }
+ }
+
+ HashMap<String,String> reverseTableIdMap = new HashMap<String,String>();
+ for (Entry<String,String> entry : conn.tableOperations().tableIdMap().entrySet())
+ reverseTableIdMap.put(entry.getValue(), entry.getKey());
+
+ TreeMap<TreeSet<String>,Long> usage = new TreeMap<TreeSet<String>,Long>(new Comparator<TreeSet<String>>() {
+
+ @Override
+ public int compare(TreeSet<String> o1, TreeSet<String> o2) {
+ int len1 = o1.size();
+ int len2 = o2.size();
+
+ int min = Math.min(len1, len2);
+
+ Iterator<String> iter1 = o1.iterator();
+ Iterator<String> iter2 = o2.iterator();
+
+ int count = 0;
+
+ while (count < min) {
+ String s1 = iter1.next();
+ String s2 = iter2.next();
+
+ int cmp = s1.compareTo(s2);
+
+ if (cmp != 0)
+ return cmp;
+
+ count++;
+ }
+
+ return len1 - len2;
+ }
+ });
+
+ for (Entry<List<String>,Long> entry : tdu.calculateUsage().entrySet()) {
+ TreeSet<String> tableNames = new TreeSet<String>();
+ for (String tableId : entry.getKey())
+ tableNames.add(reverseTableIdMap.get(tableId));
+
+ usage.put(tableNames, entry.getValue());
+ }
+
+ if (!emptyTableIds.isEmpty()) {
+ TreeSet<String> emptyTables = new TreeSet<String>();
+ for (String tableId : emptyTableIds) {
+ emptyTables.add(reverseTableIdMap.get(tableId));
+ }
+ usage.put(emptyTables, 0L);
+ }
+
+ return usage;
+ }
+
+ public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String> tables, VolumeManager fs, Connector conn, Printer printer,
+ boolean humanReadable) throws TableNotFoundException, IOException {
+
+ HashSet<String> tableIds = new HashSet<String>();
+
+ for (String tableName : tables) {
+ String tableId = conn.tableOperations().tableIdMap().get(tableName);
+ if (tableId == null)
+ throw new TableNotFoundException(null, tableName, "Table " + tableName + " not found");
+
+ tableIds.add(tableId);
+ }
+
+ Map<TreeSet<String>,Long> usage = getDiskUsage(acuConf, tableIds, fs, conn);
+
+ String valueFormat = humanReadable ? "%9s" : "%,24d";
+ for (Entry<TreeSet<String>,Long> entry : usage.entrySet()) {
+ Object value = humanReadable ? NumUtil.bigNumberForSize(entry.getValue()) : entry.getValue();
+ printer.print(String.format(valueFormat + " %s", value, entry.getKey()));
+ }
+ }
+
+ static class Opts extends ClientOpts {
+ @Parameter(description = " <table> { <table> ... } ")
+ List<String> tables = new ArrayList<String>();
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ VolumeManager fs = VolumeManagerImpl.get();
+ Opts opts = new Opts();
+ opts.parseArgs(TableDiskUsage.class.getName(), args);
+ Connector conn = opts.getConnector();
+ org.apache.accumulo.server.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(), opts.tables, fs, conn, false);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
index 2fc0bd3,0000000..34c2151
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
@@@ -1,75 -1,0 +1,72 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+import com.beust.jcommander.Parameter;
+
+public class TabletServerLocks {
+
+ static class Opts extends Help {
+ @Parameter(names="-list")
+ boolean list = false;
+ @Parameter(names="-delete")
+ String delete = null;
+ }
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+
+ Instance instance = HdfsZooInstance.getInstance();
+ String tserverPath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+ Opts opts = new Opts();
+ opts.parseArgs(TabletServerLocks.class.getName(), args);
+
+ ZooCache cache = new ZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+
+ if (opts.list) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ List<String> tabletServers = zoo.getChildren(tserverPath);
+
+ for (String tabletServer : tabletServers) {
+ byte[] lockData = ZooLock.getLockData(cache, tserverPath + "/" + tabletServer, null);
+ String holder = null;
+ if (lockData != null) {
+ holder = new String(lockData, Constants.UTF8);
+ }
+
+ System.out.printf("%32s %16s%n", tabletServer, holder);
+ }
+ } else if (opts.delete != null) {
+ ZooLock.deleteLock(tserverPath + "/" + args[1]);
+ } else {
+ System.out.println("Usage : " + TabletServerLocks.class.getName() + " -list|-delete <tserver lock>");
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
index 13bcdbe,0000000..f1fdde4
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
@@@ -1,93 -1,0 +1,95 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.accumulo.core.data.Value;
+
+/**
+ *
+ */
+public class MemValue extends Value {
+ int kvCount;
+ boolean merged = false;
+
+ /**
+ * @param value
+ * Value
+ * @param kv
+ * kv count
+ */
+ public MemValue(byte[] value, int kv) {
+ super(value);
+ this.kvCount = kv;
+ }
+
+ public MemValue() {
+ super();
+ this.kvCount = Integer.MAX_VALUE;
+ }
+
+ public MemValue(Value value, int kv) {
+ super(value);
+ this.kvCount = kv;
+ }
+
+ // Override
++ @Override
+ public void write(final DataOutput out) throws IOException {
+ if (!merged) {
+ byte[] combinedBytes = new byte[getSize() + 4];
+ System.arraycopy(value, 0, combinedBytes, 4, getSize());
+ combinedBytes[0] = (byte) (kvCount >>> 24);
+ combinedBytes[1] = (byte) (kvCount >>> 16);
+ combinedBytes[2] = (byte) (kvCount >>> 8);
+ combinedBytes[3] = (byte) (kvCount);
+ value = combinedBytes;
+ merged = true;
+ }
+ super.write(out);
+ }
+
++ @Override
+ public void set(final byte[] b) {
+ super.set(b);
+ merged = false;
+ }
+
++ @Override
+ public void copy(byte[] b) {
+ super.copy(b);
+ merged = false;
+ }
+
+ /**
+ * Takes a Value and will take out the embedded kvCount, and then return that value while replacing the Value with the original unembedded version
+ *
- * @param v
+ * @return The kvCount embedded in v.
+ */
+ public static int splitKVCount(Value v) {
+ if (v instanceof MemValue)
+ return ((MemValue) v).kvCount;
+
+ byte[] originalBytes = new byte[v.getSize() - 4];
+ byte[] combined = v.get();
+ System.arraycopy(combined, 4, originalBytes, 0, originalBytes.length);
+ v.set(originalBytes);
+ return (combined[0] << 24) + ((combined[1] & 0xFF) << 16) + ((combined[2] & 0xFF) << 8) + (combined[3] & 0xFF);
+ }
+}