You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2014/11/01 05:57:07 UTC
[13/50] [abbrv] Merge branch '1.5' into 1.6
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
index 71b7155,0000000..78f8c7b
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
@@@ -1,318 -1,0 +1,320 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tables;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.security.SecurityPermission;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.util.TablePropUtil;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.Watcher.Event.EventType;
+
+public class TableManager {
+ private static SecurityPermission TABLE_MANAGER_PERMISSION = new SecurityPermission("tableManagerPermission");
+
+ private static final Logger log = Logger.getLogger(TableManager.class);
+ private static final Set<TableObserver> observers = Collections.synchronizedSet(new HashSet<TableObserver>());
+ private static final Map<String,TableState> tableStateCache = Collections.synchronizedMap(new HashMap<String,TableState>());
+ private static final byte[] ZERO_BYTE = new byte[] {'0'};
+
+ private static TableManager tableManager = null;
+
+ private final Instance instance;
+ private ZooCache zooStateCache;
+
+ public static void prepareNewNamespaceState(String instanceId, String namespaceId, String namespace, NodeExistsPolicy existsPolicy) throws KeeperException,
+ InterruptedException {
+ log.debug("Creating ZooKeeper entries for new namespace " + namespace + " (ID: " + namespaceId + ")");
+ String zPath = Constants.ZROOT + "/" + instanceId + Constants.ZNAMESPACES + "/" + namespaceId;
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ zoo.putPersistentData(zPath, new byte[0], existsPolicy);
- zoo.putPersistentData(zPath + Constants.ZNAMESPACE_NAME, namespace.getBytes(Constants.UTF8), existsPolicy);
++ zoo.putPersistentData(zPath + Constants.ZNAMESPACE_NAME, namespace.getBytes(UTF_8), existsPolicy);
+ zoo.putPersistentData(zPath + Constants.ZNAMESPACE_CONF, new byte[0], existsPolicy);
+ }
+
+ public static void prepareNewTableState(String instanceId, String tableId, String namespaceId, String tableName, TableState state,
+ NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException {
+ // state gets created last
+ log.debug("Creating ZooKeeper entries for new table " + tableName + " (ID: " + tableId + ") in namespace (ID: " + namespaceId + ")");
+ Pair<String,String> qualifiedTableName = Tables.qualify(tableName);
+ tableName = qualifiedTableName.getSecond();
+ String zTablePath = Constants.ZROOT + "/" + instanceId + Constants.ZTABLES + "/" + tableId;
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ zoo.putPersistentData(zTablePath, new byte[0], existsPolicy);
+ zoo.putPersistentData(zTablePath + Constants.ZTABLE_CONF, new byte[0], existsPolicy);
- zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAMESPACE, namespaceId.getBytes(Constants.UTF8), existsPolicy);
- zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAME, tableName.getBytes(Constants.UTF8), existsPolicy);
++ zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAMESPACE, namespaceId.getBytes(UTF_8), existsPolicy);
++ zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAME, tableName.getBytes(UTF_8), existsPolicy);
+ zoo.putPersistentData(zTablePath + Constants.ZTABLE_FLUSH_ID, ZERO_BYTE, existsPolicy);
+ zoo.putPersistentData(zTablePath + Constants.ZTABLE_COMPACT_ID, ZERO_BYTE, existsPolicy);
+ zoo.putPersistentData(zTablePath + Constants.ZTABLE_COMPACT_CANCEL_ID, ZERO_BYTE, existsPolicy);
- zoo.putPersistentData(zTablePath + Constants.ZTABLE_STATE, state.name().getBytes(Constants.UTF8), existsPolicy);
++ zoo.putPersistentData(zTablePath + Constants.ZTABLE_STATE, state.name().getBytes(UTF_8), existsPolicy);
+ }
+
+ public synchronized static TableManager getInstance() {
+ SecurityManager sm = System.getSecurityManager();
+ if (sm != null) {
+ sm.checkPermission(TABLE_MANAGER_PERMISSION);
+ }
+ if (tableManager == null)
+ tableManager = new TableManager();
+ return tableManager;
+ }
+
+ private TableManager() {
+ instance = HdfsZooInstance.getInstance();
+ zooStateCache = new ZooCache(new TableStateWatcher());
+ updateTableStateCache();
+ }
+
+ public TableState getTableState(String tableId) {
+ return tableStateCache.get(tableId);
+ }
+
+ public static class IllegalTableTransitionException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ final TableState oldState;
+ final TableState newState;
+
+ public IllegalTableTransitionException(TableState oldState, TableState newState) {
+ this.oldState = oldState;
+ this.newState = newState;
+ }
+
+ public TableState getOldState() {
+ return oldState;
+ }
+
+ public TableState getNewState() {
+ return newState;
+ }
+
+ }
+
+ public synchronized void transitionTableState(final String tableId, final TableState newState) {
+ String statePath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE;
+
+ try {
- ZooReaderWriter.getInstance().mutate(statePath, newState.name().getBytes(Constants.UTF8), ZooUtil.PUBLIC, new Mutator() {
++ ZooReaderWriter.getInstance().mutate(statePath, newState.name().getBytes(UTF_8), ZooUtil.PUBLIC, new Mutator() {
+ @Override
+ public byte[] mutate(byte[] oldData) throws Exception {
+ TableState oldState = TableState.UNKNOWN;
+ if (oldData != null)
- oldState = TableState.valueOf(new String(oldData, Constants.UTF8));
++ oldState = TableState.valueOf(new String(oldData, UTF_8));
+ boolean transition = true;
+ // +--------+
+ // v |
+ // NEW -> (ONLINE|OFFLINE)+--- DELETING
+ switch (oldState) {
+ case NEW:
+ transition = (newState == TableState.OFFLINE || newState == TableState.ONLINE);
+ break;
+ case ONLINE: // fall-through intended
+ case UNKNOWN:// fall through intended
+ case OFFLINE:
+ transition = (newState != TableState.NEW);
+ break;
+ case DELETING:
+ // Can't transition to any state from DELETING
+ transition = false;
+ break;
+ }
+ if (!transition)
+ throw new IllegalTableTransitionException(oldState, newState);
+ log.debug("Transitioning state for table " + tableId + " from " + oldState + " to " + newState);
- return newState.name().getBytes(Constants.UTF8);
++ return newState.name().getBytes(UTF_8);
+ }
+ });
+ } catch (Exception e) {
+ log.fatal("Failed to transition table to state " + newState);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void updateTableStateCache() {
+ synchronized (tableStateCache) {
+ for (String tableId : zooStateCache.getChildren(ZooUtil.getRoot(instance) + Constants.ZTABLES))
+ if (zooStateCache.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE) != null)
+ updateTableStateCache(tableId);
+ }
+ }
+
+ public TableState updateTableStateCache(String tableId) {
+ synchronized (tableStateCache) {
+ TableState tState = TableState.UNKNOWN;
+ byte[] data = zooStateCache.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE);
+ if (data != null) {
- String sState = new String(data, Constants.UTF8);
++ String sState = new String(data, UTF_8);
+ try {
+ tState = TableState.valueOf(sState);
+ } catch (IllegalArgumentException e) {
+ log.error("Unrecognized state for table with tableId=" + tableId + ": " + sState);
+ }
+ tableStateCache.put(tableId, tState);
+ }
+ return tState;
+ }
+ }
+
+ public void addTable(String tableId, String namespaceId, String tableName, NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException,
+ NamespaceNotFoundException {
+ prepareNewTableState(instance.getInstanceID(), tableId, namespaceId, tableName, TableState.NEW, existsPolicy);
+ updateTableStateCache(tableId);
+ }
+
+ public void cloneTable(String srcTable, String tableId, String tableName, String namespaceId, Map<String,String> propertiesToSet,
+ Set<String> propertiesToExclude, NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException {
+ prepareNewTableState(instance.getInstanceID(), tableId, namespaceId, tableName, TableState.NEW, existsPolicy);
+
+ String srcTablePath = Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + srcTable + Constants.ZTABLE_CONF;
+ String newTablePath = Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_CONF;
+ ZooReaderWriter.getInstance().recursiveCopyPersistent(srcTablePath, newTablePath, NodeExistsPolicy.OVERWRITE);
+
+ for (Entry<String,String> entry : propertiesToSet.entrySet())
+ TablePropUtil.setTableProperty(tableId, entry.getKey(), entry.getValue());
+
+ for (String prop : propertiesToExclude)
+ ZooReaderWriter.getInstance().recursiveDelete(
+ Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_CONF + "/" + prop, NodeMissingPolicy.SKIP);
+
+ updateTableStateCache(tableId);
+ }
+
+ public void removeTable(String tableId) throws KeeperException, InterruptedException {
+ synchronized (tableStateCache) {
+ tableStateCache.remove(tableId);
+ ZooReaderWriter.getInstance().recursiveDelete(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE,
+ NodeMissingPolicy.SKIP);
+ ZooReaderWriter.getInstance().recursiveDelete(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId, NodeMissingPolicy.SKIP);
+ }
+ }
+
+ public boolean addObserver(TableObserver to) {
+ synchronized (observers) {
+ synchronized (tableStateCache) {
+ to.initialize(Collections.unmodifiableMap(tableStateCache));
+ return observers.add(to);
+ }
+ }
+ }
+
+ public boolean removeObserver(TableObserver to) {
+ return observers.remove(to);
+ }
+
+ private class TableStateWatcher implements Watcher {
+ @Override
+ public void process(WatchedEvent event) {
+ if (log.isTraceEnabled())
+ log.trace(event);
+
+ final String zPath = event.getPath();
+ final EventType zType = event.getType();
+
+ String tablesPrefix = ZooUtil.getRoot(instance) + Constants.ZTABLES;
+ String tableId = null;
+
+ if (zPath != null && zPath.startsWith(tablesPrefix + "/")) {
+ String suffix = zPath.substring(tablesPrefix.length() + 1);
+ if (suffix.contains("/")) {
+ String[] sa = suffix.split("/", 2);
+ if (Constants.ZTABLE_STATE.equals("/" + sa[1]))
+ tableId = sa[0];
+ }
+ if (tableId == null) {
+ log.warn("Unknown path in " + event);
+ return;
+ }
+ }
+
+ switch (zType) {
+ case NodeChildrenChanged:
+ if (zPath != null && zPath.equals(tablesPrefix)) {
+ updateTableStateCache();
+ } else {
+ log.warn("Unexpected path " + zPath);
+ }
+ break;
+ case NodeCreated:
+ case NodeDataChanged:
+ // state transition
+ TableState tState = updateTableStateCache(tableId);
+ log.debug("State transition to " + tState + " @ " + event);
+ synchronized (observers) {
+ for (TableObserver to : observers)
+ to.stateChanged(tableId, tState);
+ }
+ break;
+ case NodeDeleted:
+ if (zPath != null
+ && tableId != null
+ && (zPath.equals(tablesPrefix + "/" + tableId + Constants.ZTABLE_STATE) || zPath.equals(tablesPrefix + "/" + tableId + Constants.ZTABLE_CONF) || zPath
+ .equals(tablesPrefix + "/" + tableId + Constants.ZTABLE_NAME)))
+ tableStateCache.remove(tableId);
+ break;
+ case None:
+ switch (event.getState()) {
+ case Expired:
+ if (log.isTraceEnabled())
+ log.trace("Session expired " + event);
+ synchronized (observers) {
+ for (TableObserver to : observers)
+ to.sessionExpired();
+ }
+ break;
+ case SyncConnected:
+ default:
+ if (log.isTraceEnabled())
+ log.trace("Ignored " + event);
+ }
+ break;
+ default:
+ log.warn("Unandled " + event);
+ }
+ }
+ }
+
+ public void removeNamespace(String namespaceId) throws KeeperException, InterruptedException {
+ ZooReaderWriter.getInstance().recursiveDelete(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES + "/" + namespaceId, NodeMissingPolicy.SKIP);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
index eabdc0d,0000000..e10c0c1
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
@@@ -1,79 -1,0 +1,81 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tablets;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Random;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+/**
+ * Allocates unique names for an accumulo instance. The names are unique for the lifetime of the instance.
+ *
+ * This is useful for filenames because it makes caching easy.
+ *
+ */
+
+public class UniqueNameAllocator {
+ private long next = 0;
+ private long maxAllocated = 0;
+ private String nextNamePath;
+ private Random rand;
+
+ private UniqueNameAllocator() {
+ nextNamePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZNEXT_FILE;
+ rand = new Random();
+ }
+
+ public synchronized String getNextName() {
+
+ while (next >= maxAllocated) {
+ final int allocate = 100 + rand.nextInt(100);
+
+ try {
+ byte[] max = ZooReaderWriter.getInstance().mutate(nextNamePath, null, ZooUtil.PRIVATE, new ZooReaderWriter.Mutator() {
+ public byte[] mutate(byte[] currentValue) throws Exception {
- long l = Long.parseLong(new String(currentValue, Constants.UTF8), Character.MAX_RADIX);
++ long l = Long.parseLong(new String(currentValue, UTF_8), Character.MAX_RADIX);
+ l += allocate;
- return Long.toString(l, Character.MAX_RADIX).getBytes(Constants.UTF8);
++ return Long.toString(l, Character.MAX_RADIX).getBytes(UTF_8);
+ }
+ });
+
- maxAllocated = Long.parseLong(new String(max, Constants.UTF8), Character.MAX_RADIX);
++ maxAllocated = Long.parseLong(new String(max, UTF_8), Character.MAX_RADIX);
+ next = maxAllocated - allocate;
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
- return new String(FastFormat.toZeroPaddedString(next++, 7, Character.MAX_RADIX, new byte[0]), Constants.UTF8);
++ return new String(FastFormat.toZeroPaddedString(next++, 7, Character.MAX_RADIX, new byte[0]), UTF_8);
+ }
+
+ private static UniqueNameAllocator instance = null;
+
+ public static synchronized UniqueNameAllocator getInstance() {
+ if (instance == null)
+ instance = new UniqueNameAllocator();
+
+ return instance;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
index cdff21d,0000000..198b2d6
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
@@@ -1,136 -1,0 +1,137 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+public class AddFilesWithMissingEntries {
+
+ static final Logger log = Logger.getLogger(AddFilesWithMissingEntries.class);
+
+ public static class Opts extends ClientOpts {
+ @Parameter(names = "-update", description = "Make changes to the " + MetadataTable.NAME + " table to include missing files")
+ boolean update = false;
+ }
+
+ /**
+ * A utility to add files to the {@value MetadataTable#NAME} table that are not listed in the root tablet. This is a recovery tool for someone who knows what
+ * they are doing. It might be better to save off files, and recover your instance by re-initializing and importing the existing files.
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(AddFilesWithMissingEntries.class.getName(), args, bwOpts);
+
+ final Scanner scanner = opts.getConnector().createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ scanner.setRange(MetadataSchema.TabletsSection.getRange());
+ final Configuration conf = new Configuration();
+ final FileSystem fs = FileSystem.get(conf);
+
+ KeyExtent last = new KeyExtent();
+ String directory = null;
+ Set<String> knownFiles = new HashSet<String>();
+
+ int count = 0;
+ final MultiTableBatchWriter writer = opts.getConnector().createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
+
+ // collect the list of known files and the directory for each extent
+ for (Entry<Key,Value> entry : scanner) {
+ Key key = entry.getKey();
+ KeyExtent ke = new KeyExtent(key.getRow(), (Text) null);
+ // when the key extent changes
+ if (!ke.equals(last)) {
+ if (directory != null) {
+ // add any files in the directory unknown to the key extent
+ count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
+ }
+ directory = null;
+ knownFiles.clear();
+ last = ke;
+ }
+ if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
+ directory = entry.getValue().toString();
+ log.debug("Found directory " + directory + " for row " + key.getRow().toString());
+ } else if (key.compareColumnFamily(DataFileColumnFamily.NAME) == 0) {
+ String filename = key.getColumnQualifier().toString();
+ knownFiles.add(filename);
+ log.debug("METADATA file found: " + filename);
+ }
+ }
+ if (directory != null) {
+ // catch the last key extent
+ count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
+ }
+ log.info("There were " + count + " files that are unknown to the metadata table");
+ writer.close();
+ }
+
+ private static int addUnknownFiles(FileSystem fs, String directory, Set<String> knownFiles, KeyExtent ke, MultiTableBatchWriter writer, boolean update)
+ throws Exception {
+ int count = 0;
+ final String tableId = ke.getTableId().toString();
+ final Text row = ke.getMetadataEntry();
+ log.info(row.toString());
+ for (String dir : ServerConstants.getTablesDirs()) {
+ final Path path = new Path(dir + "/" + tableId + directory);
+ for (FileStatus file : fs.listStatus(path)) {
+ if (file.getPath().getName().endsWith("_tmp") || file.getPath().getName().endsWith("_tmp.rf"))
+ continue;
+ final String filename = directory + "/" + file.getPath().getName();
+ if (!knownFiles.contains(filename)) {
+ count++;
+ final Mutation m = new Mutation(row);
+ String size = Long.toString(file.getLen());
+ String entries = "1"; // lie
+ String value = size + "," + entries;
- m.put(DataFileColumnFamily.NAME, new Text(filename), new Value(value.getBytes(Constants.UTF8)));
++ m.put(DataFileColumnFamily.NAME, new Text(filename), new Value(value.getBytes(UTF_8)));
+ if (update) {
+ writer.getBatchWriter(MetadataTable.NAME).addMutation(m);
+ }
+ }
+ }
+ }
+ return count;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index 2926a3f,0000000..7a1e1ba
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@@ -1,166 -1,0 +1,167 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
+
+import com.beust.jcommander.Parameter;
+
+public class ChangeSecret {
+
+ static class Opts extends ClientOpts {
+ @Parameter(names="--old", description="old zookeeper password", password=true, hidden=true)
+ String oldPass;
+ @Parameter(names="--new", description="new zookeeper password", password=true, hidden=true)
+ String newPass;
+ }
+
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ List<String> argsList = new ArrayList<String>(args.length + 2);
+ argsList.add("--old");
+ argsList.add("--new");
+ argsList.addAll(Arrays.asList(args));
+ opts.parseArgs(ChangeSecret.class.getName(), argsList.toArray(new String[0]));
+ VolumeManager fs = VolumeManagerImpl.get();
+ Instance inst = opts.getInstance();
+ if (!verifyAccumuloIsDown(inst, opts.oldPass))
+ System.exit(-1);
+ String instanceId = rewriteZooKeeperInstance(inst, opts.oldPass, opts.newPass);
+ updateHdfs(fs, inst, instanceId);
+ if (opts.oldPass != null) {
+ deleteInstance(inst, opts.oldPass);
+ }
+ System.out.println("New instance id is " + instanceId);
+ System.out.println("Be sure to put your new secret in accumulo-site.xml");
+ }
+
+ interface Visitor {
+ void visit(ZooReader zoo, String path) throws Exception;
+ }
+
+ private static void recurse(ZooReader zoo, String root, Visitor v) {
+ try {
+ v.visit(zoo, root);
+ for (String child : zoo.getChildren(root)) {
+ recurse(zoo, root + "/" + child, v);
+ }
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ private static boolean verifyAccumuloIsDown(Instance inst, String oldPassword) {
+ ZooReader zooReader = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), oldPassword);
+ String root = ZooUtil.getRoot(inst);
+ final List<String> ephemerals = new ArrayList<String>();
+ recurse(zooReader, root, new Visitor() {
+ public void visit(ZooReader zoo, String path) throws Exception {
+ Stat stat = zoo.getStatus(path);
+ if (stat.getEphemeralOwner() != 0)
+ ephemerals.add(path);
+ }
+ });
+ if (ephemerals.size() == 0) {
+ return true;
+ }
+
+ System.err.println("The following ephemeral nodes exist, something is still running:");
+ for (String path : ephemerals) {
+ System.err.println(path);
+ }
+ return false;
+ }
+
+ private static String rewriteZooKeeperInstance(final Instance inst, String oldPass, String newPass) throws Exception {
+ final ZooReaderWriter orig = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), oldPass);
+ final IZooReaderWriter new_ = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), newPass);
+ final String newInstanceId = UUID.randomUUID().toString();
+ String root = ZooUtil.getRoot(inst);
+ recurse(orig, root, new Visitor() {
+ public void visit(ZooReader zoo, String path) throws Exception {
+ String newPath = path.replace(inst.getInstanceID(), newInstanceId);
+ byte[] data = zoo.getData(path, null);
+ List<ACL> acls = orig.getZooKeeper().getACL(path, new Stat());
+ if (acls.containsAll(Ids.READ_ACL_UNSAFE)) {
+ new_.putPersistentData(newPath, data, NodeExistsPolicy.FAIL);
+ } else {
+ // upgrade
+ if (acls.containsAll(Ids.OPEN_ACL_UNSAFE)) {
+ // make user nodes private, they contain the user's password
+ String parts[] = path.split("/");
+ if (parts[parts.length - 2].equals("users")) {
+ new_.putPrivatePersistentData(newPath, data, NodeExistsPolicy.FAIL);
+ } else {
+ // everything else can have the readable acl
+ new_.putPersistentData(newPath, data, NodeExistsPolicy.FAIL);
+ }
+ } else {
+ new_.putPrivatePersistentData(newPath, data, NodeExistsPolicy.FAIL);
+ }
+ }
+ }
+ });
+ String path = "/accumulo/instances/" + inst.getInstanceName();
+ orig.recursiveDelete(path, NodeMissingPolicy.SKIP);
- new_.putPersistentData(path, newInstanceId.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++ new_.putPersistentData(path, newInstanceId.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
+ return newInstanceId;
+ }
+
+ private static void updateHdfs(VolumeManager fs, Instance inst, String newInstanceId) throws IOException {
+ // Need to recreate the instanceId on all of them to keep consistency
+ for (Volume v : fs.getVolumes()) {
+ final Path instanceId = ServerConstants.getInstanceIdLocation(v);
+ if (!v.getFileSystem().delete(instanceId, true)) {
+ throw new IOException("Could not recursively delete " + instanceId);
+ }
+
+ if (!v.getFileSystem().mkdirs(instanceId)) {
+ throw new IOException("Could not create directory " + instanceId);
+ }
+
+ v.getFileSystem().create(new Path(instanceId, newInstanceId)).close();
+ }
+ }
+
+ private static void deleteInstance(Instance origInstance, String oldPass) throws Exception {
+ IZooReaderWriter orig = new ZooReaderWriter(origInstance.getZooKeepers(), origInstance.getZooKeepersSessionTimeOut(), oldPass);
+ orig.recursiveDelete("/accumulo/" + origInstance.getInstanceID(), NodeMissingPolicy.SKIP);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
index f2074a1,0000000..809a7c8
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
@@@ -1,87 -1,0 +1,89 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+import com.beust.jcommander.Parameter;
+
+public class CleanZookeeper {
+
+ private static final Logger log = Logger.getLogger(CleanZookeeper.class);
+
+ static class Opts extends Help {
+ @Parameter(names={"-z", "--keepers"}, description="comma separated list of zookeeper hosts")
+ String keepers = "localhost:2181";
+ @Parameter(names={"--password"}, description="the system secret", password=true)
+ String auth;
+ }
+
+ /**
+ * @param args
+ * must contain one element: the address of a zookeeper node a second parameter provides an additional authentication value
+ * @throws IOException
+ * error connecting to accumulo or zookeeper
+ */
+ public static void main(String[] args) throws IOException {
+ Opts opts = new Opts();
+ opts.parseArgs(CleanZookeeper.class.getName(), args);
+
+ String root = Constants.ZROOT;
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+ if (opts.auth != null) {
- zk.getZooKeeper().addAuthInfo("digest", ("accumulo:"+opts.auth).getBytes(Constants.UTF8));
++ zk.getZooKeeper().addAuthInfo("digest", ("accumulo:"+opts.auth).getBytes(UTF_8));
+ }
+
+ try {
+ for (String child : zk.getChildren(root)) {
+ if (Constants.ZINSTANCES.equals("/" + child)) {
+ for (String instanceName : zk.getChildren(root + Constants.ZINSTANCES)) {
+ String instanceNamePath = root + Constants.ZINSTANCES + "/" + instanceName;
+ byte[] id = zk.getData(instanceNamePath, null);
- if (id != null && !new String(id, Constants.UTF8).equals(HdfsZooInstance.getInstance().getInstanceID())) {
++ if (id != null && !new String(id, UTF_8).equals(HdfsZooInstance.getInstance().getInstanceID())) {
+ try {
+ zk.recursiveDelete(instanceNamePath, NodeMissingPolicy.SKIP);
+ } catch (KeeperException.NoAuthException ex) {
+ log.warn("Unable to delete " + instanceNamePath);
+ }
+ }
+ }
+ } else if (!child.equals(HdfsZooInstance.getInstance().getInstanceID())) {
+ String path = root + "/" + child;
+ try {
+ zk.recursiveDelete(path, NodeMissingPolicy.SKIP);
+ } catch (KeeperException.NoAuthException ex) {
+ log.warn("Unable to delete " + path);
+ }
+ }
+ }
+ } catch (Exception ex) {
+ System.out.println("Error Occurred: " + ex);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
index 448da86,0000000..d40830f
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
@@@ -1,81 -1,0 +1,83 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.zookeeper.KeeperException;
+
+import com.beust.jcommander.Parameter;
+
+public class DeleteZooInstance {
+
+ static class Opts extends Help {
+ @Parameter(names={"-i", "--instance"}, description="the instance name or id to delete")
+ String instance;
+ }
+
+ static void deleteRetry(IZooReaderWriter zk, String path) throws Exception {
+ for (int i = 0; i < 10; i++){
+ try {
+ zk.recursiveDelete(path, NodeMissingPolicy.SKIP);
+ return;
+ } catch (KeeperException.NotEmptyException ex) {
+ // ignored
+ } catch (Exception ex) {
+ throw ex;
+ }
+ }
+ }
+
+ /**
+ * @param args
+ * : the name or UUID of the instance to be deleted
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(DeleteZooInstance.class.getName(), args);
+
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+ // try instance name:
+ Set<String> instances = new HashSet<String>(zk.getChildren(Constants.ZROOT + Constants.ZINSTANCES));
+ Set<String> uuids = new HashSet<String>(zk.getChildren(Constants.ZROOT));
+ uuids.remove("instances");
+ if (instances.contains(opts.instance)) {
+ String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + opts.instance;
+ byte[] data = zk.getData(path, null);
+ deleteRetry(zk, path);
- deleteRetry(zk, Constants.ZROOT + "/" + new String(data, Constants.UTF8));
++ deleteRetry(zk, Constants.ZROOT + "/" + new String(data, UTF_8));
+ } else if (uuids.contains(opts.instance)) {
+ // look for the real instance name
+ for (String instance : instances) {
+ String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + instance;
+ byte[] data = zk.getData(path, null);
- if (opts.instance.equals(new String(data, Constants.UTF8)))
++ if (opts.instance.equals(new String(data, UTF_8)))
+ deleteRetry(zk, path);
+ }
+ deleteRetry(zk, Constants.ZROOT + "/" + opts.instance);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
index 504956f,0000000..43348df
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
@@@ -1,121 -1,0 +1,122 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import com.beust.jcommander.Parameter;
+
+public class DumpZookeeper {
+
+ static IZooReaderWriter zk = null;
+
+ private static final Logger log = Logger.getLogger(DumpZookeeper.class);
+
+ private static class Encoded {
+ public String encoding;
+ public String value;
+
+ Encoded(String e, String v) {
+ encoding = e;
+ value = v;
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names = "--root", description = "the root of the znode tree to dump")
+ String root = "/";
+ }
+
+ public static void main(String[] args) {
+ Opts opts = new Opts();
+ opts.parseArgs(DumpZookeeper.class.getName(), args);
+
+ Logger.getRootLogger().setLevel(Level.WARN);
+ PrintStream out = System.out;
+ try {
+ zk = ZooReaderWriter.getInstance();
+
+ write(out, 0, "<dump root='%s'>", opts.root);
+ for (String child : zk.getChildren(opts.root, null))
+ if (!child.equals("zookeeper"))
+ dump(out, opts.root, child, 1);
+ write(out, 0, "</dump>");
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+
+ private static void dump(PrintStream out, String root, String child, int indent) throws KeeperException, InterruptedException, UnsupportedEncodingException {
+ String path = root + "/" + child;
+ if (root.endsWith("/"))
+ path = root + child;
+ Stat stat = zk.getStatus(path);
+ if (stat == null)
+ return;
+ String type = "node";
+ if (stat.getEphemeralOwner() != 0) {
+ type = "ephemeral";
+ }
+ if (stat.getNumChildren() == 0) {
+ if (stat.getDataLength() == 0) {
+ write(out, indent, "<%s name='%s'/>", type, child);
+ } else {
+ Encoded value = value(path);
+ write(out, indent, "<%s name='%s' encoding='%s' value='%s'/>", type, child, value.encoding, value.value);
+ }
+ } else {
+ if (stat.getDataLength() == 0) {
+ write(out, indent, "<%s name='%s'>", type, child);
+ } else {
+ Encoded value = value(path);
+ write(out, indent, "<%s name='%s' encoding='%s' value='%s'>", type, child, value.encoding, value.value);
+ }
+ for (String c : zk.getChildren(path, null)) {
+ dump(out, path, c, indent + 1);
+ }
+ write(out, indent, "</node>");
+ }
+ }
+
+ private static Encoded value(String path) throws KeeperException, InterruptedException, UnsupportedEncodingException {
+ byte[] data = zk.getData(path, null);
+ for (int i = 0; i < data.length; i++) {
+ // does this look like simple ascii?
+ if (data[i] < ' ' || data[i] > '~')
+ return new Encoded("base64", Base64.encodeBase64String(data));
+ }
- return new Encoded(Constants.UTF8.name(), new String(data, Constants.UTF8));
++ return new Encoded(UTF_8.name(), new String(data, UTF_8));
+ }
+
+ private static void write(PrintStream out, int indent, String fmt, Object... args) {
+ for (int i = 0; i < indent; i++)
+ out.print(" ");
+ out.println(String.format(fmt, args));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
index f4655dc,0000000..be85b1b
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
@@@ -1,225 -1,0 +1,227 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Formattable;
+import java.util.Formatter;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+public class ListInstances {
+
+ private static final Logger log = Logger.getLogger(ListInstances.class);
+
+ private static final int NAME_WIDTH = 20;
+ private static final int UUID_WIDTH = 37;
+ private static final int MASTER_WIDTH = 30;
+
+ private static final int ZOOKEEPER_TIMER_MILLIS = 30 * 1000;
+
+ static class Opts extends Help {
+ @Parameter(names="--print-errors", description="display errors while listing instances")
+ boolean printErrors = false;
+ @Parameter(names="--print-all", description="print information for all instances, not just those with names")
+ boolean printAll = false;
+ @Parameter(names={"-z", "--zookeepers"}, description="the zookeepers to contact")
+ String keepers = null;
+ }
+ static Opts opts = new Opts();
+ static int errors = 0;
+
+ public static void main(String[] args) {
+ opts.parseArgs(ListInstances.class.getName(), args);
+
+ if (opts.keepers == null) {
+ opts.keepers = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST);
+ }
+
+ String keepers = opts.keepers;
+ boolean printAll = opts.printAll;
+ boolean printErrors = opts.printErrors;
+
+ listInstances(keepers, printAll, printErrors);
+
+ }
+
+ static synchronized void listInstances(String keepers, boolean printAll, boolean printErrors) {
+ errors = 0;
+
+ System.out.println("INFO : Using ZooKeepers " + keepers);
+ ZooReader rdr = new ZooReader(keepers, ZOOKEEPER_TIMER_MILLIS);
+ ZooCache cache = new ZooCache(keepers, ZOOKEEPER_TIMER_MILLIS);
+
+ TreeMap<String,UUID> instanceNames = getInstanceNames(rdr, printErrors);
+
+ System.out.println();
+ printHeader();
+
+ for (Entry<String,UUID> entry : instanceNames.entrySet()) {
+ printInstanceInfo(cache, entry.getKey(), entry.getValue(), printErrors);
+ }
+
+ TreeSet<UUID> instancedIds = getInstanceIDs(rdr, printErrors);
+ instancedIds.removeAll(instanceNames.values());
+
+ if (printAll) {
+ for (UUID uuid : instancedIds) {
+ printInstanceInfo(cache, null, uuid, printErrors);
+ }
+ } else if (instancedIds.size() > 0) {
+ System.out.println();
+ System.out.println("INFO : " + instancedIds.size() + " unamed instances were not printed, run with --print-all to see all instances");
+ } else {
+ System.out.println();
+ }
+
+ if (!printErrors && errors > 0) {
+ System.err.println("WARN : There were " + errors + " errors, run with --print-errors to see more info");
+ }
+ }
+
+ private static class CharFiller implements Formattable {
+
+ char c;
+
+ CharFiller(char c) {
+ this.c = c;
+ }
+
+ @Override
+ public void formatTo(Formatter formatter, int flags, int width, int precision) {
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < width; i++)
+ sb.append(c);
+ formatter.format(sb.toString());
+ }
+
+ }
+
+ private static void printHeader() {
+ System.out.printf(" %-" + NAME_WIDTH + "s| %-" + UUID_WIDTH + "s| %-" + MASTER_WIDTH + "s%n", "Instance Name", "Instance ID", "Master");
+ System.out.printf("%" + (NAME_WIDTH + 1) + "s+%" + (UUID_WIDTH + 1) + "s+%" + (MASTER_WIDTH + 1) + "s%n", new CharFiller('-'), new CharFiller('-'),
+ new CharFiller('-'));
+
+ }
+
+ private static void printInstanceInfo(ZooCache cache, String instanceName, UUID iid, boolean printErrors) {
+ String master = getMaster(cache, iid, printErrors);
+ if (instanceName == null) {
+ instanceName = "";
+ }
+
+ if (master == null) {
+ master = "";
+ }
+
+ System.out.printf("%" + NAME_WIDTH + "s |%" + UUID_WIDTH + "s |%" + MASTER_WIDTH + "s%n", "\"" + instanceName + "\"", iid, master);
+ }
+
+ private static String getMaster(ZooCache cache, UUID iid, boolean printErrors) {
+
+ if (iid == null) {
+ return null;
+ }
+
+ try {
+ String masterLocPath = Constants.ZROOT + "/" + iid + Constants.ZMASTER_LOCK;
+ byte[] master = ZooLock.getLockData(cache, masterLocPath, null);
+ if (master == null) {
+ return null;
+ }
- return new String(master, Constants.UTF8);
++ return new String(master, UTF_8);
+ } catch (Exception e) {
+ handleException(e, printErrors);
+ return null;
+ }
+ }
+
+ private static TreeMap<String,UUID> getInstanceNames(ZooReader zk, boolean printErrors) {
+
+ String instancesPath = Constants.ZROOT + Constants.ZINSTANCES;
+
+ TreeMap<String,UUID> tm = new TreeMap<String,UUID>();
+
+ List<String> names;
+
+ try {
+ names = zk.getChildren(instancesPath);
+ } catch (Exception e) {
+ handleException(e, printErrors);
+ return tm;
+ }
+
+ for (String name : names) {
+ String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
+ try {
- UUID iid = UUID.fromString(new String(zk.getData(instanceNamePath, null), Constants.UTF8));
++ UUID iid = UUID.fromString(new String(zk.getData(instanceNamePath, null), UTF_8));
+ tm.put(name, iid);
+ } catch (Exception e) {
+ handleException(e, printErrors);
+ tm.put(name, null);
+ }
+ }
+
+ return tm;
+ }
+
+ private static TreeSet<UUID> getInstanceIDs(ZooReader zk, boolean printErrors) {
+ TreeSet<UUID> ts = new TreeSet<UUID>();
+
+ try {
+ List<String> children = zk.getChildren(Constants.ZROOT);
+
+ for (String iid : children) {
+ if (iid.equals("instances"))
+ continue;
+ try {
+ ts.add(UUID.fromString(iid));
+ } catch (Exception e) {
+ log.error("Exception: " + e);
+ }
+ }
+ } catch (Exception e) {
+ handleException(e, printErrors);
+ }
+
+ return ts;
+ }
+
+ private static void handleException(Exception e, boolean printErrors) {
+ if (printErrors) {
+ e.printStackTrace();
+ }
+
+ errors++;
+ }
+}