You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mm...@apache.org on 2021/01/04 20:16:19 UTC
[accumulo] branch main updated: Create listtablets shell command.
Closes #1317 (#1821)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 376910e Create listtablets shell command. Closes #1317 (#1821)
376910e is described below
commit 376910eb26e51d7361f3852705a954ad998e4c6b
Author: Mike Miller <mm...@apache.org>
AuthorDate: Mon Jan 4 15:16:08 2021 -0500
Create listtablets shell command. Closes #1317 (#1821)
* New command for debugging tablets called listtablets
* Added getLiveTServers() to TabletMetadata for generating a list of
tservers that currently have a lock in ZK, similar to master.
* The list of live tservers is passed to TabletMetadata in order to get
the current state of a tablet
* Command will print one line for every tablet in a table
* Created TabletMetadataIT for testing getLiveTServers()
Co-authored-by: EdColeman <de...@etcoleman.com>
---
.../core/metadata/schema/TabletMetadata.java | 48 +++
.../main/java/org/apache/accumulo/shell/Shell.java | 8 +-
.../shell/commands/ListTabletsCommand.java | 447 +++++++++++++++++++++
.../shell/commands/ListTabletsCommandTest.java | 193 +++++++++
.../accumulo/test/functional/TabletMetadataIT.java | 78 ++++
5 files changed, 771 insertions(+), 3 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index 8be6cbd..1286fce 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@ -18,6 +18,7 @@
*/
package org.apache.accumulo.core.metadata.schema;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn;
import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_QUAL;
import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_QUAL;
@@ -29,18 +30,22 @@ import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSec
import java.util.Collection;
import java.util.EnumSet;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
+import java.util.Optional;
import java.util.OptionalLong;
import java.util.Set;
import java.util.SortedMap;
import java.util.function.Function;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.RowIterator;
import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
@@ -65,7 +70,12 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Se
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
import org.apache.accumulo.core.util.HostAndPort;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooLock;
import org.apache.hadoop.io.Text;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -75,6 +85,7 @@ import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Iterators;
public class TabletMetadata {
+ private static final Logger log = LoggerFactory.getLogger(TabletMetadata.class);
private TableId tableId;
private Text prevEndRow;
@@ -435,4 +446,41 @@ public class TabletMetadata {
te.fetchedCols = EnumSet.of(ColumnType.PREV_ROW);
return te;
}
+
+ /**
+ * Get the tservers that are live from ZK. Live servers will have a valid ZooLock. This method was
+ * pulled from org.apache.accumulo.server.master.LiveTServerSet
+ */
+ public static synchronized Set<TServerInstance> getLiveTServers(ClientContext context) {
+ final Set<TServerInstance> liveServers = new HashSet<>();
+
+ final String path = context.getZooKeeperRoot() + Constants.ZTSERVERS;
+
+ for (String child : context.getZooCache().getChildren(path)) {
+ checkServer(context, path, child).ifPresent(liveServers::add);
+ }
+ log.trace("Found {} live tservers at ZK path: {}", liveServers.size(), path);
+
+ return liveServers;
+ }
+
+ /**
+ * Check for tserver ZooLock at the ZK location. Return Optional containing TServerInstance if a
+ * valid Zoolock exists.
+ */
+ private static Optional<TServerInstance> checkServer(ClientContext context, String path,
+ String zPath) {
+ Optional<TServerInstance> server = Optional.empty();
+ final String lockPath = path + "/" + zPath;
+ ZooCache.ZcStat stat = new ZooCache.ZcStat();
+ byte[] lockData = ZooLock.getLockData(context.getZooCache(), lockPath, stat);
+
+ log.trace("Checking server at ZK path = " + lockPath);
+ if (lockData != null) {
+ ServerServices services = new ServerServices(new String(lockData, UTF_8));
+ HostAndPort client = services.getAddress(ServerServices.Service.TSERV_CLIENT);
+ server = Optional.of(new TServerInstance(client, stat.getEphemeralOwner()));
+ }
+ return server;
+ }
}
diff --git a/shell/src/main/java/org/apache/accumulo/shell/Shell.java b/shell/src/main/java/org/apache/accumulo/shell/Shell.java
index dc6327b..ccfaf03 100644
--- a/shell/src/main/java/org/apache/accumulo/shell/Shell.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/Shell.java
@@ -129,6 +129,7 @@ import org.apache.accumulo.shell.commands.ListCompactionsCommand;
import org.apache.accumulo.shell.commands.ListIterCommand;
import org.apache.accumulo.shell.commands.ListScansCommand;
import org.apache.accumulo.shell.commands.ListShellIterCommand;
+import org.apache.accumulo.shell.commands.ListTabletsCommand;
import org.apache.accumulo.shell.commands.MaxRowCommand;
import org.apache.accumulo.shell.commands.MergeCommand;
import org.apache.accumulo.shell.commands.NamespacePermissionsCommand;
@@ -378,9 +379,10 @@ public class Shell extends ShellOptions implements KeywordExecutable {
new EGrepCommand(), new FormatterCommand(), new InterpreterCommand(), new GrepCommand(),
new ImportDirectoryCommand(), new InsertCommand(), new MaxRowCommand(), new ScanCommand()};
@SuppressWarnings("deprecation")
- Command[] debuggingCommands = {new ClasspathCommand(),
- new org.apache.accumulo.shell.commands.DebugCommand(), new ListScansCommand(),
- new ListCompactionsCommand(), new TraceCommand(), new PingCommand(), new ListBulkCommand()};
+ Command[] debuggingCommands =
+ {new ClasspathCommand(), new org.apache.accumulo.shell.commands.DebugCommand(),
+ new ListScansCommand(), new ListCompactionsCommand(), new TraceCommand(),
+ new PingCommand(), new ListBulkCommand(), new ListTabletsCommand()};
Command[] execCommands =
{new ExecfileCommand(), new HistoryCommand(), new ExtensionCommand(), new ScriptCommand()};
Command[] exitCommands = {new ByeCommand(), new ExitCommand(), new QuitCommand()};
diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java
new file mode 100644
index 0000000..5226e8e
--- /dev/null
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java
@@ -0,0 +1,447 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.shell.commands;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.clientImpl.Namespaces;
+import org.apache.accumulo.core.data.NamespaceId;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.TServerInstance;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
+import org.apache.accumulo.core.util.NumUtil;
+import org.apache.accumulo.shell.Shell;
+import org.apache.accumulo.shell.Shell.Command;
+import org.apache.accumulo.shell.ShellOptions;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.io.Text;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Utility that generates single line tablet info. The output of this could be fed to sort, awk,
+ * grep, etc inorder to answer questions like which tablets have the most files.
+ */
+public class ListTabletsCommand extends Command {
+
+ private static final Logger log = LoggerFactory.getLogger(ListTabletsCommand.class);
+
+ private Option outputFileOpt;
+ private Option optTablePattern;
+ private Option optHumanReadable;
+ private Option optNamespace;
+ private Option disablePaginationOpt;
+
+ @Override
+ public int execute(String fullCommand, CommandLine cl, Shell shellState) throws Exception {
+ final Set<TableInfo> tableInfoSet = populateTables(cl, shellState);
+ if (tableInfoSet.isEmpty()) {
+ log.warn("No tables found that match your criteria");
+ return 0;
+ }
+ boolean humanReadable = cl.hasOption(optHumanReadable.getOpt());
+
+ List<String> lines = new LinkedList<>();
+ lines.add(TabletRowInfo.header);
+ for (TableInfo tableInfo : tableInfoSet) {
+ String name = tableInfo.name;
+ lines.add("TABLE: " + name);
+
+ List<TabletRowInfo> rows = getTabletRowInfo(shellState, tableInfo);
+ for (int i = 0; i < rows.size(); i++) {
+ TabletRowInfo row = rows.get(i);
+ lines.add(row.format(i + 1, humanReadable));
+ }
+ }
+
+ if (lines.size() == 1) {
+ lines.add("No data");
+ }
+
+ printResults(cl, shellState, lines);
+ return 0;
+ }
+
+ @VisibleForTesting
+ protected void printResults(CommandLine cl, Shell shellState, List<String> lines)
+ throws Exception {
+ if (cl.hasOption(outputFileOpt.getOpt())) {
+ final String outputFile = cl.getOptionValue(outputFileOpt.getOpt());
+ Shell.PrintFile printFile = new Shell.PrintFile(outputFile);
+ shellState.printLines(lines.iterator(), false, printFile);
+ printFile.close();
+ } else {
+ boolean paginate = !cl.hasOption(disablePaginationOpt.getOpt());
+ shellState.printLines(lines.iterator(), paginate);
+ }
+ }
+
+ /**
+ * Process the command line for table names using table option, table name pattern, or default to
+ * current table.
+ *
+ * @param cl
+ * command line
+ * @param shellState
+ * shell state
+ * @return set of table names.
+ * @throws NamespaceNotFoundException
+ * if the namespace option is specified and namespace does not exist
+ */
+ private Set<TableInfo> populateTables(final CommandLine cl, final Shell shellState)
+ throws NamespaceNotFoundException {
+
+ final TableOperations tableOps = shellState.getAccumuloClient().tableOperations();
+ var tableIdMap = tableOps.tableIdMap();
+
+ Set<TableInfo> tableSet = new TreeSet<>();
+
+ if (cl.hasOption(optTablePattern.getOpt())) {
+ String tablePattern = cl.getOptionValue(optTablePattern.getOpt());
+ for (String table : tableOps.list()) {
+ if (table.matches(tablePattern)) {
+ TableId id = TableId.of(tableIdMap.get(table));
+ tableSet.add(new TableInfo(table, id));
+ }
+ }
+ return tableSet;
+ }
+
+ if (cl.hasOption(optNamespace.getOpt())) {
+ String nsName = cl.getOptionValue(optNamespace.getOpt());
+ NamespaceId namespaceId = Namespaces.getNamespaceId(shellState.getContext(), nsName);
+ List<String> tables = Namespaces.getTableNames(shellState.getContext(), namespaceId);
+ tables.forEach(name -> {
+ String tableIdString = tableIdMap.get(name);
+ if (tableIdString != null) {
+ TableId id = TableId.of(tableIdString);
+ tableSet.add(new TableInfo(name, id));
+ } else {
+ log.warn("Table not found: {}", name);
+ }
+ });
+ return tableSet;
+ }
+
+ if (cl.hasOption(ShellOptions.tableOption)) {
+ String table = cl.getOptionValue(ShellOptions.tableOption);
+ String idString = tableIdMap.get(table);
+ if (idString != null) {
+ TableId id = TableId.of(idString);
+ tableSet.add(new TableInfo(table, id));
+ } else {
+ log.warn("Table not found: {}", table);
+ }
+ return tableSet;
+ }
+
+ // If we didn't get any tables, and we have a table selected, add the current table
+ String table = shellState.getTableName();
+ if (!table.isEmpty()) {
+ TableId id = TableId.of(tableIdMap.get(table));
+ tableSet.add(new TableInfo(table, id));
+ return tableSet;
+ }
+
+ return Collections.emptySet();
+ }
+
+ /**
+ * Wrapper for tablename and id. Comparisons, equals and hash code use tablename (id is ignored)
+ */
+ static class TableInfo implements Comparable<TableInfo> {
+
+ public final String name;
+ public final TableId id;
+
+ public TableInfo(final String name, final TableId id) {
+ this.name = name;
+ this.id = id;
+ }
+
+ @Override
+ public int compareTo(TableInfo other) {
+ return name.compareTo(other.name);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ TableInfo tableInfo = (TableInfo) o;
+ return name.equals(tableInfo.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name);
+ }
+ }
+
+ private List<TabletRowInfo> getTabletRowInfo(Shell shellState, TableInfo tableInfo)
+ throws Exception {
+ log.trace("scan metadata for tablet info table name: \'{}\', tableId: \'{}\' ", tableInfo.name,
+ tableInfo.id);
+
+ List<TabletRowInfo> tResults = getMetadataInfo(shellState, tableInfo);
+
+ if (log.isTraceEnabled()) {
+ for (TabletRowInfo tabletRowInfo : tResults) {
+ log.trace("Tablet info: {}", tabletRowInfo);
+ }
+ }
+
+ return tResults;
+ }
+
+ protected List<TabletRowInfo> getMetadataInfo(Shell shellState, TableInfo tableInfo)
+ throws Exception {
+ List<TabletRowInfo> results = new ArrayList<>();
+ final ClientContext context = shellState.getContext();
+ Set<TServerInstance> liveTserverSet = TabletMetadata.getLiveTServers(context);
+
+ for (var md : TabletsMetadata.builder().forTable(tableInfo.id).build(context)) {
+ TabletRowInfo.Factory factory = new TabletRowInfo.Factory(tableInfo.name, md.getExtent());
+ var fileMap = md.getFilesMap();
+ factory.numFiles(fileMap.size());
+ long entries = 0L;
+ long size = 0L;
+ for (DataFileValue dfv : fileMap.values()) {
+ entries += dfv.getNumEntries();
+ size += dfv.getSize();
+ }
+ factory.numEntries(entries);
+ factory.size(size);
+ factory.numWalLogs(md.getLogs().size());
+ factory.dir(md.getDirName());
+ factory.location(md.getLocation());
+ factory.status(md.getTabletState(liveTserverSet).toString());
+ results.add(factory.build());
+ }
+
+ return results;
+ }
+
+ @Override
+ public String description() {
+ return "Prints info about every tablet for a table, one tablet per line.";
+ }
+
+ @Override
+ public int numArgs() {
+ return 0;
+ }
+
+ @Override
+ public Options getOptions() {
+
+ final Options opts = new Options();
+ opts.addOption(OptUtil.tableOpt("table to be scanned"));
+
+ optTablePattern = new Option("p", "pattern", true, "regex pattern of table names");
+ optTablePattern.setArgName("pattern");
+ opts.addOption(optTablePattern);
+
+ optNamespace =
+ new Option(ShellOptions.namespaceOption, "namespace", true, "name of a namespace");
+ optNamespace.setArgName("namespace");
+ opts.addOption(optNamespace);
+
+ optHumanReadable =
+ new Option("h", "human-readable", false, "format large sizes to human readable units");
+ optHumanReadable.setArgName("human readable output");
+ opts.addOption(optHumanReadable);
+
+ disablePaginationOpt =
+ new Option("np", "no-pagination", false, "disables pagination of output");
+ opts.addOption(disablePaginationOpt);
+
+ outputFileOpt = new Option("o", "output", true, "local file to write output to");
+ outputFileOpt.setArgName("file");
+ opts.addOption(outputFileOpt);
+
+ return opts;
+ }
+
+ static class TabletRowInfo {
+
+ public final String tableName;
+ public final int numFiles;
+ public final int numWalLogs;
+ public final long numEntries;
+ public final long size;
+ public final String status;
+ public final String location;
+ public final String dir;
+ public final TableId tableId;
+ public final KeyExtent tablet;
+ public final boolean tableExists;
+
+ private TabletRowInfo(String tableName, KeyExtent tablet, int numFiles, int numWalLogs,
+ long numEntries, long size, String status, String location, String dir,
+ boolean tableExists) {
+ this.tableName = tableName;
+ this.tableId = tablet.tableId();
+ this.tablet = tablet;
+ this.numFiles = numFiles;
+ this.numWalLogs = numWalLogs;
+ this.numEntries = numEntries;
+ this.size = size;
+ this.status = status;
+ this.location = location;
+ this.dir = dir;
+ this.tableExists = tableExists;
+ }
+
+ String getNumEntries(final boolean humanReadable) {
+ if (humanReadable) {
+ return String.format("%9s", NumUtil.bigNumberForQuantity(numEntries));
+ }
+ // return String.format("%,24d", numEntries);
+ return Long.toString(numEntries);
+ }
+
+ String getSize(final boolean humanReadable) {
+ if (humanReadable) {
+ return String.format("%9s", NumUtil.bigNumberForSize(size));
+ }
+ // return String.format("%,24d", size);
+ return Long.toString(size);
+ }
+
+ public String getEndRow() {
+ Text t = tablet.endRow();
+ if (t == null)
+ return "+INF";
+ else
+ return t.toString();
+ }
+
+ public String getStartRow() {
+ Text t = tablet.prevEndRow();
+ if (t == null)
+ return "-INF";
+ else
+ return t.toString();
+ }
+
+ public static final String header = String.format(
+ "%-4s %-15s %-5s %-5s %-9s %-9s %-10s %-30s %-5s %-20s %-20s", "NUM", "TABLET_DIR", "FILES",
+ "WALS", "ENTRIES", "SIZE", "STATUS", "LOCATION", "ID", "START (Exclusive)", "END");
+
+ String format(int number, boolean prettyPrint) {
+ return String.format("%-4d %-15s %-5d %-5s %-9s %-9s %-10s %-30s %-5s %-20s %-20s", number,
+ dir, numFiles, numWalLogs, getNumEntries(prettyPrint), getSize(prettyPrint), status,
+ location, tableId, getStartRow(), getEndRow());
+ }
+
+ public String getTablet() {
+ return getStartRow() + " " + getEndRow();
+ }
+
+ public static class Factory {
+ final String tableName;
+ final KeyExtent tablet;
+ final TableId tableId;
+ int numFiles = 0;
+ int numWalLogs = 0;
+ long numEntries = 0;
+ long size = 0;
+ String status = "";
+ String location = "";
+ String dir = "";
+ boolean tableExists = false;
+
+ Factory(final String tableName, KeyExtent tablet) {
+ this.tableName = tableName;
+ this.tablet = tablet;
+ this.tableId = tablet.tableId();
+ }
+
+ Factory numFiles(int numFiles) {
+ this.numFiles = numFiles;
+ return this;
+ }
+
+ Factory numWalLogs(int numWalLogs) {
+ this.numWalLogs = numWalLogs;
+ return this;
+ }
+
+ public Factory numEntries(long numEntries) {
+ this.numEntries = numEntries;
+ return this;
+ }
+
+ public Factory size(long size) {
+ this.size = size;
+ return this;
+ }
+
+ public Factory status(String status) {
+ this.status = status;
+ return this;
+ }
+
+ public Factory location(TabletMetadata.Location location) {
+ if (location == null) {
+ this.location = "None";
+ } else {
+ String server = location.getHostPort();
+ this.location = location.getType() + ":" + server;
+ }
+ return this;
+ }
+
+ public Factory dir(String dirName) {
+ this.dir = dirName;
+ return this;
+ }
+
+ public Factory tableExists(boolean tableExists) {
+ this.tableExists = tableExists;
+ return this;
+ }
+
+ public TabletRowInfo build() {
+ return new TabletRowInfo(tableName, tablet, numFiles, numWalLogs, numEntries, size, status,
+ location, dir, tableExists);
+ }
+ }
+ }
+
+}
diff --git a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java
new file mode 100644
index 0000000..b9a8c16
--- /dev/null
+++ b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.shell.commands;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.admin.InstanceOperations;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.TabletState;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.shell.Shell;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.io.Text;
+import org.easymock.EasyMock;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ListTabletsCommandTest {
+
+ private static final Logger log = LoggerFactory.getLogger(ListTabletsCommandTest.class);
+ final String tableName = ListTabletsCommandTest.class.getName() + "-aTable";
+ private static final TableId tableId = TableId.of("123");
+ private static final String rowString = "123;a 123;m 123<";
+ private static final List<String> rows = new ArrayList<>(Arrays.asList(rowString.split(" ")));
+
+ private class TestListTabletsCommand extends ListTabletsCommand {
+ @Override
+ protected void printResults(CommandLine cl, Shell shellState, List<String> lines) {
+ log.debug("Command run successfully. Output below...");
+ for (String line : lines)
+ log.debug(line);
+ assertEquals(TabletRowInfo.header, lines.get(0));
+ assertTrue(lines.get(1).startsWith("TABLE:"));
+ // first table info
+ assertTrue(lines.get(2).startsWith("1"));
+ assertTrue(lines.get(2).contains("t-dir1"));
+ assertTrue(lines.get(2).contains("100"));
+ assertTrue(lines.get(2).contains("-INF"));
+ assertTrue(lines.get(2).endsWith("a "));
+ // second tablet info
+ assertTrue(lines.get(3).startsWith("2"));
+ assertTrue(lines.get(3).contains("t-dir2"));
+ assertTrue(lines.get(3).contains("200"));
+ assertTrue(lines.get(3).contains("a"));
+ assertTrue(lines.get(3).endsWith("m "));
+ // third tablet info
+ assertTrue(lines.get(4).startsWith("3"));
+ assertTrue(lines.get(4).contains("t-dir3"));
+ assertTrue(lines.get(4).contains("300"));
+ assertTrue(lines.get(4).contains("m"));
+ assertTrue(lines.get(4).endsWith("+INF "));
+ }
+
+ @Override
+ protected List<TabletRowInfo> getMetadataInfo(Shell shellState,
+ ListTabletsCommand.TableInfo tableInfo) throws Exception {
+ List<TabletRowInfo> tablets = new ArrayList<>();
+ KeyExtent ke1 = new KeyExtent(tableId, new Text("a"), null);
+ KeyExtent ke2 = new KeyExtent(tableId, new Text("m"), new Text("a"));
+ KeyExtent ke3 = new KeyExtent(tableId, null, new Text("m"));
+ TabletMetadata.Location loc =
+ new TabletMetadata.Location("localhost", "", TabletMetadata.LocationType.CURRENT);
+ ListTabletsCommand.TabletRowInfo.Factory factory =
+ new ListTabletsCommand.TabletRowInfo.Factory(tableName, ke1).dir("t-dir1").numFiles(1)
+ .numWalLogs(1).numEntries(1).size(100).status(TabletState.HOSTED.toString())
+ .location(loc).tableExists(true);
+ tablets.add(factory.build());
+ factory = new ListTabletsCommand.TabletRowInfo.Factory(tableName, ke2).dir("t-dir2")
+ .numFiles(2).numWalLogs(2).numEntries(2).size(200).status(TabletState.HOSTED.toString())
+ .location(loc).tableExists(true);
+ tablets.add(factory.build());
+ factory = new ListTabletsCommand.TabletRowInfo.Factory(tableName, ke3).dir("t-dir3")
+ .numFiles(3).numWalLogs(3).numEntries(3).size(300).status(TabletState.HOSTED.toString())
+ .location(loc).tableExists(true);
+ tablets.add(factory.build());
+ return tablets;
+ }
+ }
+
+ @Test
+ public void mockTest() throws Exception {
+ ListTabletsCommand cmd = new TestListTabletsCommand();
+
+ AccumuloClient client = EasyMock.createMock(AccumuloClient.class);
+ ClientContext context = EasyMock.createMock(ClientContext.class);
+ TableOperations tableOps = EasyMock.createMock(TableOperations.class);
+ InstanceOperations instOps = EasyMock.createMock(InstanceOperations.class);
+ Shell shellState = EasyMock.createMock(Shell.class);
+
+ Options opts = cmd.getOptions();
+
+ CommandLineParser parser = new DefaultParser();
+ String[] args = {"-t", tableName};
+ CommandLine cli = parser.parse(opts, args);
+
+ EasyMock.expect(shellState.getAccumuloClient()).andReturn(client).anyTimes();
+ EasyMock.expect(shellState.getContext()).andReturn(context).anyTimes();
+ EasyMock.expect(client.tableOperations()).andReturn(tableOps).anyTimes();
+
+ Map<String,String> idMap = new TreeMap<>();
+ idMap.put(tableName, tableId.canonical());
+ EasyMock.expect(tableOps.tableIdMap()).andReturn(idMap);
+
+ assertEquals("Incorrect number of rows: " + rows, rows.size(), 3);
+
+ EasyMock.replay(client, context, tableOps, instOps, shellState);
+ cmd.execute("listTablets -t " + tableName, cli, shellState);
+ EasyMock.verify(client, context, tableOps, instOps, shellState);
+ }
+
+ @Test
+ public void defaultBuilderTest() {
+ TableId id = TableId.of("123");
+ Text startRow = new Text("a");
+ Text endRow = new Text("z");
+ ListTabletsCommand.TabletRowInfo.Factory factory =
+ new ListTabletsCommand.TabletRowInfo.Factory("aName", new KeyExtent(id, endRow, startRow));
+
+ ListTabletsCommand.TabletRowInfo info = factory.build();
+
+ assertEquals("aName", info.tableName);
+ assertEquals(id, info.tableId);
+ assertEquals("a z", info.getTablet());
+ assertEquals(0, info.numFiles);
+ assertEquals(0, info.numWalLogs);
+ assertEquals(0, info.numEntries);
+ assertEquals(0, info.size);
+ assertEquals("", info.status);
+ assertEquals("", info.location);
+ assertFalse(info.tableExists);
+ }
+
+ @Test
+ public void builderTest() {
+ TableId id = TableId.of("123");
+ Text startRow = new Text("a");
+ Text endRow = new Text("z");
+ KeyExtent ke = new KeyExtent(id, endRow, startRow);
+ TabletMetadata.Location loc =
+ new TabletMetadata.Location("localhost", "", TabletMetadata.LocationType.CURRENT);
+ ListTabletsCommand.TabletRowInfo.Factory factory =
+ new ListTabletsCommand.TabletRowInfo.Factory("aName", ke).numFiles(1).numWalLogs(2)
+ .numEntries(3).size(4).status(TabletState.HOSTED.toString()).location(loc)
+ .tableExists(true);
+
+ ListTabletsCommand.TabletRowInfo info = factory.build();
+
+ assertEquals("aName", info.tableName);
+ assertEquals(1, info.numFiles);
+ assertEquals(2, info.numWalLogs);
+ assertEquals("3", info.getNumEntries(false));
+ assertEquals(3, info.numEntries);
+ assertEquals("4", info.getSize(false));
+ assertEquals(4, info.size);
+ assertEquals("HOSTED", info.status);
+ assertEquals("CURRENT:localhost", info.location);
+ assertEquals(TableId.of("123"), info.tableId);
+ assertEquals(startRow + " " + endRow, info.getTablet());
+ assertTrue(info.tableExists);
+ }
+}
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/TabletMetadataIT.java b/test/src/main/java/org/apache/accumulo/test/functional/TabletMetadataIT.java
new file mode 100644
index 0000000..479492f
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/functional/TabletMetadataIT.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
+import static org.apache.accumulo.minicluster.ServerType.TABLET_SERVER;
+import static org.junit.Assert.assertEquals;
+
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.metadata.TServerInstance;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Tests features of the Ample TabletMetadata class that can't be tested in TabletMetadataTest
+ */
+public class TabletMetadataIT extends ConfigurableMacBase {
+ private static final Logger log = LoggerFactory.getLogger(TabletMetadataIT.class);
+ private static final int NUM_TSERVERS = 3;
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 120;
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration conf) {
+ cfg.setNumTservers(NUM_TSERVERS);
+ }
+
+ @Test
+ public void getLiveTServersTest() throws Exception {
+ try (AccumuloClient c = Accumulo.newClient().from(getClientProperties()).build()) {
+ while (c.instanceOperations().getTabletServers().size() != NUM_TSERVERS) {
+ log.info("Waiting for tservers to start up...");
+ sleepUninterruptibly(5, TimeUnit.SECONDS);
+ }
+ Set<TServerInstance> servers = TabletMetadata.getLiveTServers((ClientContext) c);
+ assertEquals(NUM_TSERVERS, servers.size());
+
+ // kill a tserver and see if its gone from the list
+ getCluster().killProcess(TABLET_SERVER,
+ getCluster().getProcesses().get(TABLET_SERVER).iterator().next());
+
+ while (c.instanceOperations().getTabletServers().size() == NUM_TSERVERS) {
+ log.info("Waiting for a tserver to die...");
+ sleepUninterruptibly(5, TimeUnit.SECONDS);
+ }
+ servers = TabletMetadata.getLiveTServers((ClientContext) c);
+ assertEquals(NUM_TSERVERS - 1, servers.size());
+ }
+ }
+}