You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ed...@apache.org on 2022/07/20 12:51:55 UTC
[accumulo] branch main updated: Add fate summary command option (#2801)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 2ac8c7f97f Add fate summary command option (#2801)
2ac8c7f97f is described below
commit 2ac8c7f97f1935db37c35593d7d8ec50790fad29
Author: EdColeman <de...@etcoleman.com>
AuthorDate: Wed Jul 20 12:51:50 2022 +0000
Add fate summary command option (#2801)
Provide additional information similar to fate print by adding a fate summary option.
---
.../accumulo/shell/commands/FateCommand.java | 87 ++++++++++--
.../commands/fateCommand/FateSummaryReport.java | 151 +++++++++++++++++++++
.../shell/commands/fateCommand/FateTxnDetails.java | 140 +++++++++++++++++++
.../accumulo/shell/commands/FateCommandTest.java | 51 +++++++
.../commands/fateCommand/SummaryReportTest.java | 96 +++++++++++++
.../shell/commands/fateCommand/TxnDetailsTest.java | 113 +++++++++++++++
.../apache/accumulo/test/shell/ShellServerIT.java | 100 +++++++++++++-
7 files changed, 726 insertions(+), 12 deletions(-)
diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/FateCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/FateCommand.java
index 9a83dfadfd..2736205db6 100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/FateCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/FateCommand.java
@@ -23,12 +23,15 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.IOException;
import java.lang.reflect.Type;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Formatter;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.accumulo.core.Constants;
@@ -37,10 +40,12 @@ import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.manager.thrift.FateService;
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
import org.apache.accumulo.core.trace.TraceUtil;
+import org.apache.accumulo.core.util.tables.TableMap;
import org.apache.accumulo.fate.AdminUtil;
import org.apache.accumulo.fate.FateTxId;
import org.apache.accumulo.fate.ReadOnlyRepo;
@@ -52,12 +57,15 @@ import org.apache.accumulo.fate.zookeeper.ServiceLock.ServiceLockPath;
import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.shell.Shell;
import org.apache.accumulo.shell.Shell.Command;
+import org.apache.accumulo.shell.commands.fateCommand.FateSummaryReport;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
@@ -71,6 +79,8 @@ import com.google.gson.JsonSerializer;
*/
public class FateCommand extends Command {
+ private final static Logger LOG = LoggerFactory.getLogger(FateCommand.class);
+
// this class serializes references to interfaces with the concrete class name
private static class InterfaceSerializer<T> implements JsonSerializer<T> {
@Override
@@ -118,6 +128,7 @@ public class FateCommand extends Command {
private Option fail;
private Option list;
private Option print;
+ private Option summary;
private Option secretOption;
private Option statusOption;
private Option disablePaginationOpt;
@@ -178,6 +189,9 @@ public class FateCommand extends Command {
printTx(shellState, admin, zs, zk, tableLocksPath, cl.getOptionValues(list.getOpt()), cl);
} else if (cl.hasOption(print.getOpt())) {
printTx(shellState, admin, zs, zk, tableLocksPath, cl.getOptionValues(print.getOpt()), cl);
+ } else if (cl.hasOption(summary.getOpt())) {
+ summarizeTx(shellState, admin, zs, zk, tableLocksPath, cl.getOptionValues(summary.getOpt()),
+ cl);
} else if (cl.hasOption(dump.getOpt())) {
String output = dumpTx(zs, cl.getOptionValues(dump.getOpt()));
System.out.println(output);
@@ -228,22 +242,51 @@ public class FateCommand extends Command {
}
// Parse TStatus filters for print display
- EnumSet<TStatus> filterStatus = null;
- if (cl.hasOption(statusOption.getOpt())) {
- filterStatus = EnumSet.noneOf(TStatus.class);
- String[] tstat = cl.getOptionValues(statusOption.getOpt());
- for (String element : tstat) {
- filterStatus.add(TStatus.valueOf(element));
- }
- }
+ EnumSet<TStatus> statusFilter = getCmdLineStatusFilters(cl);
StringBuilder buf = new StringBuilder(8096);
Formatter fmt = new Formatter(buf);
- admin.print(zs, zk, tableLocksPath, fmt, filterTxid, filterStatus);
+ admin.print(zs, zk, tableLocksPath, fmt, filterTxid, statusFilter);
shellState.printLines(Collections.singletonList(buf.toString()).iterator(),
!cl.hasOption(disablePaginationOpt.getOpt()));
}
+ protected void summarizeTx(Shell shellState, AdminUtil<FateCommand> admin,
+ ZooStore<FateCommand> zs, ZooReaderWriter zk, ServiceLockPath tableLocksPath, String[] args,
+ CommandLine cl) throws InterruptedException, AccumuloException, AccumuloSecurityException,
+ KeeperException, IOException {
+
+ var transactions = admin.getStatus(zs, zk, tableLocksPath, null, null);
+
+ // build id map - relies on unique ids for tables and namespaces
+ // used to look up the names of either table or namespace by id.
+ Map<TableId,String> tidToNameMap = new TableMap(shellState.getContext()).getIdtoNameMap();
+ Map<String,String> idsToNameMap = new HashMap<>(tidToNameMap.size() * 2);
+ tidToNameMap.forEach((tid, name) -> idsToNameMap.put(tid.canonical(), "t:" + name));
+ shellState.getContext().namespaceOperations().namespaceIdMap().forEach((name, nsid) -> {
+ String prev = idsToNameMap.put(nsid, "ns:" + name);
+ if (prev != null) {
+ LOG.warn("duplicate id found for table / namespace id. table name: {}, namespace name: {}",
+ prev, name);
+ }
+ });
+
+ EnumSet<TStatus> statusFilter = getCmdLineStatusFilters(cl);
+
+ FateSummaryReport report = new FateSummaryReport(idsToNameMap, statusFilter);
+
+ // gather statistics
+ transactions.getTransactions().forEach(report::gatherTxnStatus);
+ if (Arrays.asList(cl.getArgs()).contains("json")) {
+ shellState.printLines(Collections.singletonList(report.toJson()).iterator(),
+ !cl.hasOption(disablePaginationOpt.getOpt()));
+ } else {
+ // print the formatted report by lines to allow pagination
+ shellState.printLines(report.formatLines().iterator(),
+ !cl.hasOption(disablePaginationOpt.getOpt()));
+ }
+ }
+
protected boolean deleteTx(AdminUtil<FateCommand> admin, ZooStore<FateCommand> zs,
ZooReaderWriter zk, ServiceLockPath zLockManagerPath, String[] args)
throws InterruptedException, KeeperException {
@@ -357,6 +400,11 @@ public class FateCommand extends Command {
print.setArgs(Option.UNLIMITED_VALUES);
print.setOptionalArg(true);
+ summary =
+ new Option("summary", "summary", true, "print a summary of FaTE transaction information");
+ summary.setArgName("--json");
+ summary.setOptionalArg(true);
+
dump = new Option("dump", "dump", true, "dump FaTE transaction information details");
dump.setArgName("txid");
dump.setArgs(Option.UNLIMITED_VALUES);
@@ -367,6 +415,7 @@ public class FateCommand extends Command {
commands.addOption(delete);
commands.addOption(list);
commands.addOption(print);
+ commands.addOption(summary);
commands.addOption(dump);
o.addOptionGroup(commands);
@@ -390,4 +439,24 @@ public class FateCommand extends Command {
// Arg length varies between 1 to n
return -1;
}
+
+ /**
+ * If provided on the command line, get the TStatus values provided.
+ *
+ * @param cl
+ * the command line
+ * @return a set of status filters, or an empty set if none provides
+ */
+ private EnumSet<TStatus> getCmdLineStatusFilters(CommandLine cl) {
+ EnumSet<TStatus> statusFilter = null;
+ if (cl.hasOption(statusOption.getOpt())) {
+ statusFilter = EnumSet.noneOf(TStatus.class);
+ String[] tstat = cl.getOptionValues(statusOption.getOpt());
+ for (String element : tstat) {
+ statusFilter.add(TStatus.valueOf(element));
+ }
+ }
+ return statusFilter;
+ }
+
}
diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/fateCommand/FateSummaryReport.java b/shell/src/main/java/org/apache/accumulo/shell/commands/fateCommand/FateSummaryReport.java
new file mode 100644
index 0000000000..2a76c08ba1
--- /dev/null
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/fateCommand/FateSummaryReport.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.shell.commands.fateCommand;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.accumulo.fate.AdminUtil;
+import org.apache.accumulo.fate.ReadOnlyTStore;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+public class FateSummaryReport {
+
+ private final Map<String,Integer> statusCounts = new TreeMap<>();
+ private final Map<String,Integer> cmdCounts = new TreeMap<>();
+ private final Map<String,Integer> stepCounts = new TreeMap<>();
+ private final Set<FateTxnDetails> fateDetails = new TreeSet<>();
+ // epoch millis to avoid needing gson type adapter.
+ private final long reportTime = Instant.now().toEpochMilli();
+
+ private final Set<String> statusFilterNames = new TreeSet<>();
+
+ private final static Gson gson = new GsonBuilder().setPrettyPrinting().create();
+
+ // exclude from json output
+ private final transient Map<String,String> idsToNameMap;
+
+ public FateSummaryReport(Map<String,String> idsToNameMap,
+ EnumSet<ReadOnlyTStore.TStatus> statusFilter) {
+ this.idsToNameMap = idsToNameMap;
+ if (statusFilter != null) {
+ statusFilter.forEach(f -> this.statusFilterNames.add(f.name()));
+ }
+ }
+
+ public void gatherTxnStatus(AdminUtil.TransactionStatus txnStatus) {
+ var status = txnStatus.getStatus();
+ if (status == null) {
+ statusCounts.merge("?", 1, Integer::sum);
+ } else {
+ String name = txnStatus.getStatus().name();
+ statusCounts.merge(name, 1, Integer::sum);
+ }
+ String top = txnStatus.getTop();
+ stepCounts.merge(Objects.requireNonNullElse(top, "?"), 1, Integer::sum);
+ String debug = txnStatus.getDebug();
+ cmdCounts.merge(Objects.requireNonNullElse(debug, "?"), 1, Integer::sum);
+
+ // filter status if provided.
+ if (!statusFilterNames.isEmpty() && !statusFilterNames.contains(txnStatus.getStatus().name())) {
+ return;
+ }
+ fateDetails.add(new FateTxnDetails(reportTime, txnStatus, idsToNameMap));
+ }
+
+ public Map<String,Integer> getStatusCounts() {
+ return statusCounts;
+ }
+
+ public Map<String,Integer> getCmdCounts() {
+ return cmdCounts;
+ }
+
+ public Map<String,Integer> getStepCounts() {
+ return stepCounts;
+ }
+
+ public Set<FateTxnDetails> getFateDetails() {
+ return fateDetails;
+ }
+
+ public long getReportTime() {
+ return reportTime;
+ }
+
+ public Set<String> getStatusFilterNames() {
+ return statusFilterNames;
+ }
+
+ public String toJson() {
+ return gson.toJson(this);
+ }
+
+ public static FateSummaryReport fromJson(final String jsonString) {
+ return gson.fromJson(jsonString, FateSummaryReport.class);
+ }
+
+ /**
+ * Generate a summary report in a format suitable for pagination in fate commands that expects a
+ * list of lines.
+ *
+ * @return formatted report lines.
+ */
+ public List<String> formatLines() {
+ List<String> lines = new ArrayList<>();
+
+ final DateTimeFormatter fmt =
+ DateTimeFormatter.ISO_INSTANT.withZone(ZoneId.from(ZoneOffset.UTC));
+
+ // output report
+ lines.add(String.format("Report Time: %s",
+ fmt.format(Instant.ofEpochMilli(reportTime).truncatedTo(ChronoUnit.SECONDS))));
+
+ lines.add("Status counts:\n\n");
+ statusCounts.forEach((status, count) -> lines.add(String.format(" %s: %d\n", status, count)));
+
+ lines.add("\nCommand counts:\n\n");
+ cmdCounts.forEach((cmd, count) -> lines.add(String.format(" %s: %d\n", cmd, count)));
+
+ lines.add("\nStep counts:\n\n");
+ stepCounts.forEach((step, count) -> lines.add(String.format(" %s: %d\n", step, count)));
+
+ lines.add("\nFate transactions (oldest first):\n\n");
+ lines.add("Status Filters: "
+ + (statusFilterNames.isEmpty() ? "[NONE]" : statusFilterNames.toString()));
+
+ lines.add("\n" + FateTxnDetails.TXN_HEADER);
+ fateDetails.forEach(txnDetails -> lines.add(txnDetails.toString()));
+
+ return lines;
+ }
+}
diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/fateCommand/FateTxnDetails.java b/shell/src/main/java/org/apache/accumulo/shell/commands/fateCommand/FateTxnDetails.java
new file mode 100644
index 0000000000..fd20b9946a
--- /dev/null
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/fateCommand/FateTxnDetails.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.shell.commands.fateCommand;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.accumulo.fate.AdminUtil;
+
+public class FateTxnDetails implements Comparable<FateTxnDetails> {
+ final static String TXN_HEADER =
+ "Running\ttxn_id\t\t\t\tStatus\t\tCommand\t\tStep (top)\t\tlocks held:(table id, name)\tlocks waiting:(table id, name)\n";
+
+ private long running;
+ private String status = "?";
+ private String command = "?";
+ private String step = "?";
+ private String txnId = "?";
+ private List<String> locksHeld = List.of();
+ private List<String> locksWaiting = List.of();
+
+ /**
+ * Create a detailed FaTE transaction that can be formatted for status reports.
+ * <p>
+ * Implementation note: Instance of this class are expected to be used for status reporting that
+ * represent a snapshot at the time of measurement. This class is conservative in handling
+ * possible vales - gathering FaTE information is done asynchronously and when the measurement is
+ * captured it may not be complete.
+ *
+ * @param reportTime
+ * the Instant that the report snapshot was created
+ * @param txnStatus
+ * the FaTE transaction status
+ * @param idsToNameMap
+ * a map of namespace, table ids to names.
+ */
+ public FateTxnDetails(final long reportTime, final AdminUtil.TransactionStatus txnStatus,
+ final Map<String,String> idsToNameMap) {
+
+ // guard against invalid transaction
+ if (txnStatus == null) {
+ return;
+ }
+
+ // guard against partial / invalid info
+ if (txnStatus.getTimeCreated() != 0) {
+ running = reportTime - txnStatus.getTimeCreated();
+ }
+ if (txnStatus.getStatus() != null) {
+ status = txnStatus.getStatus().name();
+ }
+ if (txnStatus.getTop() != null) {
+ step = txnStatus.getTop();
+ }
+ if (txnStatus.getDebug() != null) {
+ command = txnStatus.getDebug();
+ }
+ if (txnStatus.getTxid() != null) {
+ txnId = txnStatus.getTxid();
+ }
+ locksHeld = formatLockInfo(txnStatus.getHeldLocks(), idsToNameMap);
+ locksWaiting = formatLockInfo(txnStatus.getWaitingLocks(), idsToNameMap);
+ }
+
+ private List<String> formatLockInfo(final List<String> lockInfo,
+ final Map<String,String> idsToNameMap) {
+ List<String> formattedLocks = new ArrayList<>();
+ for (String lock : lockInfo) {
+ String[] parts = lock.split(":");
+ if (parts.length == 2) {
+ String lockType = parts[0];
+ String id = parts[1];
+ formattedLocks.add(String.format("%s:(%s,%s)", lockType, id, idsToNameMap.get(id)));
+ }
+ }
+ return formattedLocks;
+ }
+
+ /**
+ * Sort by running time in reverse (oldest txn first). txid is unique as used to break times and
+ * so that compareTo remains consistent with hashCode and equals methods.
+ *
+ * @param other
+ * the FateTxnDetails to be compared.
+ * @return -1, 0 or 1 if older, equal or newer than the other
+ */
+ @Override
+ public int compareTo(FateTxnDetails other) {
+ int v = Long.compare(other.running, this.running);
+ if (v != 0)
+ return v;
+ return txnId.compareTo(other.txnId);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ FateTxnDetails that = (FateTxnDetails) o;
+ return running == that.running && txnId.equals(that.txnId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(running, txnId);
+ }
+
+ @Override
+ public String toString() {
+ Duration elapsed = Duration.ofMillis(running);
+ String hms = String.format("%d:%02d:%02d", elapsed.toHours(), elapsed.toMinutesPart(),
+ elapsed.toSecondsPart());
+
+ return hms + "\t" + txnId + "\t" + status + "\t" + command + "\t" + step + "\theld:"
+ + locksHeld.toString() + "\twaiting:" + locksWaiting.toString() + "\n";
+ }
+
+}
diff --git a/shell/src/test/java/org/apache/accumulo/shell/commands/FateCommandTest.java b/shell/src/test/java/org/apache/accumulo/shell/commands/FateCommandTest.java
index 5f526c6939..ef73b028ea 100644
--- a/shell/src/test/java/org/apache/accumulo/shell/commands/FateCommandTest.java
+++ b/shell/src/test/java/org/apache/accumulo/shell/commands/FateCommandTest.java
@@ -23,6 +23,7 @@ import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
import static org.easymock.EasyMock.verify;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -66,6 +67,7 @@ public class FateCommandTest {
private boolean failCalled = false;
private boolean cancelCalled = false;
private boolean printCalled = false;
+ private boolean summarizeCalled = false;
@Override
public String getName() {
@@ -123,12 +125,20 @@ public class FateCommandTest {
printCalled = true;
}
+ @Override
+ protected void summarizeTx(Shell shellState, AdminUtil<FateCommand> admin,
+ ZooStore<FateCommand> zs, ZooReaderWriter zk, ServiceLockPath tableLocksPath, String[] args,
+ CommandLine cl) {
+ summarizeCalled = true;
+ }
+
public void reset() {
dumpCalled = false;
deleteCalled = false;
failCalled = false;
cancelCalled = false;
printCalled = false;
+ summarizeCalled = false;
}
}
@@ -192,6 +202,7 @@ public class FateCommandTest {
@Test
public void testPrintAndList() throws IOException, InterruptedException, KeeperException {
+ reset(zk);
PrintStream out = System.out;
File config = Files.createTempFile(null, null).toFile();
TestOutputStream output = new TestOutputStream();
@@ -236,6 +247,42 @@ public class FateCommandTest {
verify(zs, zk);
}
+ @Test
+ public void testSummary() throws IOException, InterruptedException, AccumuloException,
+ AccumuloSecurityException, KeeperException {
+ reset(zk);
+ PrintStream out = System.out;
+ File config = Files.createTempFile(null, null).toFile();
+ TestOutputStream output = new TestOutputStream();
+ Shell shell = createShell(output);
+
+ ServiceLockPath tableLocksPath = ServiceLock.path("/accumulo" + ZTABLE_LOCKS);
+ ZooStore<FateCommand> zs = createMock(ZooStore.class);
+ expect(zk.getChildren(tableLocksPath.toString())).andReturn(List.of("5")).anyTimes();
+ expect(zk.getChildren("/accumulo/table_locks/5")).andReturn(List.of()).anyTimes();
+ expect(zs.list()).andReturn(List.of()).anyTimes();
+
+ replay(zs, zk);
+
+ TestHelper helper = new TestHelper(true);
+ FateCommand cmd = new TestFateCommand();
+ var options = cmd.getOptions();
+ CommandLine cli = new CommandLine.Builder().addOption(options.getOption("summary"))
+ .addOption(options.getOption("np")).build();
+
+ try {
+ cmd.summarizeTx(shell, helper, zs, zk, tableLocksPath, cli.getOptionValues("list"), cli);
+ } finally {
+ output.clear();
+ System.setOut(out);
+ if (config.exists()) {
+ assertTrue(config.delete());
+ }
+ }
+
+ verify(zs, zk);
+ }
+
@Test
public void testCommandLineOptions() throws Exception {
PrintStream out = System.out;
@@ -324,6 +371,10 @@ public class FateCommandTest {
cmd.reset();
shell.execCommand("fate --list 12345 67890", true, false);
assertTrue(cmd.printCalled);
+ shell.execCommand("fate -summary", true, false);
+ assertTrue(cmd.summarizeCalled);
+ shell.execCommand("fate --summary", true, false);
+ assertTrue(cmd.summarizeCalled);
cmd.reset();
} finally {
shell.shutdown();
diff --git a/shell/src/test/java/org/apache/accumulo/shell/commands/fateCommand/SummaryReportTest.java b/shell/src/test/java/org/apache/accumulo/shell/commands/fateCommand/SummaryReportTest.java
new file mode 100644
index 0000000000..7fcbfacfa1
--- /dev/null
+++ b/shell/src/test/java/org/apache/accumulo/shell/commands/fateCommand/SummaryReportTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.shell.commands.fateCommand;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.fate.AdminUtil;
+import org.apache.accumulo.fate.ReadOnlyTStore;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class SummaryReportTest {
+
+ private static final Logger log = LoggerFactory.getLogger(SummaryReportTest.class);
+
+ @Test
+ public void blankReport() {
+ Map<String,String> idMap = Map.of("1", "ns1", "2", "tbl1");
+ FateSummaryReport report = new FateSummaryReport(idMap, null);
+ assertNotNull(report);
+ assertTrue(report.getReportTime() != 0);
+ assertEquals(Map.of(), report.getStatusCounts());
+ assertEquals(Map.of(), report.getCmdCounts());
+ assertEquals(Map.of(), report.getStepCounts());
+ assertEquals(Set.of(), report.getFateDetails());
+ assertEquals(Set.of(), report.getStatusFilterNames());
+ assertNotNull(report.toJson());
+ assertNotNull(report.formatLines());
+ log.info("json: {}", report.toJson());
+ log.info("formatted: {}", report.formatLines());
+ }
+
+ @Test
+ public void noTablenameReport() {
+
+ long now = System.currentTimeMillis();
+
+ AdminUtil.TransactionStatus status1 = createMock(AdminUtil.TransactionStatus.class);
+ expect(status1.getTimeCreated()).andReturn(now - TimeUnit.DAYS.toMillis(1)).anyTimes();
+ expect(status1.getStatus()).andReturn(ReadOnlyTStore.TStatus.IN_PROGRESS).anyTimes();
+ expect(status1.getTop()).andReturn(null).anyTimes();
+ expect(status1.getDebug()).andReturn(null).anyTimes();
+ expect(status1.getTxid()).andReturn("abcdabcd").anyTimes();
+ expect(status1.getHeldLocks()).andReturn(List.of()).anyTimes();
+ expect(status1.getWaitingLocks()).andReturn(List.of()).anyTimes();
+
+ replay(status1);
+ Map<String,String> idMap = Map.of("1", "ns1", "2", "");
+ FateSummaryReport report = new FateSummaryReport(idMap, null);
+ report.gatherTxnStatus(status1);
+
+ assertNotNull(report);
+ assertTrue(report.getReportTime() != 0);
+ assertEquals(Map.of("IN_PROGRESS", 1), report.getStatusCounts());
+ assertEquals(Map.of("?", 1), report.getCmdCounts());
+ assertEquals(Map.of("?", 1), report.getStepCounts());
+ assertEquals(Set.of(), report.getStatusFilterNames());
+ assertNotNull(report.toJson());
+ assertNotNull(report.formatLines());
+
+ assertNotNull(report.getFateDetails());
+
+ log.debug("json: {}", report.toJson());
+ log.debug("formatted: {}", report.formatLines());
+
+ verify(status1);
+ }
+}
diff --git a/shell/src/test/java/org/apache/accumulo/shell/commands/fateCommand/TxnDetailsTest.java b/shell/src/test/java/org/apache/accumulo/shell/commands/fateCommand/TxnDetailsTest.java
new file mode 100644
index 0000000000..9889e3f609
--- /dev/null
+++ b/shell/src/test/java/org/apache/accumulo/shell/commands/fateCommand/TxnDetailsTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.shell.commands.fateCommand;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.fate.AdminUtil;
+import org.apache.accumulo.fate.ReadOnlyTStore;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class TxnDetailsTest {
+
+ private final static Logger log = LoggerFactory.getLogger(FateTxnDetails.class);
+
+ @Test
+ void orderingByDuration() {
+ Map<String,String> idMap = Map.of("1", "ns1", "2", "tbl1");
+
+ long now = System.currentTimeMillis();
+
+ AdminUtil.TransactionStatus status1 = createMock(AdminUtil.TransactionStatus.class);
+ expect(status1.getTimeCreated()).andReturn(now - TimeUnit.DAYS.toMillis(1)).anyTimes();
+ expect(status1.getStatus()).andReturn(ReadOnlyTStore.TStatus.IN_PROGRESS).anyTimes();
+ expect(status1.getTop()).andReturn("step1").anyTimes();
+ expect(status1.getDebug()).andReturn("command1").anyTimes();
+ expect(status1.getTxid()).andReturn("abcdabcd").anyTimes();
+ expect(status1.getHeldLocks()).andReturn(List.of()).anyTimes();
+ expect(status1.getWaitingLocks()).andReturn(List.of()).anyTimes();
+
+ AdminUtil.TransactionStatus status2 = createMock(AdminUtil.TransactionStatus.class);
+ expect(status2.getTimeCreated()).andReturn(now - TimeUnit.DAYS.toMillis(7)).anyTimes();
+ expect(status2.getStatus()).andReturn(ReadOnlyTStore.TStatus.IN_PROGRESS).anyTimes();
+ expect(status2.getTop()).andReturn("step2").anyTimes();
+ expect(status2.getDebug()).andReturn("command2").anyTimes();
+ expect(status2.getTxid()).andReturn("123456789").anyTimes();
+ expect(status2.getHeldLocks()).andReturn(List.of()).anyTimes();
+ expect(status2.getWaitingLocks()).andReturn(List.of()).anyTimes();
+
+ replay(status1, status2);
+
+ FateTxnDetails txn1 = new FateTxnDetails(System.currentTimeMillis(), status1, idMap);
+ FateTxnDetails txn2 = new FateTxnDetails(System.currentTimeMillis(), status2, idMap);
+
+ Set<FateTxnDetails> sorted = new TreeSet<>();
+ sorted.add(txn1);
+ sorted.add(txn2);
+
+ log.trace("Sorted: {}", sorted);
+
+ Iterator<FateTxnDetails> itor = sorted.iterator();
+
+ assertTrue(itor.next().toString().contains("123456789"));
+ assertTrue(itor.next().toString().contains("abcdabcd"));
+
+ verify(status1, status2);
+ }
+
+ @Test
+ public void lockTest() {
+ Map<String,String> idMap = Map.of("1", "ns1", "2", "tbl1", "3", "", "4", "");
+
+ long now = System.currentTimeMillis();
+
+ AdminUtil.TransactionStatus status1 = createMock(AdminUtil.TransactionStatus.class);
+ expect(status1.getTimeCreated()).andReturn(now - TimeUnit.DAYS.toMillis(1)).anyTimes();
+ expect(status1.getStatus()).andReturn(ReadOnlyTStore.TStatus.IN_PROGRESS).anyTimes();
+ expect(status1.getTop()).andReturn("step1").anyTimes();
+ expect(status1.getDebug()).andReturn("command1").anyTimes();
+ expect(status1.getTxid()).andReturn("abcdabcd").anyTimes();
+ // incomplete lock info (W unknown ns id, no table))
+ expect(status1.getHeldLocks()).andReturn(List.of("R:1", "R:2", "W:a")).anyTimes();
+ // blank names
+ expect(status1.getWaitingLocks()).andReturn(List.of("W:3", "W:4")).anyTimes();
+ replay(status1);
+
+ // R:+default, R:1
+ FateTxnDetails txn1 = new FateTxnDetails(System.currentTimeMillis(), status1, idMap);
+ assertNotNull(txn1);
+ log.info("T: {}", txn1);
+
+ verify(status1);
+ }
+}
diff --git a/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java b/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
index 628879be51..af18971d48 100644
--- a/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
@@ -24,6 +24,7 @@ import static org.apache.accumulo.harness.AccumuloITBase.MINI_CLUSTER_ONLY;
import static org.apache.accumulo.harness.AccumuloITBase.SUNNY_DAY;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -44,6 +45,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
@@ -76,6 +78,7 @@ import org.apache.accumulo.core.util.format.FormatterConfig;
import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
import org.apache.accumulo.harness.SharedMiniClusterBase;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.shell.commands.fateCommand.FateSummaryReport;
import org.apache.accumulo.test.compaction.TestCompactionStrategy;
import org.apache.accumulo.test.functional.SlowIterator;
import org.apache.hadoop.conf.Configuration;
@@ -2099,8 +2102,6 @@ public class ShellServerIT extends SharedMiniClusterBase {
+ " -s table.iterator.majc.slow=1,org.apache.accumulo.test.functional.SlowIterator");
ts.exec("config -t " + table + " -s table.iterator.majc.slow.opt.sleepTime=10000");
- String tableId = getTableId(table);
-
// make two files
ts.exec("insert a1 b c v_a1");
ts.exec("insert a2 b c v_a2");
@@ -2108,7 +2109,6 @@ public class ShellServerIT extends SharedMiniClusterBase {
ts.exec("insert x1 b c v_x1");
ts.exec("insert x2 b c v_x2");
ts.exec("flush -w");
- int oldCount = countFiles(tableId);
// no transactions running
ts.exec("fate -print", true, "0 transactions", true);
@@ -2139,6 +2139,100 @@ public class ShellServerIT extends SharedMiniClusterBase {
if (orgProps != null) {
System.setProperty("accumulo.properties", orgProps);
}
+ }
+
+ @Test
+ public void testFateSummaryCommandWithSlowCompaction() throws Exception {
+ String namespace = "ns1";
+ final String table = namespace + "." + getUniqueNames(1)[0];
+ String orgProps = System.getProperty("accumulo.properties");
+
+ System.setProperty("accumulo.properties",
+ "file://" + getCluster().getConfig().getAccumuloPropsFile().getCanonicalPath());
+ // compact
+ ts.exec("createnamespace " + namespace);
+ ts.exec("createtable " + table);
+ ts.exec("addsplits h m r w -t " + table);
+ ts.exec("offline -t " + table);
+ ts.exec("online h m r w -t " + table);
+
+ // setup SlowIterator to sleep for 10 seconds
+ ts.exec("config -t " + table
+ + " -s table.iterator.majc.slow=1,org.apache.accumulo.test.functional.SlowIterator");
+ ts.exec("config -t " + table + " -s table.iterator.majc.slow.opt.sleepTime=10000");
+
+ // make two files
+ ts.exec("insert a1 b c v_a1");
+ ts.exec("insert a2 b c v_a2");
+ ts.exec("flush -w");
+ ts.exec("insert x1 b c v_x1");
+ ts.exec("insert x2 b c v_x2");
+ ts.exec("flush -w");
+
+ // no transactions running
+
+ String cmdOut =
+ ts.exec("fate -summary -np json -t NEW IN_PROGRESS FAILED", true, "reportTime", true);
+ // strip command included in shell output
+ String jsonOut = cmdOut.substring(cmdOut.indexOf("{"));
+ FateSummaryReport report = FateSummaryReport.fromJson(jsonOut);
+
+ // validate blank report
+ assertNotNull(report);
+ assertNotEquals(0, report.getReportTime());
+ assertEquals(Set.of("NEW", "IN_PROGRESS", "FAILED"), report.getStatusFilterNames());
+ assertEquals(Map.of(), report.getStatusCounts());
+ assertEquals(Map.of(), report.getStepCounts());
+ assertEquals(Map.of(), report.getCmdCounts());
+ assertEquals(Set.of(), report.getFateDetails());
+
+ ts.exec("fate -summary -np", true, "Report Time:", true);
+
+ // merge two files into one
+ ts.exec("compact -t " + table);
+ Thread.sleep(1_000);
+ // start 2nd transaction
+ ts.exec("compact -t " + table);
+ Thread.sleep(3_000);
+
+ // 2 compactions should be running so parse the output to get one of the transaction ids
+ log.debug("Calling fate summary");
+ ts.exec("fate -summary -np", true, "Report Time:", true);
+
+ cmdOut = ts.exec("fate -summary -np json", true, "reportTime", true);
+ // strip command included in shell output
+ jsonOut = cmdOut.substring(cmdOut.indexOf("{"));
+ log.debug("report to json:\n{}", jsonOut);
+ report = FateSummaryReport.fromJson(jsonOut);
+
+ // validate no filters
+ assertNotNull(report);
+ assertNotEquals(0, report.getReportTime());
+ assertEquals(Set.of(), report.getStatusFilterNames());
+ assertFalse(report.getStatusCounts().isEmpty());
+ assertFalse(report.getStepCounts().isEmpty());
+ assertFalse(report.getCmdCounts().isEmpty());
+ assertFalse(report.getFateDetails().isEmpty());
+
+ // validate filter by excluding all
+ cmdOut = ts.exec("fate -summary -np json -t FAILED", true, "reportTime", true);
+ jsonOut = cmdOut.substring(cmdOut.indexOf("{"));
+ report = FateSummaryReport.fromJson(jsonOut);
+
+ // validate blank report
+ assertNotNull(report);
+ assertNotEquals(0, report.getReportTime());
+ assertEquals(Set.of("FAILED"), report.getStatusFilterNames());
+ assertFalse(report.getStatusCounts().isEmpty());
+ assertFalse(report.getStepCounts().isEmpty());
+ assertFalse(report.getCmdCounts().isEmpty());
+ assertEquals(0, report.getFateDetails().size());
+
+ ts.exec("deletetable -f " + table);
+
+ if (orgProps != null) {
+ System.setProperty("accumulo.properties", orgProps);
+ }
}
}