You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by bs...@apache.org on 2017/08/15 23:10:24 UTC
[13/50] [abbrv] geode git commit: GEODE-3436: revert recent
refactoring of GFSH commands
http://git-wip-us.apache.org/repos/asf/geode/blob/645a32d0/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/MiscellaneousCommands.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/MiscellaneousCommands.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/MiscellaneousCommands.java
new file mode 100644
index 0000000..1415bc6
--- /dev/null
+++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/MiscellaneousCommands.java
@@ -0,0 +1,1926 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.management.internal.cli.commands;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.zip.DataFormatException;
+import java.util.zip.GZIPInputStream;
+
+import javax.management.ObjectName;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.logging.log4j.Logger;
+import org.springframework.shell.core.annotation.CliCommand;
+import org.springframework.shell.core.annotation.CliOption;
+
+import org.apache.geode.LogWriter;
+import org.apache.geode.cache.execute.Execution;
+import org.apache.geode.cache.execute.Function;
+import org.apache.geode.cache.execute.FunctionException;
+import org.apache.geode.cache.execute.FunctionService;
+import org.apache.geode.cache.execute.ResultCollector;
+import org.apache.geode.distributed.DistributedMember;
+import org.apache.geode.distributed.internal.InternalDistributedSystem;
+import org.apache.geode.distributed.internal.deadlock.DeadlockDetector;
+import org.apache.geode.distributed.internal.deadlock.Dependency;
+import org.apache.geode.distributed.internal.deadlock.DependencyGraph;
+import org.apache.geode.distributed.internal.deadlock.GemFireDeadlockDetector;
+import org.apache.geode.internal.cache.GemFireCacheImpl;
+import org.apache.geode.internal.cache.InternalCache;
+import org.apache.geode.internal.logging.LogService;
+import org.apache.geode.internal.logging.log4j.LogLevel;
+import org.apache.geode.management.CacheServerMXBean;
+import org.apache.geode.management.DistributedRegionMXBean;
+import org.apache.geode.management.DistributedSystemMXBean;
+import org.apache.geode.management.JVMMetrics;
+import org.apache.geode.management.ManagementService;
+import org.apache.geode.management.MemberMXBean;
+import org.apache.geode.management.RegionMXBean;
+import org.apache.geode.management.cli.CliMetaData;
+import org.apache.geode.management.cli.ConverterHint;
+import org.apache.geode.management.cli.Result;
+import org.apache.geode.management.internal.MBeanJMXAdapter;
+import org.apache.geode.management.internal.ManagementConstants;
+import org.apache.geode.management.internal.SystemManagementService;
+import org.apache.geode.management.internal.cli.AbstractCliAroundInterceptor;
+import org.apache.geode.management.internal.cli.CliUtil;
+import org.apache.geode.management.internal.cli.CliUtil.DeflaterInflaterData;
+import org.apache.geode.management.internal.cli.GfshParseResult;
+import org.apache.geode.management.internal.cli.GfshParser;
+import org.apache.geode.management.internal.cli.LogWrapper;
+import org.apache.geode.management.internal.cli.domain.StackTracesPerMember;
+import org.apache.geode.management.internal.cli.functions.ChangeLogLevelFunction;
+import org.apache.geode.management.internal.cli.functions.GarbageCollectionFunction;
+import org.apache.geode.management.internal.cli.functions.GetStackTracesFunction;
+import org.apache.geode.management.internal.cli.functions.NetstatFunction;
+import org.apache.geode.management.internal.cli.functions.NetstatFunction.NetstatFunctionArgument;
+import org.apache.geode.management.internal.cli.functions.NetstatFunction.NetstatFunctionResult;
+import org.apache.geode.management.internal.cli.functions.ShutDownFunction;
+import org.apache.geode.management.internal.cli.i18n.CliStrings;
+import org.apache.geode.management.internal.cli.remote.CommandExecutionContext;
+import org.apache.geode.management.internal.cli.result.CompositeResultData;
+import org.apache.geode.management.internal.cli.result.CompositeResultData.SectionResultData;
+import org.apache.geode.management.internal.cli.result.ErrorResultData;
+import org.apache.geode.management.internal.cli.result.InfoResultData;
+import org.apache.geode.management.internal.cli.result.ResultBuilder;
+import org.apache.geode.management.internal.cli.result.ResultData;
+import org.apache.geode.management.internal.cli.result.ResultDataException;
+import org.apache.geode.management.internal.cli.result.TabularResultData;
+import org.apache.geode.management.internal.security.ResourceOperation;
+import org.apache.geode.security.ResourcePermission.Operation;
+import org.apache.geode.security.ResourcePermission.Resource;
+
+/**
+ * @since GemFire 7.0
+ */
+public class MiscellaneousCommands implements GfshCommand {
+
+ public static final String NETSTAT_FILE_REQUIRED_EXTENSION = ".txt";
+ public final static String DEFAULT_TIME_OUT = "10";
+ private final static Logger logger = LogService.getLogger();
+
+ private final GetStackTracesFunction getStackTracesFunction = new GetStackTracesFunction();
+
+ public void shutdownNode(final long timeout, final Set<DistributedMember> includeMembers)
+ throws TimeoutException, InterruptedException, ExecutionException {
+ ExecutorService exec = Executors.newSingleThreadExecutor();
+ try {
+ final Function shutDownFunction = new ShutDownFunction();
+
+ logger.info("Gfsh executing shutdown on members " + includeMembers);
+
+ Callable<String> shutdownNodes = new Callable<String>() {
+
+ @Override
+ public String call() {
+ try {
+ Execution execution = FunctionService.onMembers(includeMembers);
+ execution.execute(shutDownFunction);
+ } catch (FunctionException functionEx) {
+ // Expected Exception as the function is shutting down the target members and the result
+ // collector will get member departed exception
+ }
+ return "SUCCESS";
+ }
+ };
+
+ Future<String> result = exec.submit(shutdownNodes);
+ result.get(timeout, TimeUnit.MILLISECONDS);
+
+ } catch (TimeoutException te) {
+ logger.error("TimeoutException in shutting down members." + includeMembers);
+ throw te;
+ } catch (InterruptedException e) {
+ logger.error("InterruptedException in shutting down members." + includeMembers);
+ throw e;
+ } catch (ExecutionException e) {
+ logger.error("ExecutionException in shutting down members." + includeMembers);
+ throw e;
+ } finally {
+ exec.shutdownNow();
+ }
+ }
+
+ @CliCommand(value = CliStrings.SHUTDOWN, help = CliStrings.SHUTDOWN__HELP)
+ @CliMetaData(relatedTopic = {CliStrings.TOPIC_GEODE_LIFECYCLE},
+ interceptor = "org.apache.geode.management.internal.cli.commands.MiscellaneousCommands$Interceptor")
+ @ResourceOperation(resource = Resource.CLUSTER, operation = Operation.MANAGE)
+ public Result shutdown(
+ @CliOption(key = CliStrings.SHUTDOWN__TIMEOUT, unspecifiedDefaultValue = DEFAULT_TIME_OUT,
+ help = CliStrings.SHUTDOWN__TIMEOUT__HELP) int userSpecifiedTimeout,
+ @CliOption(key = CliStrings.INCLUDE_LOCATORS, unspecifiedDefaultValue = "false",
+ help = CliStrings.INCLUDE_LOCATORS_HELP) boolean shutdownLocators) {
+ try {
+
+ if (userSpecifiedTimeout < Integer.parseInt(DEFAULT_TIME_OUT)) {
+ return ResultBuilder.createInfoResult(CliStrings.SHUTDOWN__MSG__IMPROPER_TIMEOUT);
+ }
+
+ // convert to milliseconds
+ long timeout = userSpecifiedTimeout * 1000;
+
+ InternalCache cache = getCache();
+
+ int numDataNodes = CliUtil.getAllNormalMembers(cache).size();
+
+ Set<DistributedMember> locators = CliUtil.getAllMembers(cache);
+
+ Set<DistributedMember> dataNodes = CliUtil.getAllNormalMembers(cache);
+
+ locators.removeAll(dataNodes);
+
+ if (!shutdownLocators && numDataNodes == 0) {
+ return ResultBuilder.createInfoResult(CliStrings.SHUTDOWN__MSG__NO_DATA_NODE_FOUND);
+ }
+
+ String managerName = cache.getJmxManagerAdvisor().getDistributionManager().getId().getId();
+
+ final DistributedMember manager = CliUtil.getDistributedMemberByNameOrId(managerName);
+
+ dataNodes.remove(manager);
+
+ // shut down all data members excluding this manager if manager is a data node
+ long timeElapsed = shutDownNodeWithTimeOut(timeout, dataNodes);
+ timeout = timeout - timeElapsed;
+
+ // shut down locators one by one
+ if (shutdownLocators) {
+ if (manager == null) {
+ return ResultBuilder.createUserErrorResult(CliStrings.SHUTDOWN__MSG__MANAGER_NOT_FOUND);
+ }
+
+ // remove current locator as that would get shutdown last
+ if (locators.contains(manager)) {
+ locators.remove(manager);
+ }
+
+ for (DistributedMember locator : locators) {
+ Set<DistributedMember> lsSet = new HashSet<>();
+ lsSet.add(locator);
+ long elapsedTime = shutDownNodeWithTimeOut(timeout, lsSet);
+ timeout = timeout - elapsedTime;
+ }
+ }
+
+ if (locators.contains(manager) && !shutdownLocators) { // This means manager is a locator and
+ // shutdownLocators is false. Hence we
+ // should not stop the manager
+ return ResultBuilder.createInfoResult("Shutdown is triggered");
+ }
+ // now shut down this manager
+ Set<DistributedMember> mgrSet = new HashSet<>();
+ mgrSet.add(manager);
+ // No need to check further timeout as this is the last node we will be
+ // shutting down
+ shutDownNodeWithTimeOut(timeout, mgrSet);
+
+ } catch (TimeoutException tex) {
+ return ResultBuilder.createInfoResult(CliStrings.SHUTDOWN_TIMEDOUT);
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ return ResultBuilder.createUserErrorResult(ex.getMessage());
+ }
+
+ // @TODO. List all the nodes which could be successfully shutdown
+ return ResultBuilder.createInfoResult("Shutdown is triggered");
+ }
+
+ /**
+ * @param timeout user specified timeout
+ * @param nodesToBeStopped list of nodes to be stopped
+ * @return Elapsed time to shutdown the given nodes;
+ */
+ private long shutDownNodeWithTimeOut(long timeout, Set<DistributedMember> nodesToBeStopped)
+ throws TimeoutException, InterruptedException, ExecutionException {
+
+ long shutDownTimeStart = System.currentTimeMillis();
+ shutdownNode(timeout, nodesToBeStopped);
+
+ long shutDownTimeEnd = System.currentTimeMillis();
+
+ long timeElapsed = shutDownTimeEnd - shutDownTimeStart;
+
+ if (timeElapsed > timeout || Boolean.getBoolean("ThrowTimeoutException")) {
+ // The second check for ThrowTimeoutException is a test hook
+ throw new TimeoutException();
+ }
+ return timeElapsed;
+ }
+
+ /**
+ * Interceptor used by gfsh to intercept execution of shutdownall.
+ */
+ public static class Interceptor extends AbstractCliAroundInterceptor {
+ @Override
+ public Result preExecution(GfshParseResult parseResult) {
+
+ // This hook is for testing purpose only.
+ if (Boolean.getBoolean(CliStrings.IGNORE_INTERCEPTORS)) {
+ return ResultBuilder.createInfoResult(CliStrings.SHUTDOWN__MSG__SHUTDOWN_ENTIRE_DS);
+ }
+
+ Response response = readYesNo(CliStrings.SHUTDOWN__MSG__WARN_USER, Response.YES);
+ if (response == Response.NO) {
+ return ResultBuilder
+ .createShellClientAbortOperationResult(CliStrings.SHUTDOWN__MSG__ABORTING_SHUTDOWN);
+ } else {
+ return ResultBuilder.createInfoResult(CliStrings.SHUTDOWN__MSG__SHUTDOWN_ENTIRE_DS);
+ }
+ }
+ }
+
+ @CliCommand(value = CliStrings.GC, help = CliStrings.GC__HELP)
+ @CliMetaData(relatedTopic = {CliStrings.TOPIC_GEODE_DEBUG_UTIL})
+ @ResourceOperation(resource = Resource.CLUSTER, operation = Operation.MANAGE)
+ public Result gc(
+ @CliOption(key = {CliStrings.GROUP, CliStrings.GROUPS},
+ help = CliStrings.GC__GROUP__HELP) String[] groups,
+ @CliOption(key = CliStrings.MEMBER, optionContext = ConverterHint.ALL_MEMBER_IDNAME,
+ help = CliStrings.GC__MEMBER__HELP) String memberId) {
+ InternalCache cache = getCache();
+ Result result;
+ CompositeResultData gcResultTable = ResultBuilder.createCompositeResultData();
+ TabularResultData resultTable = gcResultTable.addSection().addTable("Table1");
+ String headerText = "GC Summary";
+ resultTable.setHeader(headerText);
+ Set<DistributedMember> dsMembers = new HashSet<>();
+ if (memberId != null && memberId.length() > 0) {
+ DistributedMember member = CliUtil.getDistributedMemberByNameOrId(memberId);
+ if (member == null) {
+ return ResultBuilder
+ .createGemFireErrorResult(memberId + CliStrings.GC__MSG__MEMBER_NOT_FOUND);
+ }
+ dsMembers.add(member);
+ result = executeAndBuildResult(resultTable, dsMembers);
+ } else if (groups != null && groups.length > 0) {
+ for (String group : groups) {
+ dsMembers.addAll(cache.getDistributedSystem().getGroupMembers(group));
+ }
+ result = executeAndBuildResult(resultTable, dsMembers);
+
+ } else {
+ // gc on entire cluster
+ // exclude locators
+ dsMembers = CliUtil.getAllNormalMembers(cache);
+ result = executeAndBuildResult(resultTable, dsMembers);
+
+ }
+ return result;
+ }
+
+ Result executeAndBuildResult(TabularResultData resultTable, Set<DistributedMember> dsMembers) {
+ try {
+ List<?> resultList;
+ Function garbageCollectionFunction = new GarbageCollectionFunction();
+ resultList =
+ (List<?>) CliUtil.executeFunction(garbageCollectionFunction, null, dsMembers).getResult();
+
+ for (Object object : resultList) {
+ if (object instanceof Exception) {
+ LogWrapper.getInstance().fine("Exception in GC " + ((Throwable) object).getMessage(),
+ ((Throwable) object));
+ continue;
+ } else if (object instanceof Throwable) {
+ LogWrapper.getInstance().fine("Exception in GC " + ((Throwable) object).getMessage(),
+ ((Throwable) object));
+ continue;
+ }
+
+ if (object != null) {
+ if (object instanceof String) {
+ // unexpected exception string - cache may be closed or something
+ return ResultBuilder.createUserErrorResult((String) object);
+ } else {
+ Map<String, String> resultMap = (Map<String, String>) object;
+ toTabularResultData(resultTable, resultMap.get("MemberId"),
+ resultMap.get("HeapSizeBeforeGC"), resultMap.get("HeapSizeAfterGC"),
+ resultMap.get("TimeSpentInGC"));
+ }
+ } else {
+ LogWrapper.getInstance().fine("ResultMap was null ");
+ }
+ }
+ } catch (Exception e) {
+ String stack = CliUtil.stackTraceAsString(e);
+ LogWrapper.getInstance().info("GC exception is " + stack);
+ return ResultBuilder.createGemFireErrorResult(e.getMessage() + ": " + stack);
+ }
+ return ResultBuilder.buildResult(resultTable);
+ }
+
+ protected void toTabularResultData(TabularResultData table, String memberId,
+ String heapSizeBefore, String heapSizeAfter, String timeTaken) {
+ table.accumulate(CliStrings.GC__MSG__MEMBER_NAME, memberId);
+ table.accumulate(CliStrings.GC__MSG__HEAP_SIZE_BEFORE_GC, heapSizeBefore);
+ table.accumulate(CliStrings.GC__MSG__HEAP_SIZE_AFTER_GC, heapSizeAfter);
+ table.accumulate(CliStrings.GC__MSG__TOTAL_TIME_IN_GC, timeTaken);
+ }
+
+ @CliCommand(value = CliStrings.NETSTAT, help = CliStrings.NETSTAT__HELP)
+ @CliMetaData(relatedTopic = {CliStrings.TOPIC_GEODE_DEBUG_UTIL})
+ @ResourceOperation(resource = Resource.CLUSTER, operation = Operation.READ)
+ // TODO : Verify the auto-completion for multiple values.
+ public Result netstat(
+ @CliOption(key = {CliStrings.MEMBER, CliStrings.MEMBERS},
+ optionContext = ConverterHint.ALL_MEMBER_IDNAME,
+ help = CliStrings.NETSTAT__MEMBER__HELP) String[] members,
+ @CliOption(key = CliStrings.GROUP, optionContext = ConverterHint.MEMBERGROUP,
+ help = CliStrings.NETSTAT__GROUP__HELP) String group,
+ @CliOption(key = CliStrings.NETSTAT__FILE,
+ help = CliStrings.NETSTAT__FILE__HELP) String saveAs,
+ @CliOption(key = CliStrings.NETSTAT__WITHLSOF, specifiedDefaultValue = "true",
+ unspecifiedDefaultValue = "false",
+ help = CliStrings.NETSTAT__WITHLSOF__HELP) boolean withlsof) {
+ Result result;
+
+ Map<String, DistributedMember> hostMemberMap = new HashMap<>();
+ Map<String, List<String>> hostMemberListMap = new HashMap<>();
+
+ try {
+ if (members != null && members.length > 0 && group != null) {
+ throw new IllegalArgumentException(
+ CliStrings.NETSTAT__MSG__ONLY_ONE_OF_MEMBER_OR_GROUP_SHOULD_BE_SPECIFIED);
+ }
+ StringBuilder resultInfo = new StringBuilder();
+
+ // Execute for remote members whose id or name matches
+ InternalDistributedSystem system = InternalDistributedSystem.getConnectedInstance();
+
+ if (members != null) {
+ Set<String> notFoundMembers = new HashSet<>();
+ for (String memberIdOrName : members) {
+ Set<DistributedMember> membersToExecuteOn = CliUtil.getAllMembers(system);
+ boolean memberFound = false;
+ for (DistributedMember distributedMember : membersToExecuteOn) {
+ String memberName = distributedMember.getName();
+ String memberId = distributedMember.getId();
+ if (memberName.equals(memberIdOrName) || memberId.equals(memberIdOrName)) {
+ buildMaps(hostMemberMap, hostMemberListMap, memberIdOrName, distributedMember);
+
+ memberFound = true;
+ break;
+ }
+ }
+ if (!memberFound) {
+ notFoundMembers.add(memberIdOrName);
+ }
+ }
+ // if there are not found members, it's probably unknown member or member has departed
+ if (!notFoundMembers.isEmpty()) {
+ throw new IllegalArgumentException(
+ CliStrings.format(CliStrings.NETSTAT__MSG__COULD_NOT_FIND_MEMBERS_0,
+ new Object[] {CliUtil.collectionToString(notFoundMembers, -1)}));
+ }
+ } else {
+ Set<DistributedMember> membersToExecuteOn;
+ if (group != null) {
+ membersToExecuteOn = system.getGroupMembers(group);
+ } else {
+ // consider all members
+ membersToExecuteOn = CliUtil.getAllMembers(system);
+ }
+
+ for (DistributedMember distributedMember : membersToExecuteOn) {
+ String memberName = distributedMember.getName();
+ String memberId = distributedMember.getId();
+ String memberIdOrName =
+ memberName != null && !memberName.isEmpty() ? memberName : memberId;
+
+ buildMaps(hostMemberMap, hostMemberListMap, memberIdOrName, distributedMember);
+ }
+ }
+
+ String lineSeparatorToUse;
+ lineSeparatorToUse = CommandExecutionContext.getShellLineSeparator();
+ if (lineSeparatorToUse == null) {
+ lineSeparatorToUse = GfshParser.LINE_SEPARATOR;
+ }
+ NetstatFunctionArgument nfa = new NetstatFunctionArgument(lineSeparatorToUse, withlsof);
+
+ if (!hostMemberMap.isEmpty()) {
+ Set<DistributedMember> membersToExecuteOn = new HashSet<>(hostMemberMap.values());
+ ResultCollector<?, ?> netstatResult =
+ CliUtil.executeFunction(NetstatFunction.INSTANCE, nfa, membersToExecuteOn);
+ List<?> resultList = (List<?>) netstatResult.getResult();
+ for (Object aResultList : resultList) {
+ NetstatFunctionResult netstatFunctionResult = (NetstatFunctionResult) aResultList;
+ DeflaterInflaterData deflaterInflaterData = netstatFunctionResult.getCompressedBytes();
+ try {
+ String remoteHost = netstatFunctionResult.getHost();
+ List<String> membersList = hostMemberListMap.get(remoteHost);
+ resultInfo.append(MessageFormat.format(netstatFunctionResult.getHeaderInfo(),
+ CliUtil.collectionToString(membersList, 120)));
+ DeflaterInflaterData uncompressedBytes = CliUtil.uncompressBytes(
+ deflaterInflaterData.getData(), deflaterInflaterData.getDataLength());
+ resultInfo.append(new String(uncompressedBytes.getData()));
+ } catch (DataFormatException e) {
+ resultInfo.append("Error in some data. Reason : " + e.getMessage());
+ }
+ }
+ }
+
+ InfoResultData resultData = ResultBuilder.createInfoResultData();
+ if (saveAs != null && !saveAs.isEmpty()) {
+ String saveToFile = saveAs;
+ if (!saveAs.endsWith(NETSTAT_FILE_REQUIRED_EXTENSION)) {
+ saveToFile = saveAs + NETSTAT_FILE_REQUIRED_EXTENSION;
+ }
+ resultData.addAsFile(saveToFile, resultInfo.toString(),
+ CliStrings.NETSTAT__MSG__SAVED_OUTPUT_IN_0, false); // Note: substitution for {0} will
+ // happen on client side.
+ } else {
+ resultData.addLine(resultInfo.toString());
+ }
+ result = ResultBuilder.buildResult(resultData);
+ } catch (IllegalArgumentException e) {
+ LogWrapper.getInstance()
+ .info(CliStrings.format(
+ CliStrings.NETSTAT__MSG__ERROR_OCCURRED_WHILE_EXECUTING_NETSTAT_ON_0,
+ new Object[] {Arrays.toString(members)}));
+ result = ResultBuilder.createUserErrorResult(e.getMessage());
+ } catch (RuntimeException e) {
+ LogWrapper.getInstance()
+ .info(CliStrings.format(
+ CliStrings.NETSTAT__MSG__ERROR_OCCURRED_WHILE_EXECUTING_NETSTAT_ON_0,
+ new Object[] {Arrays.toString(members)}), e);
+ result = ResultBuilder.createGemFireErrorResult(
+ CliStrings.format(CliStrings.NETSTAT__MSG__ERROR_OCCURRED_WHILE_EXECUTING_NETSTAT_ON_0,
+ new Object[] {Arrays.toString(members)}));
+ } finally {
+ hostMemberMap.clear();
+ hostMemberListMap.clear();
+ }
+ return result;
+ }
+
+ private void buildMaps(Map<String, DistributedMember> hostMemberMap,
+ Map<String, List<String>> hostMemberListMap, String memberIdOrName,
+ DistributedMember distributedMember) {
+ String host = distributedMember.getHost();
+
+ // Maintain one member for a host - function execution purpose - once only for a host
+ if (!hostMemberMap.containsKey(host)) {
+ hostMemberMap.put(host, distributedMember);
+ }
+
+ // Maintain all members for a host - display purpose
+ List<String> list;
+ if (!hostMemberListMap.containsKey(host)) {
+ list = new ArrayList<>();
+ hostMemberListMap.put(host, list);
+ } else {
+ list = hostMemberListMap.get(host);
+ }
+ list.add(memberIdOrName);
+ }
+
+ @CliCommand(value = CliStrings.SHOW_DEADLOCK, help = CliStrings.SHOW_DEADLOCK__HELP)
+ @CliMetaData(relatedTopic = {CliStrings.TOPIC_GEODE_DEBUG_UTIL})
+ @ResourceOperation(resource = Resource.CLUSTER, operation = Operation.READ)
+ public Result showDeadlock(@CliOption(key = CliStrings.SHOW_DEADLOCK__DEPENDENCIES__FILE,
+ help = CliStrings.SHOW_DEADLOCK__DEPENDENCIES__FILE__HELP,
+ mandatory = true) String filename) {
+
+ Result result;
+ try {
+ if (!filename.endsWith(".txt")) {
+ return ResultBuilder
+ .createUserErrorResult(CliStrings.format(CliStrings.INVALID_FILE_EXTENSION, ".txt"));
+ }
+ InternalCache cache = getCache();
+
+ Set<DistributedMember> allMembers = CliUtil.getAllMembers(cache);
+ GemFireDeadlockDetector gfeDeadLockDetector = new GemFireDeadlockDetector(allMembers);
+ DependencyGraph dependencyGraph = gfeDeadLockDetector.find();
+ Collection<Dependency> deadlock = dependencyGraph.findCycle();
+ DependencyGraph deepest = null;
+ if (deadlock == null) {
+ deepest = dependencyGraph.findLongestCallChain();
+ if (deepest != null) {
+ deadlock = deepest.getEdges();
+ }
+ }
+ Set<Dependency> dependencies = (Set<Dependency>) dependencyGraph.getEdges();
+
+ InfoResultData resultData = ResultBuilder.createInfoResultData();
+
+ if (deadlock != null) {
+ if (deepest != null) {
+ resultData.addLine(CliStrings.SHOW_DEADLOCK__DEEPEST_FOUND);
+ } else {
+ resultData.addLine(CliStrings.SHOW_DEADLOCK__DEADLOCK__DETECTED);
+ }
+ resultData.addLine(DeadlockDetector.prettyFormat(deadlock));
+ } else {
+ resultData.addLine(CliStrings.SHOW_DEADLOCK__NO__DEADLOCK);
+ }
+ resultData.addAsFile(filename, DeadlockDetector.prettyFormat(dependencies),
+ MessageFormat.format(CliStrings.SHOW_DEADLOCK__DEPENDENCIES__REVIEW, filename), false);
+ result = ResultBuilder.buildResult(resultData);
+
+ } catch (Exception e) {
+ result = ResultBuilder
+ .createGemFireErrorResult(CliStrings.SHOW_DEADLOCK__ERROR + " : " + e.getMessage());
+ }
+ return result;
+ }
+
+ @CliCommand(value = CliStrings.SHOW_LOG, help = CliStrings.SHOW_LOG_HELP)
+ @CliMetaData(relatedTopic = {CliStrings.TOPIC_GEODE_DEBUG_UTIL})
+ @ResourceOperation(resource = Resource.CLUSTER, operation = Operation.READ)
+ public Result showLog(
+ @CliOption(key = CliStrings.MEMBER, optionContext = ConverterHint.ALL_MEMBER_IDNAME,
+ help = CliStrings.SHOW_LOG_MEMBER_HELP, mandatory = true) String memberNameOrId,
+ @CliOption(key = CliStrings.SHOW_LOG_LINE_NUM, unspecifiedDefaultValue = "0",
+ help = CliStrings.SHOW_LOG_LINE_NUM_HELP) int numberOfLines) {
+ Result result;
+ try {
+ InternalCache cache = getCache();
+ SystemManagementService service =
+ (SystemManagementService) ManagementService.getExistingManagementService(cache);
+ MemberMXBean bean;
+ DistributedMember memberToBeInvoked = CliUtil.getDistributedMemberByNameOrId(memberNameOrId);
+ if (memberToBeInvoked != null) {
+ String memberId = memberToBeInvoked.getId();
+
+ if (cache.getDistributedSystem().getDistributedMember().getId().equals(memberId)) {
+ bean = service.getMemberMXBean();
+ } else {
+ ObjectName objectName = service.getMemberMBeanName(memberToBeInvoked);
+ bean = service.getMBeanProxy(objectName, MemberMXBean.class);
+ }
+
+ if (numberOfLines > ManagementConstants.MAX_SHOW_LOG_LINES) {
+ numberOfLines = ManagementConstants.MAX_SHOW_LOG_LINES;
+ }
+ if (numberOfLines == 0 || numberOfLines < 0) {
+ numberOfLines = ManagementConstants.DEFAULT_SHOW_LOG_LINES;
+ }
+ InfoResultData resultData = ResultBuilder.createInfoResultData();
+ if (bean != null) {
+ String log = bean.showLog(numberOfLines);
+ if (log != null) {
+ resultData.addLine(log);
+ } else {
+ resultData.addLine(CliStrings.SHOW_LOG_NO_LOG);
+ }
+ } else {
+ ErrorResultData errorResultData =
+ ResultBuilder.createErrorResultData().setErrorCode(ResultBuilder.ERRORCODE_DEFAULT)
+ .addLine(memberNameOrId + CliStrings.SHOW_LOG_MSG_MEMBER_NOT_FOUND);
+ return (ResultBuilder.buildResult(errorResultData));
+ }
+
+ result = ResultBuilder.buildResult(resultData);
+ } else {
+ ErrorResultData errorResultData =
+ ResultBuilder.createErrorResultData().setErrorCode(ResultBuilder.ERRORCODE_DEFAULT)
+ .addLine(memberNameOrId + CliStrings.SHOW_LOG_MSG_MEMBER_NOT_FOUND);
+ return (ResultBuilder.buildResult(errorResultData));
+ }
+
+ } catch (Exception e) {
+ result = ResultBuilder
+ .createGemFireErrorResult(CliStrings.SHOW_LOG_ERROR + CliUtil.stackTraceAsString(e));
+ }
+ return result;
+ }
+
+ /**
+ * Current implementation supports writing it to a file and returning the location of the file
+ */
+ @CliCommand(value = CliStrings.EXPORT_STACKTRACE, help = CliStrings.EXPORT_STACKTRACE__HELP)
+ @CliMetaData(relatedTopic = {CliStrings.TOPIC_GEODE_DEBUG_UTIL})
+ @ResourceOperation(resource = Resource.CLUSTER, operation = Operation.READ)
+ public Result exportStackTrace(@CliOption(key = {CliStrings.MEMBER, CliStrings.MEMBERS},
+ optionContext = ConverterHint.ALL_MEMBER_IDNAME,
+ help = CliStrings.EXPORT_STACKTRACE__HELP) String[] memberNameOrId,
+
+ @CliOption(key = {CliStrings.GROUP, CliStrings.GROUPS},
+ optionContext = ConverterHint.ALL_MEMBER_IDNAME, help = CliStrings.GROUP) String[] group,
+
+ @CliOption(key = CliStrings.EXPORT_STACKTRACE__FILE,
+ help = CliStrings.EXPORT_STACKTRACE__FILE__HELP) String fileName,
+
+ @CliOption(key = CliStrings.EXPORT_STACKTRACE__FAIL__IF__FILE__PRESENT,
+ unspecifiedDefaultValue = "false",
+ help = CliStrings.EXPORT_STACKTRACE__FAIL__IF__FILE__PRESENT__HELP) boolean failIfFilePresent) {
+
+ Result result;
+ StringBuffer filePrefix = new StringBuffer("stacktrace");
+
+ if (fileName == null) {
+ fileName = filePrefix.append("_").append(System.currentTimeMillis()).toString();
+ }
+ final File outFile = new File(fileName);
+ try {
+ if (outFile.exists() && failIfFilePresent) {
+ return ResultBuilder.createShellClientErrorResult(CliStrings.format(
+ CliStrings.EXPORT_STACKTRACE__ERROR__FILE__PRESENT, outFile.getCanonicalPath()));
+ }
+
+
+ InternalCache cache = getCache();
+ InternalDistributedSystem ads = cache.getInternalDistributedSystem();
+
+ InfoResultData resultData = ResultBuilder.createInfoResultData();
+
+ Map<String, byte[]> dumps = new HashMap<>();
+ Set<DistributedMember> targetMembers = CliUtil.findMembers(group, memberNameOrId);
+ if (targetMembers.isEmpty()) {
+ return ResultBuilder.createUserErrorResult(CliStrings.NO_MEMBERS_FOUND_MESSAGE);
+ }
+
+ ResultCollector<?, ?> rc =
+ CliUtil.executeFunction(getStackTracesFunction, null, targetMembers);
+ ArrayList<Object> resultList = (ArrayList<Object>) rc.getResult();
+
+ for (Object resultObj : resultList) {
+ if (resultObj instanceof StackTracesPerMember) {
+ StackTracesPerMember stackTracePerMember = (StackTracesPerMember) resultObj;
+ dumps.put(stackTracePerMember.getMemberNameOrId(), stackTracePerMember.getStackTraces());
+ }
+ }
+
+ String filePath = writeStacksToFile(dumps, fileName);
+ resultData.addLine(CliStrings.format(CliStrings.EXPORT_STACKTRACE__SUCCESS, filePath));
+ resultData.addLine(CliStrings.EXPORT_STACKTRACE__HOST + ads.getDistributedMember().getHost());
+
+ result = ResultBuilder.buildResult(resultData);
+ } catch (IOException ex) {
+ result = ResultBuilder
+ .createGemFireErrorResult(CliStrings.EXPORT_STACKTRACE__ERROR + ex.getMessage());
+ }
+ return result;
+ }
+
+ /**
+ * Interceptor used by gfsh to intercept execution of shutdownall.
+ */
+ public static class ExportStackTraceInterceptor extends AbstractCliAroundInterceptor {
+ @Override
+ public Result preExecution(GfshParseResult parseResult) {
+
+ Map<String, String> paramValueMap = parseResult.getParamValueStrings();
+ String fileName = paramValueMap.get(CliStrings.EXPORT_STACKTRACE__FILE);
+
+ Response response = readYesNo(
+ CliStrings.format(CliStrings.EXPORT_STACKTRACE_WARN_USER, fileName), Response.YES);
+ if (response == Response.NO) {
+ return ResultBuilder
+ .createShellClientAbortOperationResult(CliStrings.EXPORT_STACKTRACE_MSG_ABORTING);
+ } else {
+ // we don't to show any info result
+ return ResultBuilder.createInfoResult("");
+ }
+ }
+ }
+
+ /***
+ * Writes the Stack traces member-wise to a text file
+ *
+ * @param dumps - Map containing key : member , value : zipped stack traces
+ * @param fileName - Name of the file to which the stack-traces are written to
+ * @return Canonical path of the file which contains the stack-traces
+ * @throws IOException
+ */
+ private String writeStacksToFile(Map<String, byte[]> dumps, String fileName) throws IOException {
+ String filePath;
+ PrintWriter ps;
+ File outputFile;
+
+ outputFile = new File(fileName);
+ try (OutputStream os = new FileOutputStream(outputFile)) {
+ ps = new PrintWriter(os);
+
+ for (Entry<String, byte[]> entry : dumps.entrySet()) {
+ ps.append("*** Stack-trace for member " + entry.getKey() + " ***");
+ ps.flush();
+ GZIPInputStream zipIn = new GZIPInputStream(new ByteArrayInputStream(entry.getValue()));
+ BufferedInputStream bin = new BufferedInputStream(zipIn);
+ byte[] buffer = new byte[10000];
+ int count;
+ while ((count = bin.read(buffer)) != -1) {
+ os.write(buffer, 0, count);
+ }
+ ps.append('\n');
+ }
+ ps.flush();
+ filePath = outputFile.getCanonicalPath();
+ }
+
+ return filePath;
+ }
+
+ @CliCommand(value = CliStrings.SHOW_METRICS, help = CliStrings.SHOW_METRICS__HELP)
+ @CliMetaData(relatedTopic = {CliStrings.TOPIC_GEODE_STATISTICS})
+ @ResourceOperation(resource = Resource.CLUSTER, operation = Operation.READ)
+ public Result showMetrics(
+ @CliOption(key = {CliStrings.MEMBER}, optionContext = ConverterHint.ALL_MEMBER_IDNAME,
+ help = CliStrings.SHOW_METRICS__MEMBER__HELP) String memberNameOrId,
+ @CliOption(key = {CliStrings.SHOW_METRICS__REGION}, optionContext = ConverterHint.REGION_PATH,
+ help = CliStrings.SHOW_METRICS__REGION__HELP) String regionName,
+ @CliOption(key = {CliStrings.SHOW_METRICS__FILE},
+ help = CliStrings.SHOW_METRICS__FILE__HELP) String export_to_report_to,
+ @CliOption(key = {CliStrings.SHOW_METRICS__CACHESERVER__PORT},
+ help = CliStrings.SHOW_METRICS__CACHESERVER__PORT__HELP) String cacheServerPortString,
+ @CliOption(key = {CliStrings.SHOW_METRICS__CATEGORY},
+ help = CliStrings.SHOW_METRICS__CATEGORY__HELP) String[] categories) {
+
+ Result result;
+ try {
+
+ if (export_to_report_to != null && !export_to_report_to.isEmpty()) {
+ if (!export_to_report_to.endsWith(".csv")) {
+ return ResultBuilder
+ .createUserErrorResult(CliStrings.format(CliStrings.INVALID_FILE_EXTENSION, ".csv"));
+ }
+ }
+ if (regionName != null && !regionName.isEmpty()) {
+
+ if (!org.apache.geode.internal.lang.StringUtils.isBlank(cacheServerPortString)) {
+ return ResultBuilder
+ .createUserErrorResult(CliStrings.SHOW_METRICS__CANNOT__USE__CACHESERVERPORT);
+ }
+
+ // MBean names contain the forward slash
+ if (!regionName.startsWith("/")) {
+ regionName = "/" + regionName;
+ }
+
+ if (memberNameOrId == null || memberNameOrId.isEmpty()) {
+ result = ResultBuilder.buildResult(
+ getDistributedRegionMetrics(regionName, export_to_report_to, categories));
+ } else {
+ DistributedMember member = CliUtil.getDistributedMemberByNameOrId(memberNameOrId);
+
+ if (member != null) {
+ result = ResultBuilder.buildResult(
+ getRegionMetricsFromMember(regionName, member, export_to_report_to, categories));
+ } else {
+ ErrorResultData erd = ResultBuilder.createErrorResultData();
+ erd.addLine(
+ CliStrings.format(CliStrings.MEMBER_NOT_FOUND_ERROR_MESSAGE, memberNameOrId));
+ result = ResultBuilder.buildResult(erd);
+ }
+ }
+ } else if (memberNameOrId != null && !memberNameOrId.isEmpty()) {
+
+ DistributedMember member = CliUtil.getDistributedMemberByNameOrId(memberNameOrId);
+
+ if (member != null) {
+ int cacheServerPort = -1;
+ if (cacheServerPortString != null && !cacheServerPortString.isEmpty()) {
+ try {
+ cacheServerPort = Integer.parseInt(cacheServerPortString);
+ } catch (NumberFormatException nfe) {
+ return ResultBuilder.createUserErrorResult("Invalid port");
+ }
+ }
+ result = ResultBuilder.buildResult(
+ getMemberMetrics(member, export_to_report_to, categories, cacheServerPort));
+ } else {
+ ErrorResultData erd = ResultBuilder.createErrorResultData();
+ erd.addLine(CliStrings.format(CliStrings.MEMBER_NOT_FOUND_ERROR_MESSAGE, memberNameOrId));
+ result = ResultBuilder.buildResult(erd);
+ }
+ } else {
+ if (!org.apache.geode.internal.lang.StringUtils.isBlank(cacheServerPortString)) {
+ return ResultBuilder
+ .createUserErrorResult(CliStrings.SHOW_METRICS__CANNOT__USE__CACHESERVERPORT);
+ }
+ result = ResultBuilder.buildResult(getSystemWideMetrics(export_to_report_to, categories));
+ }
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ return ResultBuilder.createGemFireErrorResult(CliUtil.stackTraceAsString(e));
+ }
+ return result;
+ }
+
+ /**
+ * Gets the system wide metrics
+ *
+ * @return ResultData with required System wide statistics or ErrorResultData if DS MBean is not
+ * found to gather metrics
+ */
+ private ResultData getSystemWideMetrics(String export_to_report_to, String[] categoriesArr) {
+ final InternalCache cache = getCache();
+ final ManagementService managementService = ManagementService.getManagementService(cache);
+ DistributedSystemMXBean dsMxBean = managementService.getDistributedSystemMXBean();
+ StringBuilder csvBuilder = null;
+ if (dsMxBean != null) {
+
+ if (export_to_report_to != null && !export_to_report_to.isEmpty()) {
+ csvBuilder = new StringBuilder();
+ csvBuilder.append("Category");
+ csvBuilder.append(',');
+ csvBuilder.append(CliStrings.SHOW_METRICS__METRIC__HEADER);
+ csvBuilder.append(',');
+ csvBuilder.append(CliStrings.SHOW_METRICS__VALUE__HEADER);
+ csvBuilder.append('\n');
+ }
+
+ CompositeResultData crd = ResultBuilder.createCompositeResultData();
+ SectionResultData section = crd.addSection();
+ TabularResultData metricsTable = section.addTable();
+ Map<String, Boolean> categoriesMap = getSystemMetricsCategories();
+
+ if (categoriesArr != null && categoriesArr.length != 0) {
+ Set<String> categories = createSet(categoriesArr);
+ Set<String> checkSet = new HashSet<>(categoriesMap.keySet());
+ Set<String> userCategories = getSetDifference(categories, checkSet);
+
+ // Checking if the categories specified by the user are valid or not
+
+ if (userCategories.isEmpty()) {
+ for (String category : checkSet) {
+ categoriesMap.put(category, false);
+ }
+ for (String category : categories) {
+ categoriesMap.put(category.toLowerCase(), true);
+ }
+ } else {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Invalid Categories\n");
+
+ for (String category : userCategories) {
+ sb.append(category);
+ sb.append('\n');
+ }
+ return ResultBuilder.createErrorResultData().addLine(sb.toString());
+ }
+ }
+ metricsTable.setHeader("Cluster-wide Metrics");
+
+ if (categoriesMap.get("cluster")) {
+ writeToTableAndCsv(metricsTable, "cluster", "totalHeapSize", dsMxBean.getTotalHeapSize(),
+ csvBuilder);
+ }
+
+ if (categoriesMap.get("cache")) {
+ writeToTableAndCsv(metricsTable, "cache", "totalRegionEntryCount",
+ dsMxBean.getTotalRegionEntryCount(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalRegionCount", dsMxBean.getTotalRegionCount(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalMissCount", dsMxBean.getTotalMissCount(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalHitCount", dsMxBean.getTotalHitCount(),
+ csvBuilder);
+ }
+
+ if (categoriesMap.get("diskstore")) {
+ writeToTableAndCsv(metricsTable, "diskstore", "totalDiskUsage",
+ dsMxBean.getTotalDiskUsage(), csvBuilder); // deadcoded to workaround bug 46397
+ writeToTableAndCsv(metricsTable, ""/* 46608 */, "diskReadsRate",
+ dsMxBean.getDiskReadsRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "diskWritesRate", dsMxBean.getDiskWritesRate(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "flushTimeAvgLatency",
+ dsMxBean.getDiskFlushAvgLatency(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalBackupInProgress",
+ dsMxBean.getTotalBackupInProgress(), csvBuilder);
+ }
+
+ if (categoriesMap.get("query")) {
+ writeToTableAndCsv(metricsTable, "query", "activeCQCount", dsMxBean.getActiveCQCount(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "queryRequestRate", dsMxBean.getQueryRequestRate(),
+ csvBuilder);
+ }
+ if (export_to_report_to != null && !export_to_report_to.isEmpty()) {
+ crd.addAsFile(export_to_report_to, csvBuilder.toString(),
+ "Cluster wide metrics exported to {0}.", false);
+ }
+
+ return crd;
+ } else {
+ String errorMessage =
+ CliStrings.format(CliStrings.SHOW_METRICS__ERROR, "Distributed System MBean not found");
+ return ResultBuilder.createErrorResultData().addLine(errorMessage);
+ }
+ }
+
+ /**
+ * Gets the Cluster wide metrics for a given member
+ *
+ * @return ResultData with required Member statistics or ErrorResultData if MemberMbean is not
+ * found to gather metrics
+ * @throws ResultDataException if building result fails
+ */
+ private ResultData getMemberMetrics(DistributedMember distributedMember,
+ String export_to_report_to, String[] categoriesArr, int cacheServerPort)
+ throws ResultDataException {
+ final InternalCache cache = getCache();
+ final SystemManagementService managementService =
+ (SystemManagementService) ManagementService.getManagementService(cache);
+
+ ObjectName memberMBeanName = managementService.getMemberMBeanName(distributedMember);
+ MemberMXBean memberMxBean =
+ managementService.getMBeanInstance(memberMBeanName, MemberMXBean.class);
+ ObjectName csMxBeanName;
+ CacheServerMXBean csMxBean = null;
+
+ if (memberMxBean != null) {
+
+ if (cacheServerPort != -1) {
+ csMxBeanName =
+ managementService.getCacheServerMBeanName(cacheServerPort, distributedMember);
+ csMxBean = managementService.getMBeanInstance(csMxBeanName, CacheServerMXBean.class);
+
+ if (csMxBean == null) {
+ ErrorResultData erd = ResultBuilder.createErrorResultData();
+ erd.addLine(CliStrings.format(CliStrings.SHOW_METRICS__CACHE__SERVER__NOT__FOUND,
+ cacheServerPort, MBeanJMXAdapter.getMemberNameOrId(distributedMember)));
+ return erd;
+ }
+ }
+
+ JVMMetrics jvmMetrics = memberMxBean.showJVMMetrics();
+
+ CompositeResultData crd = ResultBuilder.createCompositeResultData();
+ SectionResultData section = crd.addSection();
+ TabularResultData metricsTable = section.addTable();
+ metricsTable.setHeader("Member Metrics");
+ StringBuilder csvBuilder = null;
+
+ if (export_to_report_to != null && !export_to_report_to.isEmpty()) {
+ csvBuilder = new StringBuilder();
+ csvBuilder.append("Category");
+ csvBuilder.append(',');
+ csvBuilder.append(CliStrings.SHOW_METRICS__METRIC__HEADER);
+ csvBuilder.append(',');
+ csvBuilder.append(CliStrings.SHOW_METRICS__VALUE__HEADER);
+ csvBuilder.append('\n');
+ }
+
+ Map<String, Boolean> categoriesMap = getMemberMetricsCategories();
+
+ if (categoriesArr != null && categoriesArr.length != 0) {
+ Set<String> categories = createSet(categoriesArr);
+ Set<String> checkSet = new HashSet<>(categoriesMap.keySet());
+ Set<String> userCategories = getSetDifference(categories, checkSet);
+
+ // Checking if the categories specified by the user are valid or not
+ if (userCategories.isEmpty()) {
+ for (String category : checkSet) {
+ categoriesMap.put(category, false);
+ }
+ for (String category : categories) {
+ categoriesMap.put(category.toLowerCase(), true);
+ }
+ } else {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Invalid Categories\n");
+
+ for (String category : userCategories) {
+ sb.append(category);
+ sb.append('\n');
+ }
+ return ResultBuilder.createErrorResultData().addLine(sb.toString());
+ }
+ }
+
+ /*
+ * Member Metrics
+ */
+ // member, jvm, region, serialization, communication, function, transaction, diskstore, lock,
+ // eviction, distribution
+ if (categoriesMap.get("member")) {
+ writeToTableAndCsv(metricsTable, "member", "upTime", memberMxBean.getMemberUpTime(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "cpuUsage", memberMxBean.getCpuUsage(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "currentHeapSize", memberMxBean.getCurrentHeapSize(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "maximumHeapSize", memberMxBean.getMaximumHeapSize(),
+ csvBuilder);
+ }
+ /*
+ * JVM Metrics
+ */
+ if (categoriesMap.get("jvm")) {
+ writeToTableAndCsv(metricsTable, "jvm ", "jvmThreads ", jvmMetrics.getTotalThreads(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "fileDescriptorLimit",
+ memberMxBean.getFileDescriptorLimit(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalFileDescriptorOpen",
+ memberMxBean.getTotalFileDescriptorOpen(), csvBuilder);
+ }
+ /*
+ * Member wide region metrics
+ */
+ if (categoriesMap.get("region")) {
+ writeToTableAndCsv(metricsTable, "region ", "totalRegionCount ",
+ memberMxBean.getTotalRegionCount(), csvBuilder);
+ String[] regionNames = memberMxBean.listRegions();
+ if (regionNames != null) {
+ for (int i = 0; i < regionNames.length; i++) {
+ if (i == 0) {
+ writeToTableAndCsv(metricsTable, "", "listOfRegions", regionNames[i].substring(1),
+ csvBuilder);
+ } else {
+ writeToTableAndCsv(metricsTable, "", "", regionNames[i].substring(1), csvBuilder);
+ }
+ }
+ }
+
+ String[] rootRegionNames = memberMxBean.getRootRegionNames();
+ if (rootRegionNames != null) {
+ for (int i = 0; i < rootRegionNames.length; i++) {
+ if (i == 0) {
+ writeToTableAndCsv(metricsTable, "", "rootRegions", rootRegionNames[i], csvBuilder);
+ } else {
+ writeToTableAndCsv(metricsTable, "", "", rootRegionNames[i], csvBuilder);
+ }
+ }
+ }
+ writeToTableAndCsv(metricsTable, "", "totalRegionEntryCount",
+ memberMxBean.getTotalRegionEntryCount(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalBucketCount", memberMxBean.getTotalBucketCount(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalPrimaryBucketCount",
+ memberMxBean.getTotalPrimaryBucketCount(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "getsAvgLatency", memberMxBean.getGetsAvgLatency(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "putsAvgLatency", memberMxBean.getPutsAvgLatency(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "createsRate", memberMxBean.getCreatesRate(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "destroyRate", memberMxBean.getDestroysRate(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "putAllAvgLatency", memberMxBean.getPutAllAvgLatency(),
+ csvBuilder);
+ // Not available from stats. After Stats re-org it will be available
+ // writeToTableAndCsv(metricsTable, "", "getAllAvgLatency",
+ // memberMxBean.getGetAllAvgLatency(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalMissCount", memberMxBean.getTotalMissCount(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalHitCount", memberMxBean.getTotalHitCount(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "getsRate", memberMxBean.getGetsRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "putsRate", memberMxBean.getPutsRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "cacheWriterCallsAvgLatency",
+ memberMxBean.getCacheWriterCallsAvgLatency(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "cacheListenerCallsAvgLatency",
+ memberMxBean.getCacheListenerCallsAvgLatency(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalLoadsCompleted",
+ memberMxBean.getTotalLoadsCompleted(), csvBuilder);
+ }
+
+ /*
+ * SERIALIZATION
+ */
+ if (categoriesMap.get("serialization")) {
+ writeToTableAndCsv(metricsTable, "serialization", "serializationRate",
+ memberMxBean.getSerializationRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "serializationLatency",
+ memberMxBean.getSerializationRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "deserializationRate",
+ memberMxBean.getDeserializationRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "deserializationLatency",
+ memberMxBean.getDeserializationLatency(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "deserializationAvgLatency",
+ memberMxBean.getDeserializationAvgLatency(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "PDXDeserializationAvgLatency",
+ memberMxBean.getPDXDeserializationAvgLatency(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "PDXDeserializationRate",
+ memberMxBean.getPDXDeserializationRate(), csvBuilder);
+ }
+
+ /*
+ * Communication Metrics
+ */
+ if (categoriesMap.get("communication")) {
+ writeToTableAndCsv(metricsTable, "communication", "bytesSentRate",
+ memberMxBean.getBytesSentRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "bytesReceivedRate",
+ memberMxBean.getBytesReceivedRate(), csvBuilder);
+ String[] connectedGatewayReceivers = memberMxBean.listConnectedGatewayReceivers();
+ writeToTableAndCsv(metricsTable, "", "connectedGatewayReceivers", connectedGatewayReceivers,
+ csvBuilder);
+
+ String[] connectedGatewaySenders = memberMxBean.listConnectedGatewaySenders();
+ writeToTableAndCsv(metricsTable, "", "connectedGatewaySenders", connectedGatewaySenders,
+ csvBuilder);
+ }
+
+ /*
+ * Member wide function metrics
+ */
+ if (categoriesMap.get("function")) {
+ writeToTableAndCsv(metricsTable, "function", "numRunningFunctions",
+ memberMxBean.getNumRunningFunctions(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "functionExecutionRate",
+ memberMxBean.getFunctionExecutionRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "numRunningFunctionsHavingResults",
+ memberMxBean.getNumRunningFunctionsHavingResults(), csvBuilder);
+ }
+
+ /*
+ * totalTransactionsCount currentTransactionalThreadIds transactionCommitsAvgLatency
+ * transactionCommittedTotalCount transactionRolledBackTotalCount transactionCommitsRate
+ */
+ if (categoriesMap.get("transaction")) {
+ writeToTableAndCsv(metricsTable, "transaction", "totalTransactionsCount",
+ memberMxBean.getTotalTransactionsCount(), csvBuilder);
+
+ writeToTableAndCsv(metricsTable, "", "transactionCommitsAvgLatency",
+ memberMxBean.getTransactionCommitsAvgLatency(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "transactionCommittedTotalCount",
+ memberMxBean.getTransactionCommittedTotalCount(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "transactionRolledBackTotalCount",
+ memberMxBean.getTransactionRolledBackTotalCount(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "transactionCommitsRate",
+ memberMxBean.getTransactionCommitsRate(), csvBuilder);
+ }
+ /*
+ * Member wide disk metrics
+ */
+ if (categoriesMap.get("diskstore")) {
+ writeToTableAndCsv(metricsTable, "diskstore", "totalDiskUsage",
+ memberMxBean.getTotalDiskUsage(), csvBuilder); // deadcoded to workaround bug 46397
+ writeToTableAndCsv(metricsTable, ""/* 46608 */, "diskReadsRate",
+ memberMxBean.getDiskReadsRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "diskWritesRate", memberMxBean.getDiskWritesRate(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "flushTimeAvgLatency",
+ memberMxBean.getDiskFlushAvgLatency(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalQueueSize",
+ memberMxBean.getTotalDiskTasksWaiting(), csvBuilder); // deadcoded to workaround bug
+ // 46397
+ writeToTableAndCsv(metricsTable, "", "totalBackupInProgress",
+ memberMxBean.getTotalBackupInProgress(), csvBuilder);
+ }
+ /*
+ * Member wide Lock
+ */
+ if (categoriesMap.get("lock")) {
+ writeToTableAndCsv(metricsTable, "lock", "lockWaitsInProgress",
+ memberMxBean.getLockWaitsInProgress(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalLockWaitTime",
+ memberMxBean.getTotalLockWaitTime(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalNumberOfLockService",
+ memberMxBean.getTotalNumberOfLockService(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "requestQueues", memberMxBean.getLockRequestQueues(),
+ csvBuilder);
+ }
+ /*
+ * Eviction
+ */
+ if (categoriesMap.get("eviction")) {
+ writeToTableAndCsv(metricsTable, "eviction", "lruEvictionRate",
+ memberMxBean.getLruEvictionRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "lruDestroyRate", memberMxBean.getLruDestroyRate(),
+ csvBuilder);
+ }
+ /*
+ * Distribution
+ */
+ if (categoriesMap.get("distribution")) {
+ writeToTableAndCsv(metricsTable, "distribution", "getInitialImagesInProgress",
+ memberMxBean.getInitialImagesInProgress(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "getInitialImageTime",
+ memberMxBean.getInitialImageTime(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "getInitialImageKeysReceived",
+ memberMxBean.getInitialImageKeysReceived(), csvBuilder);
+ }
+
+ /*
+ * OffHeap
+ */
+ if (categoriesMap.get("offheap")) {
+ writeToTableAndCsv(metricsTable, "offheap", "maxMemory", memberMxBean.getOffHeapMaxMemory(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "freeMemory", memberMxBean.getOffHeapFreeMemory(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "usedMemory", memberMxBean.getOffHeapUsedMemory(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "objects", memberMxBean.getOffHeapObjects(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "fragmentation",
+ memberMxBean.getOffHeapFragmentation(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "compactionTime",
+ memberMxBean.getOffHeapCompactionTime(), csvBuilder);
+ }
+
+ /*
+ * CacheServer stats
+ */
+ if (csMxBean != null) {
+ writeToTableAndCsv(metricsTable, "cache-server", "clientConnectionCount",
+ csMxBean.getClientConnectionCount(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "hostnameForClients", csMxBean.getHostNameForClients(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "getRequestAvgLatency",
+ csMxBean.getGetRequestAvgLatency(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "putRequestAvgLatency",
+ csMxBean.getPutRequestAvgLatency(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalConnectionsTimedOut",
+ csMxBean.getTotalConnectionsTimedOut(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "threadQueueSize", csMxBean.getPutRequestAvgLatency(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "connectionThreads", csMxBean.getConnectionThreads(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "connectionLoad", csMxBean.getConnectionLoad(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "loadPerConnection", csMxBean.getLoadPerConnection(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "queueLoad", csMxBean.getQueueLoad(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "loadPerQueue", csMxBean.getLoadPerQueue(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "getRequestRate", csMxBean.getGetRequestRate(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "putRequestRate", csMxBean.getPutRequestRate(),
+ csvBuilder);
+
+ /*
+ * Notification
+ */
+ writeToTableAndCsv(metricsTable, "notification", "numClientNotificationRequests",
+ csMxBean.getNumClientNotificationRequests(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "clientNotificationRate",
+ csMxBean.getClientNotificationRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "clientNotificationAvgLatency",
+ csMxBean.getClientNotificationAvgLatency(), csvBuilder);
+
+ /*
+ * Query
+ */
+ writeToTableAndCsv(metricsTable, "query", "activeCQCount", csMxBean.getActiveCQCount(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "query", "queryRequestRate",
+ csMxBean.getQueryRequestRate(), csvBuilder);
+
+ writeToTableAndCsv(metricsTable, "", "indexCount", csMxBean.getIndexCount(), csvBuilder);
+
+ String[] indexList = csMxBean.getIndexList();
+ writeToTableAndCsv(metricsTable, "", "index list", indexList, csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalIndexMaintenanceTime",
+ csMxBean.getTotalIndexMaintenanceTime(), csvBuilder);
+ }
+
+ if (export_to_report_to != null && !export_to_report_to.isEmpty()) {
+ crd.addAsFile(export_to_report_to, csvBuilder.toString(), "Member metrics exported to {0}.",
+ false);
+ }
+ return crd;
+
+ } else {
+ ErrorResultData erd = ResultBuilder.createErrorResultData();
+ String errorMessage = CliStrings.format(CliStrings.SHOW_METRICS__ERROR, "Member MBean for "
+ + MBeanJMXAdapter.getMemberNameOrId(distributedMember) + " not found");
+ return ResultBuilder.createErrorResultData().addLine(errorMessage);
+ }
+ }
+
+ /**
+ * Gets the Cluster-wide metrics for a region
+ *
+ * @return ResultData containing the table
+ * @throws ResultDataException if building result fails
+ */
+ private ResultData getDistributedRegionMetrics(String regionName, String export_to_report_to,
+ String[] categoriesArr) throws ResultDataException {
+
+ final InternalCache cache = getCache();
+ final ManagementService managementService = ManagementService.getManagementService(cache);
+
+ DistributedRegionMXBean regionMxBean = managementService.getDistributedRegionMXBean(regionName);
+
+ if (regionMxBean != null) {
+ CompositeResultData crd = ResultBuilder.createCompositeResultData();
+ SectionResultData section = crd.addSection();
+ TabularResultData metricsTable = section.addTable();
+ metricsTable.setHeader("Cluster-wide Region Metrics");
+ StringBuilder csvBuilder = null;
+
+ if (export_to_report_to != null && !export_to_report_to.isEmpty()) {
+ csvBuilder = new StringBuilder();
+ csvBuilder.append("Category");
+ csvBuilder.append(',');
+ csvBuilder.append(CliStrings.SHOW_METRICS__METRIC__HEADER);
+ csvBuilder.append(',');
+ csvBuilder.append(CliStrings.SHOW_METRICS__VALUE__HEADER);
+ csvBuilder.append('\n');
+ }
+
+ Map<String, Boolean> categoriesMap = getSystemRegionMetricsCategories();
+
+ if (categoriesArr != null && categoriesArr.length != 0) {
+ Set<String> categories = createSet(categoriesArr);
+ Set<String> checkSet = new HashSet<>(categoriesMap.keySet());
+ Set<String> userCategories = getSetDifference(categories, checkSet);
+
+ // Checking if the categories specified by the user are valid or not
+ if (userCategories.isEmpty()) {
+ for (String category : checkSet) {
+ categoriesMap.put(category, false);
+ }
+ for (String category : categories) {
+ categoriesMap.put(category.toLowerCase(), true);
+ }
+ } else {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Invalid Categories\n");
+
+ for (String category : userCategories) {
+ sb.append(category);
+ sb.append('\n');
+ }
+ return ResultBuilder.createErrorResultData().addLine(sb.toString());
+ }
+ }
+ /*
+ * General System metrics
+ */
+ // cluster, region, partition , diskstore, callback, eviction
+ if (categoriesMap.get("cluster")) {
+ writeToTableAndCsv(metricsTable, "cluster", "member count", regionMxBean.getMemberCount(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "region entry count",
+ regionMxBean.getSystemRegionEntryCount(), csvBuilder);
+ }
+
+ if (categoriesMap.get("region")) {
+ writeToTableAndCsv(metricsTable, "region", "lastModifiedTime",
+ regionMxBean.getLastModifiedTime(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "lastAccessedTime", regionMxBean.getLastAccessedTime(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "missCount", regionMxBean.getMissCount(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "hitCount", regionMxBean.getHitCount(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "hitRatio", regionMxBean.getHitRatio(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "getsRate", regionMxBean.getGetsRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "putsRate", regionMxBean.getPutsRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "createsRate", regionMxBean.getCreatesRate(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "destroyRate", regionMxBean.getDestroyRate(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "putAllRate", regionMxBean.getPutAllRate(),
+ csvBuilder);
+ }
+
+ if (categoriesMap.get("partition")) {
+ writeToTableAndCsv(metricsTable, "partition", "putLocalRate",
+ regionMxBean.getPutLocalRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "putRemoteRate", regionMxBean.getPutRemoteRate(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "putRemoteLatency", regionMxBean.getPutRemoteLatency(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "putRemoteAvgLatency",
+ regionMxBean.getPutRemoteAvgLatency(), csvBuilder);
+
+ writeToTableAndCsv(metricsTable, "", "bucketCount", regionMxBean.getBucketCount(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "primaryBucketCount",
+ regionMxBean.getPrimaryBucketCount(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "numBucketsWithoutRedundancy",
+ regionMxBean.getNumBucketsWithoutRedundancy(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalBucketSize", regionMxBean.getTotalBucketSize(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "averageBucketSize", regionMxBean.getAvgBucketSize(),
+ csvBuilder);
+ }
+ /*
+ * Disk store
+ */
+ if (categoriesMap.get("diskstore")) {
+ writeToTableAndCsv(metricsTable, "diskstore", "totalEntriesOnlyOnDisk",
+ regionMxBean.getTotalEntriesOnlyOnDisk(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "diskReadsRate", regionMxBean.getDiskReadsRate(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "diskWritesRate", regionMxBean.getDiskWritesRate(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalDiskWriteInProgress",
+ regionMxBean.getTotalDiskWritesProgress(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "diskTaskWaiting", regionMxBean.getDiskTaskWaiting(),
+ csvBuilder);
+
+ }
+ /*
+ * LISTENER
+ */
+ if (categoriesMap.get("callback")) {
+ writeToTableAndCsv(metricsTable, "callback", "cacheWriterCallsAvgLatency",
+ regionMxBean.getCacheWriterCallsAvgLatency(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "cacheListenerCallsAvgLatency",
+ regionMxBean.getCacheListenerCallsAvgLatency(), csvBuilder);
+ }
+
+ /*
+ * Eviction
+ */
+ if (categoriesMap.get("eviction")) {
+ writeToTableAndCsv(metricsTable, "eviction", "lruEvictionRate",
+ regionMxBean.getLruEvictionRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "lruDestroyRate", regionMxBean.getLruDestroyRate(),
+ csvBuilder);
+ }
+
+ if (export_to_report_to != null && !export_to_report_to.isEmpty()) {
+ crd.addAsFile(export_to_report_to, csvBuilder.toString(),
+ "Aggregate Region Metrics exported to {0}.", false);
+ }
+
+ return crd;
+ } else {
+ ErrorResultData erd = ResultBuilder.createErrorResultData();
+ String errorMessage = CliStrings.format(CliStrings.SHOW_METRICS__ERROR,
+ "Distributed Region MBean for " + regionName + " not found");
+ erd.addLine(errorMessage);
+ return erd;
+ }
+ }
+
+ /**
+ * Gets the metrics of region on a given member
+ *
+ * @return ResultData with required Region statistics or ErrorResultData if Region MBean is not
+ * found to gather metrics
+ * @throws ResultDataException if building result fails
+ */
+ private ResultData getRegionMetricsFromMember(String regionName,
+ DistributedMember distributedMember, String export_to_report_to, String[] categoriesArr)
+ throws ResultDataException {
+
+ final InternalCache cache = getCache();
+ final SystemManagementService managementService =
+ (SystemManagementService) ManagementService.getManagementService(cache);
+
+ ObjectName regionMBeanName =
+ managementService.getRegionMBeanName(distributedMember, regionName);
+ RegionMXBean regionMxBean =
+ managementService.getMBeanInstance(regionMBeanName, RegionMXBean.class);
+
+ if (regionMxBean != null) {
+ CompositeResultData crd = ResultBuilder.createCompositeResultData();
+ SectionResultData section = crd.addSection();
+ TabularResultData metricsTable = section.addTable();
+ metricsTable.setHeader("Metrics for region:" + regionName + " On Member "
+ + MBeanJMXAdapter.getMemberNameOrId(distributedMember));
+ StringBuilder csvBuilder = null;
+
+ if (export_to_report_to != null && !export_to_report_to.isEmpty()) {
+ csvBuilder = new StringBuilder();
+ csvBuilder.append("Category");
+ csvBuilder.append(',');
+ csvBuilder.append(CliStrings.SHOW_METRICS__METRIC__HEADER);
+ csvBuilder.append(',');
+ csvBuilder.append(CliStrings.SHOW_METRICS__VALUE__HEADER);
+ csvBuilder.append('\n');
+ }
+
+ /*
+ * Region Metrics
+ */
+ Map<String, Boolean> categoriesMap = getRegionMetricsCategories();
+
+ if (categoriesArr != null && categoriesArr.length != 0) {
+ Set<String> categories = createSet(categoriesArr);
+ Set<String> checkSet = new HashSet<>(categoriesMap.keySet());
+ Set<String> userCategories = getSetDifference(categories, checkSet);
+
+ // Checking if the categories specified by the user are valid or not
+ if (userCategories.isEmpty()) {
+ for (String category : checkSet) {
+ categoriesMap.put(category, false);
+ }
+ for (String category : categories) {
+ categoriesMap.put(category.toLowerCase(), true);
+ }
+ } else {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Invalid Categories\n");
+
+ for (String category : userCategories) {
+ sb.append(category);
+ sb.append('\n');
+ }
+ return ResultBuilder.createErrorResultData().addLine(sb.toString());
+ }
+ }
+
+ if (categoriesMap.get("region")) {
+ writeToTableAndCsv(metricsTable, "region", "lastModifiedTime",
+ regionMxBean.getLastModifiedTime(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "lastAccessedTime", regionMxBean.getLastAccessedTime(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "missCount", regionMxBean.getMissCount(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "hitCount", regionMxBean.getHitCount(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "hitRatio", regionMxBean.getHitRatio(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "getsRate", regionMxBean.getGetsRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "putsRate", regionMxBean.getPutsRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "createsRate", regionMxBean.getCreatesRate(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "destroyRate", regionMxBean.getDestroyRate(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "putAllRate", regionMxBean.getPutAllRate(),
+ csvBuilder);
+ }
+
+ if (categoriesMap.get("partition")) {
+ writeToTableAndCsv(metricsTable, "partition", "putLocalRate",
+ regionMxBean.getPutLocalRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "putRemoteRate", regionMxBean.getPutRemoteRate(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "putRemoteLatency", regionMxBean.getPutRemoteLatency(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "putRemoteAvgLatency",
+ regionMxBean.getPutRemoteAvgLatency(), csvBuilder);
+
+ writeToTableAndCsv(metricsTable, "", "bucketCount", regionMxBean.getBucketCount(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "primaryBucketCount",
+ regionMxBean.getPrimaryBucketCount(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "configuredRedundancy",
+ regionMxBean.getConfiguredRedundancy(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "actualRedundancy", regionMxBean.getActualRedundancy(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "numBucketsWithoutRedundancy",
+ regionMxBean.getNumBucketsWithoutRedundancy(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalBucketSize", regionMxBean.getTotalBucketSize(),
+ csvBuilder);
+ }
+ /*
+ * Disk store
+ */
+ if (categoriesMap.get("diskstore")) {
+ writeToTableAndCsv(metricsTable, "diskstore", "totalEntriesOnlyOnDisk",
+ regionMxBean.getTotalEntriesOnlyOnDisk(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "diskReadsRate", "" + regionMxBean.getDiskReadsRate(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "diskWritesRate", regionMxBean.getDiskWritesRate(),
+ csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "totalDiskWriteInProgress",
+ regionMxBean.getTotalDiskWritesProgress(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "diskTaskWaiting", regionMxBean.getDiskTaskWaiting(),
+ csvBuilder);
+ }
+ /*
+ * LISTENER
+ */
+ if (categoriesMap.get("callback")) {
+ writeToTableAndCsv(metricsTable, "callback", "cacheWriterCallsAvgLatency",
+ regionMxBean.getCacheWriterCallsAvgLatency(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "cacheListenerCallsAvgLatency",
+ regionMxBean.getCacheListenerCallsAvgLatency(), csvBuilder);
+ }
+
+ /*
+ * Eviction
+ */
+ if (categoriesMap.get("eviction")) {
+ writeToTableAndCsv(metricsTable, "eviction", "lruEvictionRate",
+ regionMxBean.getLruEvictionRate(), csvBuilder);
+ writeToTableAndCsv(metricsTable, "", "lruDestroyRate", regionMxBean.getLruDestroyRate(),
+ csvBuilder);
+ }
+ if (export_to_report_to != null && !export_to_report_to.isEmpty()) {
+ crd.addAsFile(export_to_report_to, csvBuilder.toString(), "Region Metrics exported to {0}.",
+ false);
+ }
+
+ return crd;
+ } else {
+ ErrorResultData erd = ResultBuilder.createErrorResultData();
+ String errorMessage = CliStrings.format(CliStrings.SHOW_METRICS__ERROR,
+ "Region MBean for " + regionName + " on member "
+ + MBeanJMXAdapter.getMemberNameOrId(distributedMember) + " not found");
+ erd.addLine(errorMessage);
+ return erd;
+ }
+ }
+
+ /***
+ * Writes an entry to a TabularResultData and writes a comma separated entry to a string builder
+ */
+ private void writeToTableAndCsv(TabularResultData metricsTable, String type, String metricName,
+ long metricValue, StringBuilder csvBuilder) {
+ metricsTable.accumulate(CliStrings.SHOW_METRICS__TYPE__HEADER, type);
+ metricsTable.accumulate(CliStrings.SHOW_METRICS__METRIC__HEADER, metricName);
+ metricsTable.accumulate(CliStrings.SHOW_METRICS__VALUE__HEADER, metricValue);
+
+ if (csvBuilder != null) {
+ csvBuilder.append(type);
+ csvBuilder.append(',');
+ csvBuilder.append(metricName);
+ csvBuilder.append(',');
+ csvBuilder.append(metricValue);
+ csvBuilder.append('\n');
+ }
+ }
+
+ private void writeToTableAndCsv(TabularResultData metricsTable, String type, String metricName,
+ double metricValue, StringBuilder csvBuilder) {
+ metricsTable.accumulate(CliStrings.SHOW_METRICS__TYPE__HEADER, type);
+ metricsTable.accumulate(CliStrings.SHOW_METRICS__METRIC__HEADER, metricName);
+ metricsTable.accumulate(CliStrings.SHOW_METRICS__VALUE__HEADER, metricValue);
+
+ if (csvBuilder != null) {
+ csvBuilder.append(type);
+ csvBuilder.append(',');
+ csvBuilder.append(metricName);
+ csvBuilder.append(',');
+ csvBuilder.append(metricValue);
+ csvBuilder.append('\n');
+ }
+ }
+
+ private Set<String> createSet(String[] categories) {
+ Set<String> categoriesSet = new HashSet<>();
+ Collections.addAll(categoriesSet, categories);
+ return categoriesSet;
+ }
+
+ /**
+ * Defines and returns map of categories for System metrics.
+ *
+ * @return map with categories for system metrics and display flag set to true
+ */
+ private Map<String, Boolean> getSystemMetricsCategories() {
+ Map<String, Boolean> categories = new HashMap<>();
+ categories.put("cluster", true);
+ categories.put("cache", true);
+ categories.put("diskstore", true);
+ categories.put("query", true);
+ return categories;
+ }
+
+ /**
+ * Defines and returns map of categories for Region Metrics
+ *
+ * @return map with categories for region metrics and display flag set to true
+ */
+ private Map<String, Boolean> getRegionMetricsCategories() {
+ Map<String, Boolean> categories = new HashMap<>();
+
+ categories.put("region", true);
+ categories.put("partition", true);
+ categories.put("diskstore", true);
+ categories.put("callback", true);
+ categories.put("gatewayreceiver", true);
+ categories.put("distribution", true);
+ categories.put("query", true);
+ categories.put("eviction", true);
+ return categories;
+ }
+
+ /**
+ * Defines and returns map of categories for system-wide region metrics
+ *
+ * @return map with categories for system wide region metrics and display flag set to true
+ */
+ private Map<String, Boolean> getSystemRegionMetricsCategories() {
+ Map<String, Boolean> categories = getRegionMetricsCategories();
+ categories.put("cluster", true);
+ return categories;
+ }
+
+ /**
+ * Defines and returns map of categories for member metrics
+ *
+ * @return map with categories for member metrics and display flag set to true
+ */
+ private Map<String, Boolean> getMemberMetricsCategories() {
+ Map<String, Boolean> categories = new HashMap<>();
+ categories.put("member", true);
+ categories.put("jvm", true);
+ categories.put("region", true);
+ categories.put("serialization", true);
+ categories.put("communication", true);
+ categories.put("function", true);
+ categories.put("transaction", true);
+ categories.put("diskstore", true);
+ categories.put("lock", true);
+ categories.put("eviction", true);
+ categories.put("distribution", true);
+ categories.put("offheap", true);
+ return categories;
+ }
+
+ /**
+ * Converts an array of strings to a String delimited by a new line character for display purposes
+ *
+ * @return a String delimited by a new line character for display purposes
+ */
+ private String formatNames(String[] names, int startIndex) {
+ StringBuilder sb = new StringBuilder();
+
+ if (names != null) {
+ for (String name : names) {
+ sb.append(name.substring(startIndex));
+ sb.append('\n');
+ }
+ }
+ return sb.toString();
+ }
+
+ private void writeToTableAndCsv(TabularResultData metricsTable, String type, String metricName,
+ String metricValue[], StringBuilder csvBuilder) {
+ if (metricValue != null) {
+ for (int i = 0; i < metricValue.length; i++) {
+ if (i == 0) {
+ writeToTableAndCsv(metricsTable, type, metricName, metricValue[i], csvBuilder);
+ } else {
+ writeToTableAndCsv(metricsTable, "", "", metricValue[i], csvBuilder);
+ }
+ }
+ }
+ }
+
+ /**
+ * Writes to a TabularResultData and also appends a CSV string to a String builder
+ */
+ private void writeToTableAndCsv(TabularResultData metricsTable, String type, String metricName,
+ String metricValue, StringBuilder csvBuilder) {
+ metricsTable.accumulate(CliStrings.SHOW_METRICS__TYPE__HEADER, type);
+ metricsTable.accumulate(CliStrings.SHOW_METRICS__METRIC__HEADER, metricName);
+ metricsTable.accumulate(CliStrings.SHOW_METRICS__VALUE__HEADER, metricValue);
+
+ if (csvBuilder != null) {
+ csvBuilder.append(type);
+ csvBuilder.append(',');
+ csvBuilder.append(metricName);
+ csvBuilder.append(',');
+ csvBuilder.append(metricValue);
+ csvBuilder.append('\n');
+ }
+ }
+
+ @CliCommand(value = CliStrings.CHANGE_LOGLEVEL, help = CliStrings.CHANGE_LOGLEVEL__HELP)
+ @CliMetaData(relatedTopic = {CliStrings.TOPIC_LOGS},
+ interceptor = "org.apache.geode.management.internal.cli.commands.MiscellaneousCommands$ChangeLogLevelInterceptor")
+ @ResourceOperation(resource = Resource.CLUSTER, operation = Operation.WRITE)
+ public Result changeLogLevel(
+ @CliOption(key = {CliStrings.MEMBER, CliStrings.MEMBERS},
+ help = CliStrings.CHANGE_LOGLEVEL__MEMBER__HELP) String[] memberIds,
+ @CliOption(key = {CliStrings.GROUP, CliStrings.GROUPS}, unspecifiedDefaultValue = "",
+ help = CliStrings.CHANGE_LOGLEVEL__GROUPS__HELP) String[] grps,
+ @CliOption(key = CliStrings.CHANGE_LOGLEVEL__LOGLEVEL,
+ optionContext = ConverterHint.LOG_LEVEL, mandatory = true, unspecifiedDefaultValue = "",
+ help = CliStrings.CHANGE_LOGLEVEL__LOGLEVEL__HELP) String logLevel) {
+ try {
+ if ((memberIds == null || memberIds.length == 0) && (grps == null || grps.length == 0)) {
+ return ResultBuilder
+ .createUserErrorResult(CliStrings.CHANGE_LOGLEVEL__MSG__SPECIFY_GRP_OR_MEMBER);
+ }
+
+ InternalCache cache = GemFireCacheImpl.getInstance();
+ LogWriter logger = cache.getLogger();
+
+ Set<DistributedMember> dsMembers = new HashSet<>();
+ Set<DistributedMember> ds = CliUtil.getAllMembers(cache);
+
+ if (grps != null && grps.length > 0) {
+ for (String grp : grps) {
+ dsMembers.addAll(cache.getDistributedSystem().getGroupMembers(grp));
+ }
+ }
+
+ if (memberIds != null && memberIds.length > 0) {
+ for (String member : memberIds) {
+ for (DistributedMember mem : ds) {
+ if (mem.getName() != null
+ && (mem.getName().equals(member) || mem.getId().equals(member))) {
+ dsMembers.add(mem);
+ break;
+ }
+ }
+ }
+ }
+
+ if (dsMembers.size() == 0) {
+ return ResultBuilder.createGemFireErrorResult(CliStrings.CHANGE_LOGLEVEL__MSG_NO_MEMBERS);
+ }
+
+ Function logFunction = new ChangeLogLevelFunction();
+ FunctionService.registerFunction(logFunction);
+ Object[] functionArgs = new Object[1];
+ functionArgs[0] = logLevel;
+
+ CompositeResultData compositeResultData = ResultBuilder.createCompositeResultData();
+ SectionResultData section = compositeResultData.addSection("section");
+ TabularResultData resultTable = section.addTable("ChangeLogLevel");
+ resultTable = resultTable.setHeader("Summary");
+
+ Execution execution = FunctionService.onMembers(dsMembers).setArguments(functionArgs);
+ if (execution == null) {
+ return ResultBuilder.createUserErrorResult(CliStrings.CHANGE_LOGLEVEL__MSG__CANNOT_EXECUTE);
+ }
+ List<?> resultList = (List<?>) execution.execute(logFunction).getResult();
+
+ for (Object object : resultList) {
+ try {
+ if (object instanceof Throwable) {
+ logger.warning(
+ "Exception in ChangeLogLevelFunction " + ((Throwable) object).getMessage(),
+ ((Throwable) object));
+ continue;
+ }
+
+ if (object != null) {
+ Map<String, String> resultMap = (Map<String, String>) object;
+ Entry<String, String> entry = resultMap.entrySet().iterator().next();
+
+ if (entry.getValue().contains("ChangeLogLevelFunction exception")) {
+ resultTable.accumulate(CliStrings.CHANGE_LOGLEVEL__COLUMN_MEMBER, entry.getKey());
+ resultTable.accumulate(CliStrings.CHANGE_LOGLEVEL__COLUMN_STATUS, "false");
+ } else {
+ resultTable.accumulate(CliStrings.CHANGE_LOGLEVEL__COLUMN_MEMBER, entry.getKey());
+ resultTable.accumulate(CliStrings.CHANGE_LOGLEVEL__COLUMN_STATUS, "true");
+ }
+
+ }
+ } catch (Exception ex) {
+ LogWrapper.getInstance().warning("change log level command exception " + ex);
+ }
+ }
+
+ Result result = ResultBuilder.buildResult(compositeResultData);
+ logger.info("change log-level command result=" + result);
+ return result;
+ } catch (Exception ex) {
+ GemFireCacheImpl.getInstance().getLogger().error("GFSH Changeloglevel exception: " + ex);
+ return ResultBuilder.createUserErrorResult(ex.getMessage());
+ }
+ }
+
+ public static class ChangeLogLevelInterceptor extends AbstractCliAroundInterceptor {
+ @Override
+ public Result preExecution(GfshParseResult parseResult) {
+ Map<String, String> arguments = parseResult.getParamValueStrings();
+ // validate log level
+ String logLevel = arguments.get("loglevel");
+ if (StringUtils.isBlank(logLevel) || LogLevel.getLevel(logLevel) == null) {
+ return ResultBuilder.createUserErrorResult("Invalid log level: " + logLevel);
+ }
+
+ return ResultBuilder.createInfoResult("");
+ }
+ }
+
+ private Set<String> getSetDifference(Set<String> set1, Set<String> set2) {
+ Set<String> setDifference = new HashSet<>();
+ for (String element : set1) {
+ if (!(set2.contains(element.toLowerCase()))) {
+ setDifference.add(element);
+ }
+ }
+ return setDifference;
+ }
+}