You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by kh...@apache.org on 2017/05/23 16:48:00 UTC
[2/3] geode git commit: GEODE-2420: Enable export logs size
estimation and user warning
GEODE-2420: Enable export logs size estimation and user warning
Adds 'export logs' option, --file-limit-size, to allow user to set
maximun size of the epxorted logs zip file.
When size checking is enabled (file-limit-size > 0) then the check
will also prevent filling up the disk on each member while consolidating
and filtering the logs.
Project: http://git-wip-us.apache.org/repos/asf/geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/3ce3437d
Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/3ce3437d
Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/3ce3437d
Branch: refs/heads/develop
Commit: 3ce3437ddaf9c3614b11f066011ed33664259e9f
Parents: 18db4bf
Author: Ken Howe <kh...@pivotal.io>
Authored: Fri May 5 14:04:11 2017 -0700
Committer: Ken Howe <kh...@pivotal.io>
Committed: Tue May 23 08:16:17 2017 -0700
----------------------------------------------------------------------
.../membership/InternalDistributedMember.java | 22 +-
.../cli/commands/ExportLogsCommand.java | 277 ++++++++++++-------
.../cli/functions/SizeExportLogsFunction.java | 4 +-
.../management/internal/cli/util/LogSizer.java | 6 +-
.../cli/commands/ExportLogsCommandTest.java | 225 ++++++++++++++-
.../cli/commands/ExportLogsDUnitTest.java | 7 +
.../commands/ExportLogsFileSizeLimitTest.java | 2 +-
.../cli/commands/ExportLogsTestSuite.java | 10 +-
.../SizeExportLogsFunctionCacheTest.java | 12 +-
9 files changed, 436 insertions(+), 129 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/geode/blob/3ce3437d/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/InternalDistributedMember.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/InternalDistributedMember.java b/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/InternalDistributedMember.java
index 7170f20..6982d31 100755
--- a/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/InternalDistributedMember.java
+++ b/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/InternalDistributedMember.java
@@ -190,8 +190,8 @@ public class InternalDistributedMember implements DistributedMember, Externaliza
* string).
* <p>
*
- * <b> [bruce]THIS METHOD IS FOR TESTING ONLY. DO NOT USE IT TO CREATE IDs FOR USE IN THE PRODUCT.
- * IT DOES NOT PROPERLY INITIALIZE ATTRIBUTES NEEDED FOR P2P FUNCTIONALITY. </b>
+ * <b> THIS METHOD IS FOR TESTING ONLY. DO NOT USE IT TO CREATE IDs FOR USE IN THE PRODUCT. IT
+ * DOES NOT PROPERLY INITIALIZE ATTRIBUTES NEEDED FOR P2P FUNCTIONALITY. </b>
*
*
* @param i the hostname, must be for the current host
@@ -228,8 +228,8 @@ public class InternalDistributedMember implements DistributedMember, Externaliza
* string).
* <p>
*
- * <b> [bruce]THIS METHOD IS FOR TESTING ONLY. DO NOT USE IT TO CREATE IDs FOR USE IN THE PRODUCT.
- * IT DOES NOT PROPERLY INITIALIZE ATTRIBUTES NEEDED FOR P2P FUNCTIONALITY. </b>
+ * <b> THIS METHOD IS FOR TESTING ONLY. DO NOT USE IT TO CREATE IDs FOR USE IN THE PRODUCT. IT
+ * DOES NOT PROPERLY INITIALIZE ATTRIBUTES NEEDED FOR P2P FUNCTIONALITY. </b>
*
*
* @param i the hostname, must be for the current host
@@ -264,8 +264,8 @@ public class InternalDistributedMember implements DistributedMember, Externaliza
* LonerDistributionManager.
* <p>
*
- * < b> [bruce]DO NOT USE THIS METHOD TO CREATE ANYTHING OTHER THAN A LONER ID WITHOUT TALKING TO
- * ME FIRST. IT DOES NOT PROPERLY INITIALIZE THE ID. </b>
+ * < b> DO NOT USE THIS METHOD TO CREATE ANYTHING OTHER THAN A LONER ID. IT DOES NOT PROPERLY
+ * INITIALIZE THE ID. </b>
*
* @param host the hostname, must be for the current host
* @param p the membership listening port
@@ -298,8 +298,8 @@ public class InternalDistributedMember implements DistributedMember, Externaliza
* address).
* <p>
*
- * <b> [bruce]THIS METHOD IS FOR TESTING ONLY. DO NOT USE IT TO CREATE IDs FOR USE IN THE PRODUCT.
- * IT DOES NOT PROPERLY INITIALIZE ATTRIBUTES NEEDED FOR P2P FUNCTIONALITY. </b>
+ * <b> THIS METHOD IS FOR TESTING ONLY. DO NOT USE IT TO CREATE IDs FOR USE IN THE PRODUCT. IT
+ * DOES NOT PROPERLY INITIALIZE ATTRIBUTES NEEDED FOR P2P FUNCTIONALITY. </b>
*
*
* @param i the hostname, must be for the current host
@@ -314,8 +314,8 @@ public class InternalDistributedMember implements DistributedMember, Externaliza
* Create a InternalDistributedMember as defined by the given address.
* <p>
*
- * <b> [bruce]THIS METHOD IS FOR TESTING ONLY. DO NOT USE IT TO CREATE IDs FOR USE IN THE PRODUCT.
- * IT DOES NOT PROPERLY INITIALIZE ATTRIBUTES NEEDED FOR P2P FUNCTIONALITY. </b>
+ * <b> THIS METHOD IS FOR TESTING ONLY. DO NOT USE IT TO CREATE IDs FOR USE IN THE PRODUCT. IT
+ * DOES NOT PROPERLY INITIALIZE ATTRIBUTES NEEDED FOR P2P FUNCTIONALITY. </b>
*
* @param addr address of the server
* @param p the listening port of the server
@@ -911,7 +911,7 @@ public class InternalDistributedMember implements DistributedMember, Externaliza
public void toDataPre_GFE_7_1_0_0(DataOutput out) throws IOException {
Assert.assertTrue(netMbr.getVmKind() > 0);
- // [bruce] disabled to allow post-connect setting of the port for loner systems
+ // disabled to allow post-connect setting of the port for loner systems
// Assert.assertTrue(getPort() > 0);
// if (this.getPort() == 0) {
// InternalDistributedSystem.getLoggerI18n().warning(LocalizedStrings.DEBUG,
http://git-wip-us.apache.org/repos/asf/geode/blob/3ce3437d/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommand.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommand.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommand.java
index 20ec1f5..af681da 100644
--- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommand.java
+++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommand.java
@@ -34,9 +34,11 @@ import org.springframework.shell.core.annotation.CliOption;
import org.apache.geode.cache.CacheFactory;
import org.apache.geode.cache.Region;
+import org.apache.geode.cache.execute.ResultCollector;
import org.apache.geode.distributed.DistributedMember;
import org.apache.geode.internal.cache.InternalCache;
import org.apache.geode.internal.logging.LogService;
+import org.apache.geode.management.ManagementException;
import org.apache.geode.management.cli.CliMetaData;
import org.apache.geode.management.cli.ConverterHint;
import org.apache.geode.management.cli.Result;
@@ -47,6 +49,7 @@ import org.apache.geode.management.internal.cli.functions.SizeExportLogsFunction
import org.apache.geode.management.internal.cli.i18n.CliStrings;
import org.apache.geode.management.internal.cli.result.ResultBuilder;
import org.apache.geode.management.internal.cli.util.ExportLogsCacheWriter;
+import org.apache.geode.management.internal.cli.util.BytesToString;
import org.apache.geode.management.internal.configuration.utils.ZipUtils;
import org.apache.geode.management.internal.security.ResourceOperation;
import org.apache.geode.security.ResourcePermission;
@@ -60,7 +63,7 @@ public class ExportLogsCommand implements CommandMarker {
public final static String DEFAULT_EXPORT_LOG_LEVEL = "ALL";
- private static final Pattern DISK_SPACE_LIMIT_PATTERN = Pattern.compile("(\\d+)([mgtMGT]?)");
+ private static final Pattern DISK_SPACE_LIMIT_PATTERN = Pattern.compile("(\\d+)([kmgtKMGT]?)");
private InternalCache getCache() {
return (InternalCache) CacheFactory.getAnyInstance();
@@ -102,104 +105,130 @@ public class ExportLogsCommand implements CommandMarker {
help = CliStrings.EXPORT_LOGS__LOGSONLY__HELP) boolean logsOnly,
@CliOption(key = CliStrings.EXPORT_LOGS__STATSONLY, unspecifiedDefaultValue = "false",
specifiedDefaultValue = "true",
- help = CliStrings.EXPORT_LOGS__STATSONLY__HELP) boolean statsOnly) {
- // @CliOption(key = CliStrings.EXPORT_LOGS__FILESIZELIMIT,
- // unspecifiedDefaultValue = CliStrings.EXPORT_LOGS__FILESIZELIMIT__UNSPECIFIED_DEFAULT,
- // specifiedDefaultValue = CliStrings.EXPORT_LOGS__FILESIZELIMIT__SPECIFIED_DEFAULT,
- // help = CliStrings.EXPORT_LOGS__FILESIZELIMIT__HELP) String fileSizeLimit) {
- Result result = null;
+ help = CliStrings.EXPORT_LOGS__STATSONLY__HELP) boolean statsOnly,
+ @CliOption(key = CliStrings.EXPORT_LOGS__FILESIZELIMIT,
+ unspecifiedDefaultValue = CliStrings.EXPORT_LOGS__FILESIZELIMIT__UNSPECIFIED_DEFAULT,
+ specifiedDefaultValue = CliStrings.EXPORT_LOGS__FILESIZELIMIT__SPECIFIED_DEFAULT,
+ help = CliStrings.EXPORT_LOGS__FILESIZELIMIT__HELP) String fileSizeLimit) {
+
+ long totalEstimatedExportSize = 0;
+ Result result;
InternalCache cache = getCache();
try {
- Set<DistributedMember> targetMembers =
- CliUtil.findMembersIncludingLocators(groups, memberIds);
+ Set<DistributedMember> targetMembers = getMembers(groups, memberIds);
if (targetMembers.isEmpty()) {
return ResultBuilder.createUserErrorResult(CliStrings.NO_MEMBERS_FOUND_MESSAGE);
}
- if (false) {
- // TODO: get estimated size of exported logs from all servers first
- Map<String, Integer> fileSizesFromMembers = new HashMap<>();
+ if (parseFileSizeLimit(fileSizeLimit) > 0) {
+ // Get estimated size of exported logs from all servers before exporting anything
for (DistributedMember server : targetMembers) {
SizeExportLogsFunction.Args args = new SizeExportLogsFunction.Args(start, end, logLevel,
onlyLogLevel, logsOnly, statsOnly);
- List<Object> results = (List<Object>) CliUtil
- .executeFunction(new SizeExportLogsFunction(), args, server).getResult();
+ List<Object> results = (List<Object>) estimateLogSize(args, server).getResult();
long estimatedSize = 0;
long diskAvailable = 0;
long diskSize = 0;
- List<?> res = (List<?>) results.get(0);
- if (res.get(0) instanceof ExportedLogsSizeInfo) {
- ExportedLogsSizeInfo sizeInfo = (ExportedLogsSizeInfo) res.get(0);
- estimatedSize = sizeInfo.getLogsSize();
- diskAvailable = sizeInfo.getDiskAvailable();
- diskSize = sizeInfo.getDiskSize();
- } else {
- estimatedSize = 0;
+ if (!results.isEmpty()) {
+ List<?> res = (List<?>) results.get(0);
+ if (res.get(0) instanceof ExportedLogsSizeInfo) {
+ ExportedLogsSizeInfo sizeInfo = (ExportedLogsSizeInfo) res.get(0);
+ estimatedSize = sizeInfo.getLogsSize();
+ diskAvailable = sizeInfo.getDiskAvailable();
+ diskSize = sizeInfo.getDiskSize();
+ }
+ }
+ logger.info("Received estimated export size from member {}: {}", server.getId(),
+ estimatedSize);
+ totalEstimatedExportSize += estimatedSize;
+
+ // If export size checking is enabled, then estimated size on each member shouldn't exceed
+ // the available disk on that member
+ try {
+ isSizeCheckEnabledAndWithinDiskSpaceOfMember(server.getName(),
+ parseFileSizeLimit(fileSizeLimit), estimatedSize, diskAvailable, diskSize);
+ } catch (ManagementException e) {
+ return ResultBuilder.createUserErrorResult(e.getMessage());
}
-
-
- logger.info("Received file size from member {}: {}", server.getId(), estimatedSize);
}
- // TODO: Check log size limits on the locator
+ // The sum of the estimated export sizes from each member should not exceed the
+ // disk availble on the locator
+ try {
+ isSizeCheckEnabledAndWithinDiskSpaceOfMember("locator", parseFileSizeLimit(fileSizeLimit),
+ totalEstimatedExportSize, getLocalDiskAvailable(), getLocalDiskSize());
+ } catch (ManagementException e) {
+ return ResultBuilder.createUserErrorResult(e.getMessage());
+ }
}
- // get zipped files from all servers next
- Map<String, Path> zipFilesFromMembers = new HashMap<>();
- for (DistributedMember server : targetMembers) {
- Region region = ExportLogsFunction.createOrGetExistingExportLogsRegion(true, cache);
+ if (testhookSkipExports()) {
+ result = ResultBuilder.createInfoResult("Estimated size of exported logs is "
+ + new BytesToString().of(totalEstimatedExportSize));
+ } else {
+ // get zipped files from all servers next
+ Map<String, Path> zipFilesFromMembers = new HashMap<>();
+ for (DistributedMember server : targetMembers) {
+ Region region = ExportLogsFunction.createOrGetExistingExportLogsRegion(true, cache);
- ExportLogsCacheWriter cacheWriter =
- (ExportLogsCacheWriter) region.getAttributes().getCacheWriter();
+ ExportLogsCacheWriter cacheWriter =
+ (ExportLogsCacheWriter) region.getAttributes().getCacheWriter();
- cacheWriter.startFile(server.getName());
+ cacheWriter.startFile(server.getName());
- CliUtil.executeFunction(new ExportLogsFunction(),
- new ExportLogsFunction.Args(start, end, logLevel, onlyLogLevel, logsOnly, statsOnly),
- server).getResult();
- Path zipFile = cacheWriter.endFile();
- ExportLogsFunction.destroyExportLogsRegion(cache);
+ CliUtil.executeFunction(new ExportLogsFunction(),
+ new ExportLogsFunction.Args(start, end, logLevel, onlyLogLevel, logsOnly, statsOnly),
+ server).getResult();
+ Path zipFile = cacheWriter.endFile();
+ ExportLogsFunction.destroyExportLogsRegion(cache);
- // only put the zipfile in the map if it is not null
- if (zipFile != null) {
- logger.info("Received zip file from member {}: {}", server.getId(), zipFile);
- zipFilesFromMembers.put(server.getId(), zipFile);
+ // only put the zipfile in the map if it is not null
+ if (zipFile != null) {
+ logger.info("Received zip file from member {}: {}", server.getId(), zipFile);
+ zipFilesFromMembers.put(server.getId(), zipFile);
+ }
}
- }
- if (zipFilesFromMembers.isEmpty()) {
- return ResultBuilder.createUserErrorResult("No files to be exported.");
- }
+ if (zipFilesFromMembers.isEmpty()) {
+ return ResultBuilder.createUserErrorResult("No files to be exported.");
+ }
- Path tempDir = Files.createTempDirectory("exportedLogs");
- // make sure the directory is created, so that even if there is no files unzipped to this dir,
- // we can still zip it and send an empty zip file back to the client
- Path exportedLogsDir = tempDir.resolve("exportedLogs");
- FileUtils.forceMkdir(exportedLogsDir.toFile());
-
- for (Path zipFile : zipFilesFromMembers.values()) {
- Path unzippedMemberDir =
- exportedLogsDir.resolve(zipFile.getFileName().toString().replace(".zip", ""));
- ZipUtils.unzip(zipFile.toAbsolutePath().toString(), unzippedMemberDir.toString());
- FileUtils.deleteQuietly(zipFile.toFile());
- }
+ Path tempDir = Files.createTempDirectory("exportedLogs");
+ // make sure the directory is created, so that even if there is no files unzipped to this
+ // dir, we can still zip it and send an empty zip file back to the client
+ Path exportedLogsDir = tempDir.resolve("exportedLogs");
+ FileUtils.forceMkdir(exportedLogsDir.toFile());
+
+ for (Path zipFile : zipFilesFromMembers.values()) {
+ Path unzippedMemberDir =
+ exportedLogsDir.resolve(zipFile.getFileName().toString().replace(".zip", ""));
+ ZipUtils.unzip(zipFile.toAbsolutePath().toString(), unzippedMemberDir.toString());
+ FileUtils.deleteQuietly(zipFile.toFile());
+ }
- Path dirPath;
- if (StringUtils.isBlank(dirName)) {
- dirPath = Paths.get(System.getProperty("user.dir"));
- } else {
- dirPath = Paths.get(dirName);
+ Path dirPath;
+ if (StringUtils.isBlank(dirName)) {
+ dirPath = Paths.get(System.getProperty("user.dir"));
+ } else {
+ dirPath = Paths.get(dirName);
+ }
+ Path exportedLogsZipFile =
+ dirPath.resolve("exportedLogs_" + System.currentTimeMillis() + ".zip").toAbsolutePath();
+
+ logger.info("Zipping into: " + exportedLogsZipFile.toString());
+ ZipUtils.zipDirectory(exportedLogsDir, exportedLogsZipFile);
+ try {
+ isFileSizeCheckEnabledAndWithinLimit(parseFileSizeLimit(fileSizeLimit),
+ exportedLogsZipFile.toFile());
+ } catch (ManagementException e) {
+ return ResultBuilder.createUserErrorResult(e.getMessage());
+ } finally {
+ FileUtils.deleteDirectory(tempDir.toFile());
+ }
+ result = ResultBuilder.createInfoResult(exportedLogsZipFile.toString());
}
- Path exportedLogsZipFile =
- dirPath.resolve("exportedLogs_" + System.currentTimeMillis() + ".zip").toAbsolutePath();
-
- logger.info("Zipping into: " + exportedLogsZipFile.toString());
- ZipUtils.zipDirectory(exportedLogsDir, exportedLogsZipFile);
- FileUtils.deleteDirectory(tempDir.toFile());
-
- result = ResultBuilder.createInfoResult(exportedLogsZipFile.toString());
} catch (Exception ex) {
logger.error(ex.getMessage(), ex);
result = ResultBuilder.createGemFireErrorResult(ex.getMessage());
@@ -211,43 +240,99 @@ public class ExportLogsCommand implements CommandMarker {
}
/**
+ * Test hook for unit testing. To limit scope of test to only estimate size of exports (i.e. skip
+ * the filtering and exporting logs & stats from cluster members), stub this method to return true
+ * to skip exporting.
+ */
+ boolean testhookSkipExports() {
+ return false;
+ }
+
+ /**
+ * Wrapper to enable stubbing of static method call for unit testing
+ */
+ Set<DistributedMember> getMembers(String[] groups, String[] memberIds) {
+ return CliUtil.findMembersIncludingLocators(groups, memberIds);
+ }
+
+ /**
+ * Wrapper to enable stubbing of static method call for unit testing
+ */
+ ResultCollector estimateLogSize(SizeExportLogsFunction.Args args, DistributedMember member) {
+ return CliUtil.executeFunction(new SizeExportLogsFunction(), args, member);
+ }
+
+ /**
+ * Wrapper to enable stubbing of static method call for unit testing
+ */
+ long getLocalDiskSize() {
+ return FileUtils.getUserDirectory().getTotalSpace();
+ }
+
+ /**
+ * Wrapper to enable stubbing of static method call for unit testing
+ */
+ long getLocalDiskAvailable() {
+ return FileUtils.getUserDirectory().getUsableSpace();
+ }
+
+ /**
* Returns file size limit in bytes
*/
- int parseFileSizeLimit(String fileSizeLimit) {
+ private long parseFileSizeLimit(String fileSizeLimit) {
if (StringUtils.isEmpty(fileSizeLimit)) {
return 0;
}
- int sizeLimit = parseSize(fileSizeLimit);
- int byteMultiplier = parseByteMultiplier(fileSizeLimit);
+ long sizeLimit = parseSize(fileSizeLimit);
+ long byteMultiplier = parseByteMultiplier(fileSizeLimit);
return sizeLimit * byteMultiplier;
}
/**
- * Throws IllegalArgumentException if file size is over fileSizeLimitBytes
+ * Throws ManagementException if file size is over fileSizeLimit bytes
+ *
+ * @return false == limit is zero (checking disabled)<br>
+ * true == file size is less than limit<br>
+ * exception == file size is over limit
*/
- void checkOverDiskSpaceThreshold(int fileSizeLimitBytes, File file) {
- // TODO:GEODE-2420: warn user if exportedLogsZipFile size > threshold
- if (FileUtils.sizeOf(file) > fileSizeLimitBytes) {
- throw new IllegalArgumentException("TOO BIG"); // FileTooBigException
+ boolean isFileSizeCheckEnabledAndWithinLimit(long fileSizeLimitBytes, File file) {
+ if (fileSizeLimitBytes < 1) {
+ // size checks disabled
+ return false;
}
+ if (FileUtils.sizeOf(file) < fileSizeLimitBytes) {
+ return true;
+ }
+ StringBuilder sb = new StringBuilder();
+ sb.append("Exported logs zip file size = ").append(FileUtils.sizeOf(file)).append(", ")
+ .append(CliStrings.EXPORT_LOGS__FILESIZELIMIT).append(" = ").append(fileSizeLimitBytes)
+ .append(". To disable exported logs file size check use option \"--file-size-limit=0\".");
+ throw new ManagementException(sb.toString()); // FileTooBigException
}
/**
- * Throws IllegalArgumentException if file size is over fileSizeLimitBytes false == limit is zero
- * true == file size is less than limit exception == file size is over limit
+ * Throws ManagementException if export file size checking is enabled and the space required on a
+ * cluster member to filter and zip up files to be exported exceeds the disk space available
*/
- boolean isFileSizeCheckEnabledAndWithinLimit(int fileSizeLimitBytes, File file) {
- // TODO:GEODE-2420: warn user if exportedLogsZipFile size > threshold
+ boolean isSizeCheckEnabledAndWithinDiskSpaceOfMember(String memberName, long fileSizeLimitBytes,
+ long estimatedSize, long diskAvailable, long diskSize) {
+ // TODO:GEODE-2420: warn user if exportedLogs filtering will exceed disk available
if (fileSizeLimitBytes < 1) {
+ // size checks disabled
return false;
}
- if (FileUtils.sizeOf(file) < fileSizeLimitBytes) {
- return true;
+ StringBuilder sb = new StringBuilder();
+ BytesToString bytesToString = new BytesToString();
+ if (estimatedSize > diskAvailable) {
+ sb.append("Estimated disk space required (").append(bytesToString.of(estimatedSize))
+ .append(") to consolidate logs on member ").append(memberName)
+ .append(" will exceed available disk space (").append(bytesToString.of(diskAvailable))
+ .append(")");
+ throw new ManagementException(sb.toString()); // FileTooBigException
}
- throw new IllegalArgumentException("TOO BIG: fileSizeLimit = " + fileSizeLimitBytes
- + ", fileSize = " + FileUtils.sizeOf(file)); // FileTooBigException
+ return true;
}
static int parseSize(String diskSpaceLimit) {
@@ -259,24 +344,26 @@ public class ExportLogsCommand implements CommandMarker {
}
}
- static int parseByteMultiplier(String diskSpaceLimit) {
+ static long parseByteMultiplier(String diskSpaceLimit) {
Matcher matcher = DISK_SPACE_LIMIT_PATTERN.matcher(diskSpaceLimit);
if (!matcher.matches()) {
throw new IllegalArgumentException();
}
switch (matcher.group(2).toLowerCase()) {
+ case "k":
+ return KILOBYTE;
case "t":
- return (int) TERABYTE;
+ return TERABYTE;
case "g":
- return (int) GIGABYTE;
+ return GIGABYTE;
case "m":
default:
- return (int) MEGABYTE;
+ return MEGABYTE;
}
}
- static final int MEGABYTE = (int) Math.pow(1024, 2);
- static final int GIGABYTE = (int) Math.pow(1024, 3);
- static final int TERABYTE = (int) Math.pow(1024, 4);
-
+ static final long KILOBYTE = 1024L;
+ static final long MEGABYTE = KILOBYTE * 1024;
+ static final long GIGABYTE = MEGABYTE * 1024;
+ static final long TERABYTE = GIGABYTE * 1024;
}
http://git-wip-us.apache.org/repos/asf/geode/blob/3ce3437d/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java
index 8d20dc0..3d98fe9 100644
--- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java
+++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java
@@ -59,7 +59,9 @@ public class SizeExportLogsFunction extends ExportLogsFunction implements Functi
long estimateLogFileSize(final DistributedMember member, final File logFile,
final File statArchive, final Args args) throws ParseException, IOException {
- LOGGER.info("SizeExportLogsFunction started for member {}", member);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("SizeExportLogsFunction started for member {}", member);
+ }
File baseLogFile = null;
File baseStatsFile = null;
http://git-wip-us.apache.org/repos/asf/geode/blob/3ce3437d/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogSizer.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogSizer.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogSizer.java
index 0a799f6..f1adae6 100644
--- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogSizer.java
+++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogSizer.java
@@ -42,8 +42,10 @@ public class LogSizer {
/**
* @param logFilter the filter that's used to check if we need to accept the file or the logLine
- * @param baseLogFile if not null, we will export the logs in that directory
- * @param baseStatsFile if not null, we will export stats in that directory
+ * @param baseLogFile if not null, we will estimate the additional disk space required to filter
+ * and export the logs in that directory
+ * @param baseStatsFile if not null, we will estimate the additional disk space required to filter
+ * and export the stats in that directory
*/
public LogSizer(LogFilter logFilter, File baseLogFile, File baseStatsFile) throws ParseException {
assert logFilter != null;
http://git-wip-us.apache.org/repos/asf/geode/blob/3ce3437d/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommandTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommandTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommandTest.java
index a02c07f..2b7839a 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommandTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommandTest.java
@@ -17,25 +17,37 @@ package org.apache.geode.management.internal.cli.commands;
import static org.apache.geode.management.internal.cli.commands.ExportLogsCommand.*;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
import org.apache.geode.cache.Cache;
import org.apache.geode.cache.execute.Execution;
+import org.apache.geode.cache.execute.FunctionException;
import org.apache.geode.cache.execute.ResultCollector;
import org.apache.geode.distributed.DistributedMember;
+import org.apache.geode.distributed.internal.membership.InternalDistributedMember;
+import org.apache.geode.internal.cache.GemFireCacheImpl;
+import org.apache.geode.management.ManagementException;
+import org.apache.geode.management.cli.Result;
+import org.apache.geode.management.internal.cli.functions.ExportedLogsSizeInfo;
+import org.apache.geode.management.internal.cli.functions.SizeExportLogsFunction;
import org.apache.geode.test.junit.categories.UnitTest;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.mockito.Matchers;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
@Category(UnitTest.class)
public class ExportLogsCommandTest {
- private ExportLogsCommand createExportLogsCommand(final Cache cache,
- final DistributedMember distributedMember, final Execution functionExecutor) {
- return new TestExportLogsCommand(cache, distributedMember, functionExecutor);
- }
-
@Test
public void parseSize_sizeWithUnit_shouldReturnSize() throws Exception {
assertThat(ExportLogsCommand.parseSize("1000m")).isEqualTo(1000);
@@ -52,6 +64,11 @@ public class ExportLogsCommandTest {
}
@Test
+ public void parseByteMultiplier_sizeWith_k_shouldReturnUnit() throws Exception {
+ assertThat(ExportLogsCommand.parseByteMultiplier("1000k")).isEqualTo(KILOBYTE);
+ }
+
+ @Test
public void parseByteMultiplier_sizeWith_m_shouldReturnUnit() throws Exception {
assertThat(ExportLogsCommand.parseByteMultiplier("1000m")).isEqualTo(MEGABYTE);
}
@@ -67,6 +84,11 @@ public class ExportLogsCommandTest {
}
@Test
+ public void parseByteMultiplier_sizeWith_K_shouldReturnUnit() throws Exception {
+ assertThat(ExportLogsCommand.parseByteMultiplier("1000K")).isEqualTo(KILOBYTE);
+ }
+
+ @Test
public void parseByteMultiplier_sizeWith_M_shouldReturnUnit() throws Exception {
assertThat(ExportLogsCommand.parseByteMultiplier("1000M")).isEqualTo(MEGABYTE);
}
@@ -99,17 +121,170 @@ public class ExportLogsCommandTest {
.isInstanceOf(IllegalArgumentException.class);
}
- @Ignore
@Test
- public void sizeFromMember_withinLimits() throws Exception {
+ public void sizeTooBigOnMember_sizeChecksDisabled_returnsFalse() throws Exception {
final Cache mockCache = mock(Cache.class);
final DistributedMember mockDistributedMember = mock(DistributedMember.class);
final Execution mockFuntionExecutor = mock(Execution.class);
- final ResultCollector mockResultCollector = mock(ResultCollector.class);
+ final ExportLogsCommand cmd =
+ createExportLogsCommand(mockCache, mockDistributedMember, mockFuntionExecutor);
+ assertThat(cmd.isSizeCheckEnabledAndWithinDiskSpaceOfMember("clusterMember", 0, MEGABYTE + 1024,
+ MEGABYTE, TERABYTE)).isFalse();
+ }
+ @Test
+ public void sizeOKOnMember_sizeChecksEnabled_returnsTrue() throws Exception {
+ final Cache mockCache = mock(Cache.class);
+ final DistributedMember mockDistributedMember = mock(DistributedMember.class);
+ final Execution mockFuntionExecutor = mock(Execution.class);
final ExportLogsCommand cmd =
createExportLogsCommand(mockCache, mockDistributedMember, mockFuntionExecutor);
- // cmd.exportLogs();
+ assertThat(cmd.isSizeCheckEnabledAndWithinDiskSpaceOfMember("clusterMember", 10 * MEGABYTE,
+ MEGABYTE - 1024, MEGABYTE, TERABYTE)).isTrue();
+ }
+
+ @Test
+ public void sizeTooBigOnMember_sizeChecksEnabled_shouldThrow() throws Exception {
+ final Cache mockCache = mock(Cache.class);
+ final DistributedMember mockDistributedMember = mock(DistributedMember.class);
+ final Execution mockFuntionExecutor = mock(Execution.class);
+ final ExportLogsCommand cmd =
+ createExportLogsCommand(mockCache, mockDistributedMember, mockFuntionExecutor);
+ assertThatThrownBy(() -> cmd.isSizeCheckEnabledAndWithinDiskSpaceOfMember("clusterMember",
+ 10 * MEGABYTE, MEGABYTE + 1024, MEGABYTE, TERABYTE))
+ .isInstanceOf(ManagementException.class);
+ }
+
+ @Test
+ public void sizeFromMember_withinLimits() throws Exception {
+ final GemFireCacheImpl mockCache = mock(GemFireCacheImpl.class);
+ final ExportLogsCommand realCmd = new ExportLogsCommand();
+
+ ExportLogsCommand spyCmd = spy(realCmd);
+ String start = null;
+ String end = null;
+ String logLevel = null;
+ boolean onlyLogLevel = false;
+ boolean logsOnly = false;
+ boolean statsOnly = false;
+
+ InternalDistributedMember member1 = new InternalDistributedMember("member1", 12345);
+ InternalDistributedMember member2 = new InternalDistributedMember("member2", 98765);
+ member1.getNetMember().setName("member1");
+ member2.getNetMember().setName("member2");
+ Set<DistributedMember> testMembers = new HashSet<>();
+ testMembers.add(member1);
+ testMembers.add(member2);
+
+ ResultCollector testResults1 = new CustomCollector();
+ testResults1.addResult(member1,
+ Arrays.asList(new ExportedLogsSizeInfo(MEGABYTE, 2 * MEGABYTE, TERABYTE)));
+ ResultCollector testResults2 = new CustomCollector();
+ testResults2.addResult(member2,
+ Arrays.asList(new ExportedLogsSizeInfo(2 * MEGABYTE, 2 * MEGABYTE, TERABYTE)));
+
+ doReturn(mockCache).when(spyCmd).getCache();
+ doReturn(testMembers).when(spyCmd).getMembers(null, null);
+ doReturn(testResults1).when(spyCmd)
+ .estimateLogSize(Matchers.any(SizeExportLogsFunction.Args.class), eq(member1));
+ doReturn(testResults2).when(spyCmd)
+ .estimateLogSize(Matchers.any(SizeExportLogsFunction.Args.class), eq(member2));
+ doReturn(true).when(spyCmd).testhookSkipExports();
+
+ Result res = spyCmd.exportLogs("working dir", null, null, logLevel, onlyLogLevel, false, start,
+ end, logsOnly, statsOnly, "1g");
+ assertThat(res.getStatus()).isEqualTo(Result.Status.OK);
+ }
+
+ @Test
+ public void sizeFromMember_greaterThanDiskAvailable_shouldReturnErrorResult() throws Exception {
+ final GemFireCacheImpl mockCache = mock(GemFireCacheImpl.class);
+ final ExportLogsCommand realCmd = new ExportLogsCommand();
+ ExportLogsCommand spyCmd = spy(realCmd);
+
+ String start = null;
+ String end = null;
+ String logLevel = null;
+ boolean onlyLogLevel = false;
+ boolean logsOnly = false;
+ boolean statsOnly = false;
+
+ InternalDistributedMember member1 = new InternalDistributedMember("member1", 12345);
+ InternalDistributedMember member2 = new InternalDistributedMember("member2", 98765);
+ member1.getNetMember().setName("member1");
+ member2.getNetMember().setName("member2");
+ Set<DistributedMember> testMembers = new HashSet<>();
+ testMembers.add(member1);
+ testMembers.add(member2);
+
+ ResultCollector testResults1 = new CustomCollector();
+ testResults1.addResult(member1,
+ Arrays.asList(new ExportedLogsSizeInfo(2 * MEGABYTE, MEGABYTE, TERABYTE)));
+ ResultCollector testResults2 = new CustomCollector();
+ testResults2.addResult(member2,
+ Arrays.asList(new ExportedLogsSizeInfo(2 * MEGABYTE, 2 * MEGABYTE, TERABYTE)));
+
+ doReturn(mockCache).when(spyCmd).getCache();
+ doReturn(testMembers).when(spyCmd).getMembers(null, null);
+ doReturn(testResults1).when(spyCmd)
+ .estimateLogSize(Matchers.any(SizeExportLogsFunction.Args.class), eq(member1));
+ doReturn(testResults2).when(spyCmd)
+ .estimateLogSize(Matchers.any(SizeExportLogsFunction.Args.class), eq(member2));
+ doReturn(true).when(spyCmd).testhookSkipExports();
+
+ Result res = spyCmd.exportLogs("working dir", null, null, logLevel, onlyLogLevel, false, start,
+ end, logsOnly, statsOnly, "1g");
+ assertThat(res.getStatus()).isEqualTo(Result.Status.ERROR);
+ }
+
+ @Test
+ public void sizeFromAllMembers_greaterThanLocalDiskAvailable_shouldReturnErrorResult()
+ throws Exception {
+ final GemFireCacheImpl mockCache = mock(GemFireCacheImpl.class);
+ final ExportLogsCommand realCmd = new ExportLogsCommand();
+ ExportLogsCommand spyCmd = spy(realCmd);
+
+ String start = null;
+ String end = null;
+ String logLevel = null;
+ boolean onlyLogLevel = false;
+ boolean logsOnly = false;
+ boolean statsOnly = false;
+
+ InternalDistributedMember member1 = new InternalDistributedMember("member1", 12345);
+ InternalDistributedMember member2 = new InternalDistributedMember("member2", 98765);
+ member1.getNetMember().setName("member1");
+ member2.getNetMember().setName("member2");
+ Set<DistributedMember> testMembers = new HashSet<>();
+ testMembers.add(member1);
+ testMembers.add(member2);
+
+ ResultCollector testResults1 = new CustomCollector();
+ testResults1.addResult(member1,
+ Arrays.asList(new ExportedLogsSizeInfo(75 * MEGABYTE, GIGABYTE, TERABYTE)));
+ ResultCollector testResults2 = new CustomCollector();
+ testResults2.addResult(member2,
+ Arrays.asList(new ExportedLogsSizeInfo(60 * MEGABYTE, GIGABYTE, TERABYTE)));
+
+ doReturn(mockCache).when(spyCmd).getCache();
+ doReturn(testMembers).when(spyCmd).getMembers(null, null);
+ doReturn(testResults1).when(spyCmd)
+ .estimateLogSize(Matchers.any(SizeExportLogsFunction.Args.class), eq(member1));
+ doReturn(testResults2).when(spyCmd)
+ .estimateLogSize(Matchers.any(SizeExportLogsFunction.Args.class), eq(member2));
+ doReturn(125 * MEGABYTE).when(spyCmd).getLocalDiskAvailable();
+ doReturn(GIGABYTE).when(spyCmd).getLocalDiskSize();
+
+ doReturn(true).when(spyCmd).testhookSkipExports();
+
+ Result res = spyCmd.exportLogs("working dir", null, null, logLevel, onlyLogLevel, false, start,
+ end, logsOnly, statsOnly, "125m");
+ assertThat(res.getStatus()).isEqualTo(Result.Status.ERROR);
+ }
+
+ private ExportLogsCommand createExportLogsCommand(final Cache cache,
+ final DistributedMember distributedMember, final Execution functionExecutor) {
+ return new TestExportLogsCommand(cache, distributedMember, functionExecutor);
}
private static class TestExportLogsCommand extends ExportLogsCommand {
@@ -118,7 +293,7 @@ public class ExportLogsCommandTest {
private final DistributedMember distributedMember;
private final Execution functionExecutor;
- public TestExportLogsCommand(final Cache cache, final DistributedMember distributedMember,
+ TestExportLogsCommand(final Cache cache, final DistributedMember distributedMember,
final Execution functionExecutor) {
assert cache != null;
this.cache = cache;
@@ -126,4 +301,32 @@ public class ExportLogsCommandTest {
this.functionExecutor = functionExecutor;
}
}
+
+ public static class CustomCollector implements ResultCollector<Object, List<Object>> {
+ private ArrayList<Object> results = new ArrayList<>();
+
+ @Override
+ public List<Object> getResult() throws FunctionException {
+ return results;
+ }
+
+ @Override
+ public List<Object> getResult(final long timeout, final TimeUnit unit)
+ throws FunctionException, InterruptedException {
+ return results;
+ }
+
+ @Override
+ public void addResult(final DistributedMember memberID, final Object resultOfSingleExecution) {
+ results.add(resultOfSingleExecution);
+ }
+
+ @Override
+ public void endResults() {}
+
+ @Override
+ public void clearResults() {
+ results.clear();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/geode/blob/3ce3437d/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsDUnitTest.java
index 96ac765..0e0f6aa 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsDUnitTest.java
@@ -28,6 +28,7 @@ import org.apache.geode.cache.Cache;
import org.apache.geode.distributed.ConfigurationProperties;
import org.apache.geode.internal.cache.GemFireCacheImpl;
import org.apache.geode.internal.logging.LogService;
+import org.apache.geode.management.cli.Result;
import org.apache.geode.management.internal.cli.functions.ExportLogsFunction;
import org.apache.geode.management.internal.cli.result.CommandResult;
import org.apache.geode.management.internal.cli.util.CommandStringBuilder;
@@ -182,6 +183,12 @@ public class ExportLogsDUnitTest {
}
@Test
+ public void testExportedZipFileTooBig() throws Exception {
+ CommandResult result = gfshConnector.executeCommand("export logs --file-size-limit=10k");
+ assertThat(result.getStatus()).isEqualTo(Result.Status.ERROR);
+ }
+
+ @Test
public void testExportWithNoFilters() throws Exception {
gfshConnector.executeAndVerifyCommand("export logs --log-level=all");
http://git-wip-us.apache.org/repos/asf/geode/blob/3ce3437d/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsFileSizeLimitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsFileSizeLimitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsFileSizeLimitTest.java
index ec2bcfe..1058053 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsFileSizeLimitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsFileSizeLimitTest.java
@@ -77,7 +77,7 @@ public class ExportLogsFileSizeLimitTest {
assertThat(exportLogsCommand.isFileSizeCheckEnabledAndWithinLimit(0, file)).isFalse();
}
- private void fillUpFile(File file, int sizeInBytes) throws IOException {
+ private void fillUpFile(File file, long sizeInBytes) throws IOException {
PrintWriter writer = new PrintWriter(file, "UTF-8");
while (FileUtils.sizeOf(file) < sizeInBytes) {
writer.println("this is a line of data in the file");
http://git-wip-us.apache.org/repos/asf/geode/blob/3ce3437d/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsTestSuite.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsTestSuite.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsTestSuite.java
index 90a92f3..8f8c532 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsTestSuite.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsTestSuite.java
@@ -14,14 +14,22 @@
*/
package org.apache.geode.management.internal.cli.commands;
+import org.apache.geode.management.internal.cli.functions.SizeExportLogsFunctionCacheTest;
+import org.apache.geode.management.internal.cli.functions.SizeExportLogsFunctionFileTest;
+import org.apache.geode.management.internal.cli.util.LogSizerTest;
+import org.junit.Ignore;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
/**
* All of the JUnit, DUnit and Integration tests for gfsh command export logs.
*/
+
+@Ignore
@Suite.SuiteClasses({ExportLogsCommandTest.class, ExportLogsFileSizeLimitTest.class,
- ExportLogsIntegrationTest.class, ExportLogsDUnitTest.class,})
+ ExportLogsIntegrationTest.class, ExportLogsDUnitTest.class,
+ SizeExportLogsFunctionCacheTest.class, SizeExportLogsFunctionFileTest.class,
+ LogSizerTest.class})
@RunWith(Suite.class)
public class ExportLogsTestSuite {
}
http://git-wip-us.apache.org/repos/asf/geode/blob/3ce3437d/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionCacheTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionCacheTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionCacheTest.java
index d8f2f2d..0e8348d 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionCacheTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionCacheTest.java
@@ -16,8 +16,6 @@ package org.apache.geode.management.internal.cli.functions;
import static org.assertj.core.api.Assertions.*;
import static org.apache.geode.distributed.ConfigurationProperties.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import org.apache.commons.io.FileUtils;
import org.apache.geode.cache.Cache;
@@ -50,9 +48,9 @@ public class SizeExportLogsFunctionCacheTest {
private FunctionContext functionContext;
private File dir;
private DistributedMember member;
- File logFile;
- File statFile;
- String name;
+ private File logFile;
+ private File statFile;
+ private String name;
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
@@ -142,12 +140,12 @@ public class SizeExportLogsFunctionCacheTest {
FunctionContext context = new FunctionContextImpl("functionId", null, resultSender);
new SizeExportLogsFunction().execute(context);
- assertThatThrownBy(() -> resultSender.getResults()).isInstanceOf(NullPointerException.class);
+ assertThatThrownBy(resultSender::getResults).isInstanceOf(NullPointerException.class);
}
private static class TestResultSender implements ResultSender {
- private final List<Object> results = new LinkedList<Object>();
+ private final List<Object> results = new LinkedList<>();
private Throwable t;