You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by kh...@apache.org on 2017/03/21 15:29:06 UTC
geode git commit: 1st pass at checking for log sizes on the servers
Repository: geode
Updated Branches:
refs/heads/feature/GEODE-2420 81bb0e5c1 -> 8215ae0a4
1st pass at checking for log sizes on the servers
Project: http://git-wip-us.apache.org/repos/asf/geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/8215ae0a
Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/8215ae0a
Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/8215ae0a
Branch: refs/heads/feature/GEODE-2420
Commit: 8215ae0a4158ed66283de351671a546238873f41
Parents: 81bb0e5
Author: Ken Howe <kh...@pivotal.io>
Authored: Tue Mar 21 08:27:51 2017 -0700
Committer: Ken Howe <kh...@pivotal.io>
Committed: Tue Mar 21 08:27:51 2017 -0700
----------------------------------------------------------------------
.../cli/commands/ExportLogsCommand.java | 23 +++
.../cli/functions/ExportLogsFunction.java | 1 +
.../cli/functions/SizeExportLogsFunction.java | 182 +++++++++++++++++++
.../SizeExportLogsFunctionIntegrationTest.java | 76 ++++++++
4 files changed, 282 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/geode/blob/8215ae0a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommand.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommand.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommand.java
index 81c8a6c..428dc33 100644
--- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommand.java
+++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommand.java
@@ -26,6 +26,7 @@ import org.apache.geode.management.cli.ConverterHint;
import org.apache.geode.management.cli.Result;
import org.apache.geode.management.internal.cli.CliUtil;
import org.apache.geode.management.internal.cli.functions.ExportLogsFunction;
+import org.apache.geode.management.internal.cli.functions.SizeExportLogsFunction;
import org.apache.geode.management.internal.cli.i18n.CliStrings;
import org.apache.geode.management.internal.cli.result.ResultBuilder;
import org.apache.geode.management.internal.cli.util.ExportLogsCacheWriter;
@@ -42,6 +43,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
@@ -107,6 +109,27 @@ public class ExportLogsCommand implements CommandMarker {
return ResultBuilder.createUserErrorResult(CliStrings.NO_MEMBERS_FOUND_MESSAGE);
}
+ if (true) {
+ // TODO: get unzipped sizes from all servers first
+ Map<String, Integer> fileSizesFromMembers = new HashMap<>();
+ for (DistributedMember server : targetMembers) {
+ SizeExportLogsFunction.Args
+ args =
+ new SizeExportLogsFunction.Args(start, end, logLevel, onlyLogLevel, logsOnly,
+ statsOnly);
+
+ List<Long> results = (List<Long>)CliUtil.executeFunction(new SizeExportLogsFunction(), args, server)
+ .getResult();
+
+ long estimatedSize = results.get(0);
+
+ logger.info("Received file size from member {}: {}", server.getId(), estimatedSize);
+ }
+
+ // TODO: Total all unzipped sizes
+ }
+
+ // get zipped files from all servers next
Map<String, Path> zipFilesFromMembers = new HashMap<>();
for (DistributedMember server : targetMembers) {
Region region = ExportLogsFunction.createOrGetExistingExportLogsRegion(true, cache);
http://git-wip-us.apache.org/repos/asf/geode/blob/8215ae0a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportLogsFunction.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportLogsFunction.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportLogsFunction.java
index 516a9a7..707d88b 100644
--- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportLogsFunction.java
+++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportLogsFunction.java
@@ -75,6 +75,7 @@ public class ExportLogsFunction implements Function, InternalEntity {
Args args = (Args) context.getArguments();
File baseLogFile = null;
File baseStatsFile = null;
+
if (args.isIncludeLogs() && !config.getLogFile().toString().isEmpty()) {
baseLogFile = config.getLogFile().getAbsoluteFile();
}
http://git-wip-us.apache.org/repos/asf/geode/blob/8215ae0a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java
new file mode 100644
index 0000000..4b1d954
--- /dev/null
+++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.management.internal.cli.functions;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.geode.cache.AttributesFactory;
+import org.apache.geode.cache.Cache;
+import org.apache.geode.cache.DataPolicy;
+import org.apache.geode.cache.Region;
+import org.apache.geode.cache.Scope;
+import org.apache.geode.cache.execute.Function;
+import org.apache.geode.cache.execute.FunctionContext;
+import org.apache.geode.distributed.DistributedMember;
+import org.apache.geode.distributed.internal.DistributionConfig;
+import org.apache.geode.internal.InternalEntity;
+import org.apache.geode.internal.cache.GemFireCacheImpl;
+import org.apache.geode.internal.cache.InternalRegionArguments;
+import org.apache.geode.internal.logging.LogService;
+import org.apache.geode.management.internal.cli.commands.ExportLogsCommand;
+import org.apache.geode.management.internal.cli.util.ExportLogsCacheWriter;
+import org.apache.geode.management.internal.cli.util.LogExporter;
+import org.apache.geode.management.internal.cli.util.LogFilter;
+import org.apache.geode.management.internal.configuration.domain.Configuration;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.Logger;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.file.Path;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Arrays;
+
+public class SizeExportLogsFunction implements Function, InternalEntity {
+ private static final Logger LOGGER = LogService.getLogger();
+ private static final long serialVersionUID = 1L;
+
+ private static final int BUFFER_SIZE = 1024;
+
+ @Override
+ public void execute(final FunctionContext context) {
+ try {
+ GemFireCacheImpl cache = GemFireCacheImpl.getInstance();
+ DistributionConfig config = cache.getDistributedSystem().getConfig();
+ Args args = (Args) context.getArguments();
+
+ long estimatedSize = estimateLogFileSize(cache.getMyId(), config.getLogFile(), config.getStatisticArchiveFile(), args);
+
+ if (estimatedSize == -1) {
+ context.getResultSender().lastResult(null);
+ } else {
+ context.getResultSender().lastResult(Arrays.asList(new long[]{estimatedSize}));
+ }
+
+ } catch (Exception e) {
+ LOGGER.error(e);
+ context.getResultSender().sendException(e);
+ }
+ }
+
+ long estimateLogFileSize(final DistributedMember member, final File logFile, final File statArchive, final Args args)
+ throws ParseException, IOException {
+ LOGGER.info("ExportLogsFunction started for member {}", member);
+
+ File baseLogFile = null;
+ File baseStatsFile = null;
+
+ if (args.isIncludeLogs() && !logFile.toString().isEmpty()) {
+ baseLogFile = logFile.getAbsoluteFile();
+ }
+ if (args.isIncludeStats() && !statArchive.toString().isEmpty()) {
+ baseStatsFile = statArchive.getAbsoluteFile();
+ }
+
+ LogFilter logFilter = new LogFilter(args.getLogLevel(), args.isThisLogLevelOnly(),
+ args.getStartTime(), args.getEndTime());
+
+ // TODO: int estimatedSize = new LogSizer(logFilter, baseLogFile, baseStatsFile).export();
+
+ Path exportedZipFile = new LogExporter(logFilter, baseLogFile, baseStatsFile).export();
+
+ // nothing to return back
+ if (exportedZipFile == null) {
+ return -1;
+ }
+
+ long estimatedSize = FileUtils.sizeOf(exportedZipFile.toFile());
+
+ LOGGER.info("Estimated log file size: " + estimatedSize);
+
+ return estimatedSize;
+ }
+
+ @Override
+ public boolean isHA() {
+ return false;
+ }
+
+ public static class Args implements Serializable {
+ private LocalDateTime startTime;
+ private LocalDateTime endTime;
+ private Level logLevel;
+ private boolean thisLogLevelOnly;
+ private boolean includeLogs;
+ private boolean includeStats;
+
+ public Args(String startTime, String endTime, String logLevel, boolean logLevelOnly,
+ boolean logsOnly, boolean statsOnly) {
+ this.startTime = parseTime(startTime);
+ this.endTime = parseTime(endTime);
+
+ if (StringUtils.isBlank(logLevel)) {
+ this.logLevel = Level.INFO;
+ } else {
+ this.logLevel = Level.getLevel(logLevel.toUpperCase());
+ }
+ this.thisLogLevelOnly = logLevelOnly;
+
+ this.includeLogs = !statsOnly;
+ this.includeStats = !logsOnly;
+ }
+
+ public LocalDateTime getStartTime() {
+ return startTime;
+ }
+
+ public LocalDateTime getEndTime() {
+ return endTime;
+ }
+
+ public Level getLogLevel() {
+ return logLevel;
+ }
+
+ public boolean isThisLogLevelOnly() {
+ return thisLogLevelOnly;
+ }
+
+ public boolean isIncludeLogs() {
+ return includeLogs;
+ }
+
+ public boolean isIncludeStats() {
+ return includeStats;
+ }
+ }
+
+ public static LocalDateTime parseTime(String dateString) {
+ if (dateString == null) {
+ return null;
+ }
+
+ try {
+ SimpleDateFormat df = new SimpleDateFormat(ExportLogsCommand.FORMAT);
+ return df.parse(dateString).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime();
+ } catch (ParseException e) {
+ try {
+ SimpleDateFormat df = new SimpleDateFormat(ExportLogsCommand.ONLY_DATE_FORMAT);
+ return df.parse(dateString).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime();
+ } catch (ParseException e1) {
+ return null;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/geode/blob/8215ae0a/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionIntegrationTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionIntegrationTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionIntegrationTest.java
new file mode 100644
index 0000000..7903165
--- /dev/null
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionIntegrationTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.management.internal.cli.functions;
+
+import static org.assertj.core.api.Assertions.*;
+import static org.mockito.Mockito.mock;
+
+import org.apache.geode.management.internal.cli.functions.SizeExportLogsFunction.Args;
+import org.apache.commons.io.FileUtils;
+import org.apache.geode.distributed.DistributedMember;
+import org.apache.geode.test.junit.categories.IntegrationTest;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestName;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+@Category(IntegrationTest.class)
+public class SizeExportLogsFunctionIntegrationTest {
+
+ private File dir;
+ private DistributedMember member;
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Rule
+ public TestName testName = new TestName();
+
+ @Before
+ public void before() throws Exception {
+ this.dir = this.temporaryFolder.getRoot();
+ this.member = mock(DistributedMember.class);
+ }
+
+ @Test
+ public void functionReturnsSize() throws Exception {
+ File logFile = new File(this.dir, this.testName.getMethodName() + ".log");
+ fillUpFile(logFile, (int) Math.pow(1024, 2));
+
+ File statArchive = new File(this.dir, this.testName.getMethodName() + ".gfs");
+ fillUpFile(statArchive, (int) Math.pow(1024, 2));
+
+ SizeExportLogsFunction.Args args = new Args(null, null, null, false,
+ false, false);
+
+ SizeExportLogsFunction function = new SizeExportLogsFunction();
+ assertThat(function.estimateLogFileSize(this.member, logFile, statArchive, args)).isEqualTo(0);
+ }
+
+ private void fillUpFile(File file, int sizeInBytes) throws IOException {
+ PrintWriter writer = new PrintWriter(file, "UTF-8");
+ while (FileUtils.sizeOf(file) < sizeInBytes) {
+ writer.println("this is a line of data in the file");
+ }
+ writer.close();
+ }
+
+}
\ No newline at end of file