You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by ds...@apache.org on 2017/09/08 23:21:42 UTC
[geode] branch develop updated: GEODE-3283: Expose parallel import
and export in gfsh (#753)
This is an automated email from the ASF dual-hosted git repository.
dschneider pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/develop by this push:
new 8f0a1d7 GEODE-3283: Expose parallel import and export in gfsh (#753)
8f0a1d7 is described below
commit 8f0a1d782766348fb8c5fdc6b7c364f49641beab
Author: Nick Reich <nr...@pivotal.io>
AuthorDate: Fri Sep 8 16:21:40 2017 -0700
GEODE-3283: Expose parallel import and export in gfsh (#753)
* GEODE-3283: Expose parallel import and export in gfsh
Allow users to make use of parallel import and export of snapshots
in gfsh.
---
.../cache/snapshot/RegionSnapshotServiceImpl.java | 6 +
.../internal/cli/commands/DataCommandUtil.java | 44 +++++
.../internal/cli/commands/ExportDataCommand.java | 76 +++++---
.../internal/cli/commands/ImportDataCommand.java | 74 ++++----
.../internal/cli/functions/ExportDataFunction.java | 14 +-
.../internal/cli/functions/ImportDataFunction.java | 11 +-
.../management/internal/cli/i18n/CliStrings.java | 16 +-
.../cache/snapshot/ParallelSnapshotDUnitTest.java | 4 -
.../cache/snapshot/SnapshotByteArrayDUnitTest.java | 44 ++---
.../cli/commands/ExportDataIntegrationTest.java | 177 ++++++++++++++++++
.../cli/commands/ImportDataIntegrationTest.java | 200 +++++++++++++++++++++
11 files changed, 568 insertions(+), 98 deletions(-)
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/snapshot/RegionSnapshotServiceImpl.java b/geode-core/src/main/java/org/apache/geode/internal/cache/snapshot/RegionSnapshotServiceImpl.java
index fb21594..0246d60 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/snapshot/RegionSnapshotServiceImpl.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/snapshot/RegionSnapshotServiceImpl.java
@@ -333,6 +333,12 @@ public class RegionSnapshotServiceImpl<K, V> implements RegionSnapshotService<K,
throw new IllegalArgumentException("Failure to export snapshot: "
+ snapshot.getCanonicalPath() + " is not a valid .gfd file");
}
+ File directory = snapshot.getParentFile();
+ if (directory == null) {
+ throw new IllegalArgumentException("Failure to export snapshot: "
+ + snapshot.getCanonicalPath() + " is not a valid location");
+ }
+ directory.mkdirs();
LocalRegion local = getLocalRegion(region);
Exporter<K, V> exp = createExporter(region, options);
diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/DataCommandUtil.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/DataCommandUtil.java
new file mode 100644
index 0000000..b15a958
--- /dev/null
+++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/DataCommandUtil.java
@@ -0,0 +1,44 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more contributor license *
+ * agreements. See the NOTICE file distributed with this work for additional information regarding *
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance with the License. You may obtain a *
+ * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by
+ * applicable law or agreed to in writing, software distributed under the License * is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied.
+ * See the License for the specific language governing permissions and limitations under * the
+ * License. *
+ *
+ */
+
+package org.apache.geode.management.internal.cli.commands;
+
+import java.util.List;
+
+import org.apache.geode.cache.execute.ResultCollector;
+import org.apache.geode.management.cli.Result;
+import org.apache.geode.management.internal.cli.i18n.CliStrings;
+import org.apache.geode.management.internal.cli.result.ResultBuilder;
+
+public class DataCommandUtil {
+ public static Result getFunctionResult(ResultCollector<?, ?> rc, String commandName) {
+ Result result;
+ List<Object> results = (List<Object>) rc.getResult();
+ if (results != null) {
+ Object resultObj = results.get(0);
+ if (resultObj instanceof String) {
+ result = ResultBuilder.createInfoResult((String) resultObj);
+ } else if (resultObj instanceof Exception) {
+ result = ResultBuilder.createGemFireErrorResult(((Exception) resultObj).getMessage());
+ } else {
+ result = ResultBuilder.createGemFireErrorResult(
+ CliStrings.format(CliStrings.COMMAND_FAILURE_MESSAGE, commandName));
+ }
+ } else {
+ result = ResultBuilder.createGemFireErrorResult(
+ CliStrings.format(CliStrings.COMMAND_FAILURE_MESSAGE, commandName));
+ }
+ return result;
+ }
+}
diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportDataCommand.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportDataCommand.java
index 40bd0f3..036d2f2 100644
--- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportDataCommand.java
+++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportDataCommand.java
@@ -15,7 +15,9 @@
package org.apache.geode.management.internal.cli.commands;
+import java.io.File;
import java.util.List;
+import java.util.Optional;
import org.springframework.shell.core.annotation.CliCommand;
import org.springframework.shell.core.annotation.CliOption;
@@ -23,6 +25,7 @@ import org.springframework.shell.core.annotation.CliOption;
import org.apache.geode.cache.CacheClosedException;
import org.apache.geode.cache.execute.FunctionInvocationTargetException;
import org.apache.geode.cache.execute.ResultCollector;
+import org.apache.geode.cache.snapshot.RegionSnapshotService;
import org.apache.geode.distributed.DistributedMember;
import org.apache.geode.management.cli.CliMetaData;
import org.apache.geode.management.cli.ConverterHint;
@@ -41,44 +44,35 @@ public class ExportDataCommand implements GfshCommand {
@CliOption(key = CliStrings.EXPORT_DATA__REGION, mandatory = true,
optionContext = ConverterHint.REGION_PATH,
help = CliStrings.EXPORT_DATA__REGION__HELP) String regionName,
- @CliOption(key = CliStrings.EXPORT_DATA__FILE, mandatory = true,
+ @CliOption(key = CliStrings.EXPORT_DATA__FILE,
help = CliStrings.EXPORT_DATA__FILE__HELP) String filePath,
+ @CliOption(key = CliStrings.EXPORT_DATA__DIR,
+ help = CliStrings.EXPORT_DATA__DIR__HELP) String dirPath,
@CliOption(key = CliStrings.MEMBER, optionContext = ConverterHint.MEMBERIDNAME,
- mandatory = true, help = CliStrings.EXPORT_DATA__MEMBER__HELP) String memberNameOrId) {
+ mandatory = true, help = CliStrings.EXPORT_DATA__MEMBER__HELP) String memberNameOrId,
+ @CliOption(key = CliStrings.EXPORT_DATA__PARALLEL, unspecifiedDefaultValue = "false",
+ specifiedDefaultValue = "true",
+ help = CliStrings.EXPORT_DATA__PARALLEL_HELP) boolean parallel) {
getSecurityService().authorizeRegionRead(regionName);
final DistributedMember targetMember = CliUtil.getDistributedMemberByNameOrId(memberNameOrId);
- Result result;
+ if (targetMember == null) {
+ return ResultBuilder.createUserErrorResult(
+ CliStrings.format(CliStrings.EXPORT_DATA__MEMBER__NOT__FOUND, memberNameOrId));
+ }
- if (!filePath.endsWith(CliStrings.GEODE_DATA_FILE_EXTENSION)) {
- return ResultBuilder.createUserErrorResult(CliStrings
- .format(CliStrings.INVALID_FILE_EXTENSION, CliStrings.GEODE_DATA_FILE_EXTENSION));
+ Optional<Result> validationResult = validatePath(filePath, dirPath, parallel);
+ if (validationResult.isPresent()) {
+ return validationResult.get();
}
- try {
- if (targetMember != null) {
- final String args[] = {regionName, filePath};
- ResultCollector<?, ?> rc = CliUtil.executeFunction(exportDataFunction, args, targetMember);
- List<Object> results = (List<Object>) rc.getResult();
+ Result result;
+ try {
+ String path = dirPath != null ? defaultFileName(dirPath, regionName) : filePath;
+ final String args[] = {regionName, path, Boolean.toString(parallel)};
- if (results != null) {
- Object resultObj = results.get(0);
- if (resultObj instanceof String) {
- result = ResultBuilder.createInfoResult((String) resultObj);
- } else if (resultObj instanceof Exception) {
- result = ResultBuilder.createGemFireErrorResult(((Exception) resultObj).getMessage());
- } else {
- result = ResultBuilder.createGemFireErrorResult(
- CliStrings.format(CliStrings.COMMAND_FAILURE_MESSAGE, CliStrings.EXPORT_DATA));
- }
- } else {
- result = ResultBuilder.createGemFireErrorResult(
- CliStrings.format(CliStrings.COMMAND_FAILURE_MESSAGE, CliStrings.EXPORT_DATA));
- }
- } else {
- result = ResultBuilder.createUserErrorResult(
- CliStrings.format(CliStrings.EXPORT_DATA__MEMBER__NOT__FOUND, memberNameOrId));
- }
+ ResultCollector<?, ?> rc = CliUtil.executeFunction(exportDataFunction, args, targetMember);
+ result = DataCommandUtil.getFunctionResult(rc, CliStrings.EXPORT_DATA);
} catch (CacheClosedException e) {
result = ResultBuilder.createGemFireErrorResult(e.getMessage());
} catch (FunctionInvocationTargetException e) {
@@ -87,4 +81,28 @@ public class ExportDataCommand implements GfshCommand {
}
return result;
}
+
+ private String defaultFileName(String dirPath, String regionName) {
+ return new File(dirPath, regionName + RegionSnapshotService.SNAPSHOT_FILE_EXTENSION)
+ .getAbsolutePath();
+ }
+
+ private Optional<Result> validatePath(String filePath, String dirPath, boolean parallel) {
+ if (filePath == null && dirPath == null) {
+ return Optional
+ .of(ResultBuilder.createUserErrorResult("Must specify a location to save snapshot"));
+ } else if (filePath != null && dirPath != null) {
+ return Optional.of(ResultBuilder.createUserErrorResult(
+ "Options \"file\" and \"dir\" cannot be specified at the same time"));
+ } else if (parallel && dirPath == null) {
+ return Optional.of(
+ ResultBuilder.createUserErrorResult("Must specify a directory to save snapshot files"));
+ }
+
+ if (dirPath == null && !filePath.endsWith(CliStrings.GEODE_DATA_FILE_EXTENSION)) {
+ return Optional.of(ResultBuilder.createUserErrorResult(CliStrings
+ .format(CliStrings.INVALID_FILE_EXTENSION, CliStrings.GEODE_DATA_FILE_EXTENSION)));
+ }
+ return Optional.empty();
+ }
}
diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ImportDataCommand.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ImportDataCommand.java
index 2308405..c3ca0c1 100644
--- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ImportDataCommand.java
+++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ImportDataCommand.java
@@ -16,6 +16,7 @@
package org.apache.geode.management.internal.cli.commands;
import java.util.List;
+import java.util.Optional;
import org.springframework.shell.core.annotation.CliCommand;
import org.springframework.shell.core.annotation.CliOption;
@@ -40,49 +41,39 @@ public class ImportDataCommand implements GfshCommand {
public Result importData(
@CliOption(key = CliStrings.IMPORT_DATA__REGION, optionContext = ConverterHint.REGION_PATH,
mandatory = true, help = CliStrings.IMPORT_DATA__REGION__HELP) String regionName,
- @CliOption(key = CliStrings.IMPORT_DATA__FILE, mandatory = true,
+ @CliOption(key = CliStrings.IMPORT_DATA__FILE,
help = CliStrings.IMPORT_DATA__FILE__HELP) String filePath,
+ @CliOption(key = CliStrings.IMPORT_DATA__DIR,
+ help = CliStrings.IMPORT_DATA__DIR__HELP) String dirPath,
@CliOption(key = CliStrings.MEMBER, mandatory = true,
optionContext = ConverterHint.MEMBERIDNAME,
help = CliStrings.IMPORT_DATA__MEMBER__HELP) String memberNameOrId,
@CliOption(key = CliStrings.IMPORT_DATA__INVOKE_CALLBACKS, unspecifiedDefaultValue = "false",
- help = CliStrings.IMPORT_DATA__INVOKE_CALLBACKS__HELP) boolean invokeCallbacks) {
+ help = CliStrings.IMPORT_DATA__INVOKE_CALLBACKS__HELP) boolean invokeCallbacks,
+ @CliOption(key = CliStrings.IMPORT_DATA__PARALLEL, unspecifiedDefaultValue = "false",
+ specifiedDefaultValue = "true",
+ help = CliStrings.IMPORT_DATA__PARALLEL_HELP) boolean parallel) {
getSecurityService().authorizeRegionWrite(regionName);
- Result result;
-
- try {
- final DistributedMember targetMember = CliUtil.getDistributedMemberByNameOrId(memberNameOrId);
+ final DistributedMember targetMember = CliUtil.getDistributedMemberByNameOrId(memberNameOrId);
+ if (targetMember == null) {
+ return ResultBuilder.createUserErrorResult(
+ CliStrings.format(CliStrings.IMPORT_DATA__MEMBER__NOT__FOUND, memberNameOrId));
+ }
- if (!filePath.endsWith(CliStrings.GEODE_DATA_FILE_EXTENSION)) {
- return ResultBuilder.createUserErrorResult(CliStrings
- .format(CliStrings.INVALID_FILE_EXTENSION, CliStrings.GEODE_DATA_FILE_EXTENSION));
- }
- if (targetMember != null) {
- final Object args[] = {regionName, filePath, invokeCallbacks};
- ResultCollector<?, ?> rc = CliUtil.executeFunction(importDataFunction, args, targetMember);
- List<Object> results = (List<Object>) rc.getResult();
+ Optional<Result> validationResult = validatePath(filePath, dirPath, parallel);
+ if (validationResult.isPresent()) {
+ return validationResult.get();
+ }
- if (results != null) {
- Object resultObj = results.get(0);
+ Result result;
+ try {
+ String path = dirPath != null ? dirPath : filePath;
+ final Object args[] = {regionName, path, invokeCallbacks, parallel};
- if (resultObj instanceof String) {
- result = ResultBuilder.createInfoResult((String) resultObj);
- } else if (resultObj instanceof Exception) {
- result = ResultBuilder.createGemFireErrorResult(((Exception) resultObj).getMessage());
- } else {
- result = ResultBuilder.createGemFireErrorResult(
- CliStrings.format(CliStrings.COMMAND_FAILURE_MESSAGE, CliStrings.IMPORT_DATA));
- }
- } else {
- result = ResultBuilder.createGemFireErrorResult(
- CliStrings.format(CliStrings.COMMAND_FAILURE_MESSAGE, CliStrings.IMPORT_DATA));
- }
- } else {
- result = ResultBuilder.createUserErrorResult(
- CliStrings.format(CliStrings.IMPORT_DATA__MEMBER__NOT__FOUND, memberNameOrId));
- }
+ ResultCollector<?, ?> rc = CliUtil.executeFunction(importDataFunction, args, targetMember);
+ result = DataCommandUtil.getFunctionResult(rc, CliStrings.IMPORT_DATA);
} catch (CacheClosedException e) {
result = ResultBuilder.createGemFireErrorResult(e.getMessage());
} catch (FunctionInvocationTargetException e) {
@@ -91,4 +82,23 @@ public class ImportDataCommand implements GfshCommand {
}
return result;
}
+
+ private Optional<Result> validatePath(String filePath, String dirPath, boolean parallel) {
+ if (filePath == null && dirPath == null) {
+ return Optional
+ .of(ResultBuilder.createUserErrorResult("Must specify a location to load snapshot from"));
+ } else if (filePath != null && dirPath != null) {
+ return Optional.of(ResultBuilder.createUserErrorResult(
+ "Options \"file\" and \"dir\" cannot be specified at the same time"));
+ } else if (parallel && dirPath == null) {
+ return Optional.of(ResultBuilder
+ .createUserErrorResult("Must specify a directory to load snapshot files from"));
+ }
+
+ if (dirPath == null && !filePath.endsWith(CliStrings.GEODE_DATA_FILE_EXTENSION)) {
+ return Optional.of(ResultBuilder.createUserErrorResult(CliStrings
+ .format(CliStrings.INVALID_FILE_EXTENSION, CliStrings.GEODE_DATA_FILE_EXTENSION)));
+ }
+ return Optional.empty();
+ }
}
diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportDataFunction.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportDataFunction.java
index 537678f..52561ca 100644
--- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportDataFunction.java
+++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportDataFunction.java
@@ -22,8 +22,10 @@ import org.apache.geode.cache.Region;
import org.apache.geode.cache.execute.FunctionAdapter;
import org.apache.geode.cache.execute.FunctionContext;
import org.apache.geode.cache.snapshot.RegionSnapshotService;
+import org.apache.geode.cache.snapshot.SnapshotOptions;
import org.apache.geode.cache.snapshot.SnapshotOptions.SnapshotFormat;
import org.apache.geode.internal.InternalEntity;
+import org.apache.geode.internal.cache.snapshot.SnapshotOptionsImpl;
import org.apache.geode.management.internal.cli.i18n.CliStrings;
/***
@@ -41,8 +43,13 @@ public class ExportDataFunction extends FunctionAdapter implements InternalEntit
public void execute(FunctionContext context) {
final String[] args = (String[]) context.getArguments();
+ if (args.length < 3) {
+ throw new IllegalStateException(
+ "Arguments length does not match required length. Export command may have been sent from incompatible older version");
+ }
final String regionName = args[0];
final String fileName = args[1];
+ final boolean parallel = Boolean.parseBoolean(args[2]);
try {
Cache cache = CacheFactory.getAnyInstance();
@@ -51,7 +58,12 @@ public class ExportDataFunction extends FunctionAdapter implements InternalEntit
if (region != null) {
RegionSnapshotService<?, ?> snapshotService = region.getSnapshotService();
final File exportFile = new File(fileName);
- snapshotService.save(exportFile, SnapshotFormat.GEMFIRE);
+ if (parallel) {
+ SnapshotOptions options = new SnapshotOptionsImpl<>().setParallelMode(true);
+ snapshotService.save(exportFile, SnapshotFormat.GEMFIRE, options);
+ } else {
+ snapshotService.save(exportFile, SnapshotFormat.GEMFIRE);
+ }
String successMessage = CliStrings.format(CliStrings.EXPORT_DATA__SUCCESS__MESSAGE,
regionName, exportFile.getCanonicalPath(), hostName);
context.getResultSender().lastResult(successMessage);
diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ImportDataFunction.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ImportDataFunction.java
index 13949c8..e776c8e 100644
--- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ImportDataFunction.java
+++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ImportDataFunction.java
@@ -38,12 +38,14 @@ public class ImportDataFunction extends FunctionAdapter implements InternalEntit
public void execute(FunctionContext context) {
final Object[] args = (Object[]) context.getArguments();
+ if (args.length < 4) {
+ throw new IllegalStateException(
+ "Arguments length does not match required length. Import command may have been sent from incompatible older version");
+ }
final String regionName = (String) args[0];
final String importFileName = (String) args[1];
- boolean invokeCallbacks = false;
- if (args.length > 2) {
- invokeCallbacks = (boolean) args[2];
- }
+ final boolean invokeCallbacks = (boolean) args[2];
+ final boolean parallel = (boolean) args[3];
try {
final Cache cache = CacheFactory.getAnyInstance();
@@ -53,6 +55,7 @@ public class ImportDataFunction extends FunctionAdapter implements InternalEntit
RegionSnapshotService<?, ?> snapshotService = region.getSnapshotService();
SnapshotOptions options = snapshotService.createOptions();
options.invokeCallbacks(invokeCallbacks);
+ options.setParallelMode(parallel);
File importFile = new File(importFileName);
snapshotService.load(new File(importFileName), SnapshotFormat.GEMFIRE, options);
String successMessage = CliStrings.format(CliStrings.IMPORT_DATA__SUCCESS__MESSAGE,
diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/i18n/CliStrings.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/i18n/CliStrings.java
index 5e97976..2492490 100644
--- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/i18n/CliStrings.java
+++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/i18n/CliStrings.java
@@ -1348,9 +1348,15 @@ public class CliStrings {
public static final String EXPORT_DATA__REGION__HELP = "Region from which data will be exported.";
public static final String EXPORT_DATA__FILE = "file";
public static final String EXPORT_DATA__FILE__HELP =
- "File to which the exported data will be written. The file must have an extension of \".gfd\".";
+ "File to which the exported data will be written. The file must have an extension of \".gfd\". Cannot be specified at the same time as \"dir\"";
public static final String EXPORT_DATA__MEMBER__HELP =
"Name/Id of a member which hosts the region. The data will be exported to the specified file on the host where the member is running.";
+ public static final String EXPORT_DATA__DIR = "dir";
+ public static final String EXPORT_DATA__DIR__HELP =
+ "Directory to which the exported data will be written. Required if parallel set to true. Cannot be specified at the same time as \"file\"";
+ public static final String EXPORT_DATA__PARALLEL = "parallel";
+ public static final String EXPORT_DATA__PARALLEL_HELP =
+ "Export local data on each node to a directory on that machine. Available for partitioned regions only";
public static final String EXPORT_DATA__MEMBER__NOT__FOUND = "Member {0} not found";
public static final String EXPORT_DATA__REGION__NOT__FOUND = "Region {0} not found";
public static final String EXPORT_DATA__SUCCESS__MESSAGE =
@@ -1528,7 +1534,13 @@ public class CliStrings {
public static final String IMPORT_DATA__REGION__HELP = "Region into which data will be imported.";
public static final String IMPORT_DATA__FILE = "file";
public static final String IMPORT_DATA__FILE__HELP =
- "File from which the imported data will be read. The file must have an extension of \".gfd\".";
+ "File from which the imported data will be read. The file must have an extension of \".gfd\". Cannot be specified at the same time as \"dir\"";
+ public static final String IMPORT_DATA__DIR = "dir";
+ public static final String IMPORT_DATA__DIR__HELP =
+ "Directory from which all data files (\".gfd\") will be read. Required if parallel set to true. Cannot be specified at the same time as \"file\"";
+ public static final String IMPORT_DATA__PARALLEL = "parallel";
+ public static final String IMPORT_DATA__PARALLEL_HELP =
+ "Import data from given directory on all members. Used to import data from a parallel export. Available for partitioned regions only";
public static final String IMPORT_DATA__MEMBER__HELP =
"Name/Id of a member which hosts the region. The data will be imported from the specified file on the host where the member is running.";
public static final String IMPORT_DATA__MEMBER__NOT__FOUND = "Member {0} not found.";
diff --git a/geode-core/src/test/java/org/apache/geode/cache/snapshot/ParallelSnapshotDUnitTest.java b/geode-core/src/test/java/org/apache/geode/cache/snapshot/ParallelSnapshotDUnitTest.java
index 5b59674..a8791e4 100644
--- a/geode-core/src/test/java/org/apache/geode/cache/snapshot/ParallelSnapshotDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/cache/snapshot/ParallelSnapshotDUnitTest.java
@@ -181,10 +181,6 @@ public class ParallelSnapshotDUnitTest extends JUnit4CacheTestCase {
}
}
- private void forEachVm(SerializableCallable call, boolean local) throws Exception {
- this.forEachVm(call, local, Integer.MAX_VALUE);
- }
-
private void forEachVm(SerializableCallable call, boolean local, int maxNodes) throws Exception {
Host host = Host.getHost(0);
int vms = Math.min(host.getVMCount(), maxNodes);
diff --git a/geode-core/src/test/java/org/apache/geode/cache/snapshot/SnapshotByteArrayDUnitTest.java b/geode-core/src/test/java/org/apache/geode/cache/snapshot/SnapshotByteArrayDUnitTest.java
index 8565ff5..487dd03 100644
--- a/geode-core/src/test/java/org/apache/geode/cache/snapshot/SnapshotByteArrayDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/cache/snapshot/SnapshotByteArrayDUnitTest.java
@@ -14,18 +14,15 @@
*/
package org.apache.geode.cache.snapshot;
-import org.junit.experimental.categories.Category;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-import org.apache.geode.test.dunit.cache.internal.JUnit4CacheTestCase;
-import org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase;
-import org.apache.geode.test.junit.categories.DistributedTest;
-
import java.io.File;
import com.examples.snapshot.MyPdxSerializer;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TemporaryFolder;
+
import org.apache.geode.cache.Cache;
import org.apache.geode.cache.CacheFactory;
import org.apache.geode.cache.EntryEvent;
@@ -33,17 +30,24 @@ import org.apache.geode.cache.Region;
import org.apache.geode.cache.snapshot.RegionGenerator.RegionType;
import org.apache.geode.cache.snapshot.SnapshotOptions.SnapshotFormat;
import org.apache.geode.cache.util.CacheListenerAdapter;
-import org.apache.geode.cache30.CacheTestCase;
import org.apache.geode.test.dunit.Host;
import org.apache.geode.test.dunit.LogWriterUtils;
import org.apache.geode.test.dunit.SerializableCallable;
+import org.apache.geode.test.dunit.cache.internal.JUnit4CacheTestCase;
+import org.apache.geode.test.junit.categories.DistributedTest;
+import org.apache.geode.test.junit.rules.serializable.SerializableTemporaryFolder;
@Category(DistributedTest.class)
public class SnapshotByteArrayDUnitTest extends JUnit4CacheTestCase {
- private final File snap = new File("snapshot-ops.gfd");
+ @Rule
+ public TemporaryFolder tempDir = new SerializableTemporaryFolder();
+
+ private File snap;
- public SnapshotByteArrayDUnitTest() {
- super();
+ @Before
+ public void setup() throws Exception {
+ snap = new File(tempDir.getRoot(), "snapshot-ops.gfd");
+ loadCache();
}
@Test
@@ -113,19 +117,7 @@ public class SnapshotByteArrayDUnitTest extends JUnit4CacheTestCase {
}
}
- @Override
- public final void postSetUp() throws Exception {
- loadCache();
- }
-
- @Override
- public final void preTearDownCacheTestCase() throws Exception {
- if (snap.exists()) {
- snap.delete();
- }
- }
-
- public void loadCache() throws Exception {
+ private void loadCache() throws Exception {
SerializableCallable setup = new SerializableCallable() {
@Override
public Object call() throws Exception {
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportDataIntegrationTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportDataIntegrationTest.java
new file mode 100644
index 0000000..8c821a5
--- /dev/null
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportDataIntegrationTest.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+
+package org.apache.geode.management.internal.cli.commands;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.stream.IntStream;
+
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TemporaryFolder;
+
+import org.apache.geode.cache.Region;
+import org.apache.geode.cache.RegionShortcut;
+import org.apache.geode.management.internal.cli.i18n.CliStrings;
+import org.apache.geode.management.internal.cli.util.CommandStringBuilder;
+import org.apache.geode.test.dunit.rules.GfshShellConnectionRule;
+import org.apache.geode.test.dunit.rules.ServerStarterRule;
+import org.apache.geode.test.junit.categories.IntegrationTest;
+
+@Category(IntegrationTest.class)
+public class ExportDataIntegrationTest {
+ private static final String TEST_REGION_NAME = "testRegion";
+ private static final String SNAPSHOT_FILE = "snapshot.gfd";
+ private static final String SNAPSHOT_DIR = "snapshots";
+ private static final int DATA_POINTS = 10;
+
+ @ClassRule
+ public static ServerStarterRule server = new ServerStarterRule().withJMXManager()
+ .withRegion(RegionShortcut.PARTITION, TEST_REGION_NAME).withEmbeddedLocator();
+
+ @Rule
+ public GfshShellConnectionRule gfsh = new GfshShellConnectionRule();
+
+ @Rule
+ public TemporaryFolder tempDir = new TemporaryFolder();
+
+ private Region<String, String> region;
+ private Path snapshotFile;
+ private Path snapshotDir;
+
+ @Before
+ public void setup() throws Exception {
+ gfsh.connectAndVerify(server.getEmbeddedLocatorPort(),
+ GfshShellConnectionRule.PortType.locator);
+ region = server.getCache().getRegion(TEST_REGION_NAME);
+ loadRegion("value");
+ Path basePath = tempDir.getRoot().toPath();
+ snapshotFile = basePath.resolve(SNAPSHOT_FILE);
+ snapshotDir = basePath.resolve(SNAPSHOT_DIR);
+ }
+
+ @Test
+ public void testExport() throws Exception {
+ String exportCommand = buildBaseExportCommand()
+ .addOption(CliStrings.EXPORT_DATA__FILE, snapshotFile.toString()).getCommandString();
+ gfsh.executeAndVerifyCommand(exportCommand);
+ assertThat(gfsh.getGfshOutput()).contains("Data successfully exported ");
+ }
+
+ @Test
+ public void testParallelExport() throws Exception {
+ String exportCommand =
+ buildBaseExportCommand().addOption(CliStrings.EXPORT_DATA__DIR, snapshotDir.toString())
+ .addOption(CliStrings.EXPORT_DATA__PARALLEL, "true").getCommandString();
+ gfsh.executeAndVerifyCommand(exportCommand);
+ assertThat(gfsh.getGfshOutput()).contains("Data successfully exported ");
+ }
+
+ @Test
+ public void testInvalidMember() throws Exception {
+ String invalidMemberName = "invalidMember";
+ String invalidMemberCommand = new CommandStringBuilder(CliStrings.EXPORT_DATA)
+ .addOption(CliStrings.MEMBER, invalidMemberName)
+ .addOption(CliStrings.EXPORT_DATA__REGION, TEST_REGION_NAME)
+ .addOption(CliStrings.EXPORT_DATA__FILE, snapshotFile.toString()).getCommandString();
+ gfsh.executeCommand(invalidMemberCommand);
+ assertThat(gfsh.getGfshOutput()).contains("Member " + invalidMemberName + " not found");
+ }
+
+ @Test
+ public void testNonExistentRegion() throws Exception {
+ String nonExistentRegionCommand = new CommandStringBuilder(CliStrings.EXPORT_DATA)
+ .addOption(CliStrings.MEMBER, server.getName())
+ .addOption(CliStrings.EXPORT_DATA__REGION, "/nonExistentRegion")
+ .addOption(CliStrings.EXPORT_DATA__FILE, snapshotFile.toString()).getCommandString();
+ gfsh.executeCommand(nonExistentRegionCommand);
+ assertThat(gfsh.getGfshOutput()).contains("Could not process command due to error. Region");
+ }
+
+ @Test
+ public void testInvalidFile() throws Exception {
+ String invalidFileCommand = buildBaseExportCommand()
+ .addOption(CliStrings.EXPORT_DATA__FILE, snapshotFile.toString() + ".invalid")
+ .getCommandString();
+ gfsh.executeCommand(invalidFileCommand);
+ assertThat(gfsh.getGfshOutput())
+ .contains("Invalid file type, the file extension must be \".gfd\"");
+ }
+
+ @Test
+ public void testMissingRegion() throws Exception {
+ String missingRegionCommand = new CommandStringBuilder(CliStrings.EXPORT_DATA)
+ .addOption(CliStrings.MEMBER, server.getName())
+ .addOption(CliStrings.EXPORT_DATA__FILE, snapshotFile.toString()).getCommandString();
+ gfsh.executeCommand(missingRegionCommand);
+ assertThat(gfsh.getGfshOutput()).contains("You should specify option");
+ }
+
+ @Test
+ public void testMissingMember() throws Exception {
+ String missingMemberCommand = new CommandStringBuilder(CliStrings.EXPORT_DATA)
+ .addOption(CliStrings.EXPORT_DATA__REGION, TEST_REGION_NAME)
+ .addOption(CliStrings.EXPORT_DATA__FILE, snapshotFile.toString()).getCommandString();
+ gfsh.executeCommand(missingMemberCommand);
+ assertThat(gfsh.getGfshOutput()).contains("You should specify option");
+ }
+
+ @Test
+ public void testMissingFileAndDirectory() throws Exception {
+ String missingFileAndDirCommand = buildBaseExportCommand().getCommandString();
+ gfsh.executeCommand(missingFileAndDirCommand);
+ assertThat(gfsh.getGfshOutput()).contains("Must specify a location to save snapshot");
+ }
+
+ @Test
+ public void testParallelExportWithOnlyFile() throws Exception {
+ String exportCommand =
+ buildBaseExportCommand().addOption(CliStrings.EXPORT_DATA__FILE, snapshotFile.toString())
+ .addOption(CliStrings.EXPORT_DATA__PARALLEL, "true").getCommandString();
+ gfsh.executeCommand(exportCommand);
+ assertThat(gfsh.getGfshOutput()).contains("Must specify a directory to save snapshot files");
+ }
+
+ @Test
+ public void testSpecifyingDirectoryAndFileCommands() throws Exception {
+ String exportCommand =
+ buildBaseExportCommand().addOption(CliStrings.EXPORT_DATA__FILE, snapshotFile.toString())
+ .addOption(CliStrings.EXPORT_DATA__DIR, snapshotDir.toString()).getCommandString();
+ gfsh.executeCommand(exportCommand);
+ assertThat(gfsh.getGfshOutput())
+ .contains("Options \"file\" and \"dir\" cannot be specified at the same time");
+
+ assertFalse(Files.exists(snapshotDir));
+ }
+
+ private void loadRegion(String value) {
+ IntStream.range(0, DATA_POINTS).forEach(i -> region.put("key" + i, value));
+ }
+
+ private CommandStringBuilder buildBaseExportCommand() {
+ return new CommandStringBuilder(CliStrings.EXPORT_DATA)
+ .addOption(CliStrings.MEMBER, server.getName())
+ .addOption(CliStrings.EXPORT_DATA__REGION, TEST_REGION_NAME);
+ }
+}
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ImportDataIntegrationTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ImportDataIntegrationTest.java
new file mode 100644
index 0000000..cdc5159
--- /dev/null
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ImportDataIntegrationTest.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+
+package org.apache.geode.management.internal.cli.commands;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+
+import java.nio.file.Path;
+import java.util.stream.IntStream;
+
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TemporaryFolder;
+
+import org.apache.geode.cache.Region;
+import org.apache.geode.cache.RegionShortcut;
+import org.apache.geode.management.internal.cli.i18n.CliStrings;
+import org.apache.geode.management.internal.cli.util.CommandStringBuilder;
+import org.apache.geode.test.dunit.rules.GfshShellConnectionRule;
+import org.apache.geode.test.dunit.rules.ServerStarterRule;
+import org.apache.geode.test.junit.categories.IntegrationTest;
+
+@Category(IntegrationTest.class)
+public class ImportDataIntegrationTest {
+ private static final String TEST_REGION_NAME = "testRegion";
+ private static final String SNAPSHOT_FILE = "snapshot.gfd";
+ private static final String SNAPSHOT_DIR = "snapshots";
+ private static final int DATA_POINTS = 10;
+
+ @ClassRule
+ public static ServerStarterRule server = new ServerStarterRule().withJMXManager()
+ .withRegion(RegionShortcut.PARTITION, TEST_REGION_NAME).withEmbeddedLocator();
+
+ @Rule
+ public GfshShellConnectionRule gfsh = new GfshShellConnectionRule();
+
+ @Rule
+ public TemporaryFolder tempDir = new TemporaryFolder();
+
+ private Region<String, String> region;
+ private Path snapshotFile;
+ private Path snapshotDir;
+
+ @Before
+ public void setup() throws Exception {
+ gfsh.connectAndVerify(server.getEmbeddedLocatorPort(),
+ GfshShellConnectionRule.PortType.locator);
+ region = server.getCache().getRegion(TEST_REGION_NAME);
+ loadRegion("value");
+ Path basePath = tempDir.getRoot().toPath();
+ snapshotFile = basePath.resolve(SNAPSHOT_FILE);
+ snapshotDir = basePath.resolve(SNAPSHOT_DIR);
+ }
+
+ @Test
+ public void testExportImport() throws Exception {
+ String exportCommand = buildBaseExportCommand()
+ .addOption(CliStrings.EXPORT_DATA__FILE, snapshotFile.toString()).getCommandString();
+ gfsh.executeAndVerifyCommand(exportCommand);
+
+ loadRegion("");
+
+ String importCommand = buildBaseImportCommand()
+ .addOption(CliStrings.IMPORT_DATA__FILE, snapshotFile.toString()).getCommandString();
+ gfsh.executeAndVerifyCommand(importCommand);
+ assertThat(gfsh.getGfshOutput()).contains("Data imported from file");
+ validateImport("value");
+ }
+
+ @Test
+ public void testParallelExportImport() throws Exception {
+ String exportCommand =
+ buildBaseExportCommand().addOption(CliStrings.EXPORT_DATA__DIR, snapshotDir.toString())
+ .addOption(CliStrings.EXPORT_DATA__PARALLEL, "true").getCommandString();
+ gfsh.executeAndVerifyCommand(exportCommand);
+
+ loadRegion("");
+
+ String importCommand =
+ buildBaseImportCommand().addOption(CliStrings.IMPORT_DATA__DIR, snapshotDir.toString())
+ .addOption(CliStrings.IMPORT_DATA__PARALLEL, "true").getCommandString();
+ gfsh.executeAndVerifyCommand(importCommand);
+ assertThat(gfsh.getGfshOutput()).contains("Data imported from file");
+
+ validateImport("value");
+ }
+
+ @Test
+ public void testInvalidMember() throws Exception {
+ String invalidMemberName = "invalidMember";
+ String invalidMemberCommand = new CommandStringBuilder(CliStrings.EXPORT_DATA)
+ .addOption(CliStrings.MEMBER, invalidMemberName)
+ .addOption(CliStrings.IMPORT_DATA__REGION, TEST_REGION_NAME)
+ .addOption(CliStrings.IMPORT_DATA__FILE, snapshotFile.toString()).getCommandString();
+ gfsh.executeCommand(invalidMemberCommand);
+ assertThat(gfsh.getGfshOutput()).contains("Member " + invalidMemberName + " not found");
+ }
+
+ @Test
+ public void testNonExistentRegion() throws Exception {
+ String nonExistentRegionCommand = new CommandStringBuilder(CliStrings.EXPORT_DATA)
+ .addOption(CliStrings.MEMBER, server.getName())
+ .addOption(CliStrings.IMPORT_DATA__REGION, "/nonExistentRegion")
+ .addOption(CliStrings.IMPORT_DATA__FILE, snapshotFile.toString()).getCommandString();
+ gfsh.executeCommand(nonExistentRegionCommand);
+ assertThat(gfsh.getGfshOutput()).contains("Could not process command due to error. Region");
+ }
+
+ @Test
+ public void testInvalidFile() throws Exception {
+ String invalidFileCommand = buildBaseImportCommand()
+ .addOption(CliStrings.IMPORT_DATA__FILE, snapshotFile.toString() + ".invalid")
+ .getCommandString();
+ gfsh.executeCommand(invalidFileCommand);
+ assertThat(gfsh.getGfshOutput())
+ .contains("Invalid file type, the file extension must be \".gfd\"");
+ }
+
+ @Test
+ public void testMissingRegion() throws Exception {
+ String missingRegionCommand = new CommandStringBuilder(CliStrings.IMPORT_DATA)
+ .addOption(CliStrings.MEMBER, server.getName())
+ .addOption(CliStrings.IMPORT_DATA__FILE, snapshotFile.toString()).getCommandString();
+ gfsh.executeCommand(missingRegionCommand);
+ assertThat(gfsh.getGfshOutput()).contains("You should specify option");
+ }
+
+ @Test
+ public void testMissingMember() throws Exception {
+ String missingMemberCommand = new CommandStringBuilder(CliStrings.EXPORT_DATA)
+ .addOption(CliStrings.IMPORT_DATA__REGION, TEST_REGION_NAME)
+ .addOption(CliStrings.IMPORT_DATA__FILE, snapshotFile.toString()).getCommandString();
+ gfsh.executeCommand(missingMemberCommand);
+ assertThat(gfsh.getGfshOutput()).contains("You should specify option");
+ }
+
+ @Test
+ public void testMissingFileAndDirectory() throws Exception {
+ String missingFileAndDirCommand = buildBaseImportCommand().getCommandString();
+ gfsh.executeCommand(missingFileAndDirCommand);
+ assertThat(gfsh.getGfshOutput()).contains("Must specify a location to load snapshot from");
+ }
+
+ @Test
+ public void testParallelWithOnlyFile() throws Exception {
+ String importCommand =
+ buildBaseImportCommand().addOption(CliStrings.IMPORT_DATA__FILE, snapshotFile.toString())
+ .addOption(CliStrings.IMPORT_DATA__PARALLEL, "true").getCommandString();
+ gfsh.executeCommand(importCommand);
+ assertThat(gfsh.getGfshOutput())
+ .contains("Must specify a directory to load snapshot files from");
+ }
+
+ @Test
+ public void testSpecifyingDirectoryAndFileCommands() throws Exception {
+ String importCommand =
+ buildBaseImportCommand().addOption(CliStrings.IMPORT_DATA__FILE, snapshotFile.toString())
+ .addOption(CliStrings.IMPORT_DATA__DIR, snapshotDir.toString()).getCommandString();
+ gfsh.executeCommand(importCommand);
+ assertThat(gfsh.getGfshOutput())
+ .contains("Options \"file\" and \"dir\" cannot be specified at the same time");
+ }
+
+ private void validateImport(String value) {
+ IntStream.range(0, DATA_POINTS).forEach(i -> assertEquals(value, region.get("key" + i)));
+ }
+
+ private void loadRegion(String value) {
+ IntStream.range(0, DATA_POINTS).forEach(i -> region.put("key" + i, value));
+ }
+
+ private CommandStringBuilder buildBaseImportCommand() {
+ return new CommandStringBuilder(CliStrings.IMPORT_DATA)
+ .addOption(CliStrings.MEMBER, server.getName())
+ .addOption(CliStrings.IMPORT_DATA__REGION, TEST_REGION_NAME);
+ }
+
+ private CommandStringBuilder buildBaseExportCommand() {
+ return new CommandStringBuilder(CliStrings.EXPORT_DATA)
+ .addOption(CliStrings.MEMBER, server.getName())
+ .addOption(CliStrings.EXPORT_DATA__REGION, TEST_REGION_NAME);
+ }
+}
--
To stop receiving notification emails like this one, please contact
['"commits@geode.apache.org" <co...@geode.apache.org>'].