You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by tl...@apache.org on 2021/06/02 09:47:44 UTC
[ignite] branch master updated: IGNITE-14801 Add tests for commands
to manage metadata (#9137)
This is an automated email from the ASF dual-hosted git repository.
tledkov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 35e00f4 IGNITE-14801 Add tests for commands to manage metadata (#9137)
35e00f4 is described below
commit 35e00f408fce431ff21273f572a6256502195b56
Author: Taras Ledkov <tl...@gridgain.com>
AuthorDate: Wed Jun 2 12:47:02 2021 +0300
IGNITE-14801 Add tests for commands to manage metadata (#9137)
---
.../util/GridCommandHandlerMetadataTest.java | 291 +++++++++++++++++++--
.../commandline/meta/tasks/MetadataInfoTask.java | 13 +-
2 files changed, 275 insertions(+), 29 deletions(-)
diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerMetadataTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerMetadataTest.java
index e4a2579..f4aa759 100644
--- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerMetadataTest.java
+++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerMetadataTest.java
@@ -17,6 +17,7 @@
package org.apache.ignite.util;
+import java.io.IOException;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Files;
@@ -26,12 +27,14 @@ import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
+import java.time.LocalDate;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
-
import org.apache.ignite.IgniteBinary;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteLogger;
@@ -48,12 +51,14 @@ import org.apache.ignite.internal.binary.BinaryTypeImpl;
import org.apache.ignite.testframework.GridTestUtils;
import org.jetbrains.annotations.Nullable;
import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_INVALID_ARGUMENTS;
import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR;
import static org.apache.ignite.testframework.GridTestUtils.assertContains;
+import static org.apache.ignite.testframework.GridTestUtils.assertNotContains;
/**
* Checks command line metadata commands.
@@ -66,6 +71,12 @@ public class GridCommandHandlerMetadataTest extends GridCommandHandlerClusterByC
private static final int TYPES_CNT = 10;
/** */
+ @Before
+ public void init() {
+ injectTestSystemOut();
+ }
+
+ /** */
@After
public void clear() {
crd.binary().types().stream().forEach(type -> crd.context().cacheObjects().removeType(type.typeId()));
@@ -80,8 +91,6 @@ public class GridCommandHandlerMetadataTest extends GridCommandHandlerClusterByC
*/
@Test
public void testMetadataList() {
- injectTestSystemOut();
-
for (int typeNum = 0; typeNum < TYPES_CNT; ++typeNum) {
BinaryObjectBuilder bob = crd.binary().builder("Type_" + typeNum);
@@ -121,8 +130,6 @@ public class GridCommandHandlerMetadataTest extends GridCommandHandlerClusterByC
*/
@Test
public void testMetadataRemoveWrongType() {
- injectTestSystemOut();
-
String wrongTypeName = "Type01";
assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute("--meta", "remove", "--typeName", wrongTypeName));
@@ -160,8 +167,6 @@ public class GridCommandHandlerMetadataTest extends GridCommandHandlerClusterByC
*/
@Test
public void testMetadataForInternalClassesIsNotRegistered() {
- injectTestSystemOut();
-
IgniteCache<Object, Object> dfltCache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME);
dfltCache.put(1, new TestValue());
@@ -204,8 +209,6 @@ public class GridCommandHandlerMetadataTest extends GridCommandHandlerClusterByC
*/
@Test
public void testMetadataDetails() {
- injectTestSystemOut();
-
BinaryObjectBuilder bob0 = crd.binary().builder("TypeName0");
bob0.setField("fld0", 0);
bob0.build();
@@ -257,8 +260,6 @@ public class GridCommandHandlerMetadataTest extends GridCommandHandlerClusterByC
*/
@Test
public void testInvalidArguments() {
- injectTestSystemOut();
-
String out;
assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute("--meta", "remove"));
@@ -290,8 +291,6 @@ public class GridCommandHandlerMetadataTest extends GridCommandHandlerClusterByC
*/
@Test
public void testRemoveUpdate() throws Exception {
- injectTestSystemOut();
-
Path typeFile = FS.getPath("type0.bin");
try {
@@ -357,8 +356,6 @@ public class GridCommandHandlerMetadataTest extends GridCommandHandlerClusterByC
*/
@Test
public void testDropThinConnectionsOnRemove() throws Exception {
- injectTestSystemOut();
-
Path typeFile = FS.getPath("type0.bin");
try (IgniteClient cli = Ignition.startClient(clientConfiguration())) {
@@ -393,11 +390,9 @@ public class GridCommandHandlerMetadataTest extends GridCommandHandlerClusterByC
*/
@Test
public void testDropJdbcThinConnectionsOnRemove() throws Exception {
- injectTestSystemOut();
-
Path typeFile = FS.getPath("type0.bin");
- try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1")) {
+ try (Connection conn = DriverManager.getConnection(jdbcThinUrl())) {
try (final Statement stmt = conn.createStatement()) {
stmt.execute("CREATE TABLE test(id INT PRIMARY KEY, objVal OTHER)");
@@ -441,6 +436,245 @@ public class GridCommandHandlerMetadataTest extends GridCommandHandlerClusterByC
}
/**
+ * Check the type successfully merged after remove-recreate-update operations.
+ * Steps:
+ * - creates several types with name 'TypeNameX' where X - some index.
+ * - removes the type by cmdline utility (store removed metadata to specified file).
+ * - checks that type removed.
+ * - creates new types with the same names but different field names
+ * - restores removed types from the file
+ * - checks all types successfully merged.
+ */
+ @Test
+ public void testTypeMergedAfterRemoveUpdate() {
+ String[] typeNames = new String[]{"TypeName0", "TypeName1"};
+
+ final int cnt = typeNames.length;
+
+ Object[] typeValues = new Object[]{0, LocalDate.now()};
+
+ int[] typeIds = new int[cnt];
+
+ repeat(cnt, i -> typeIds[i] = crd.binary().builder(typeNames[i])
+ .setField("fld0", typeValues[i])
+ .build().type().typeId());
+
+ Path[] typeBackups = new Path[typeNames.length];
+
+ try {
+ repeat(cnt, i -> {
+ Path path = FS.getPath(typeNames[i] + ".bin");
+
+ typeBackups[i] = path;
+
+ assertEquals(EXIT_CODE_OK, execute("--meta", "remove",
+ "--typeName", typeNames[i],
+ "--out", path.toString()));
+ });
+
+ assertEquals(EXIT_CODE_OK, execute("--meta", "list"));
+
+ String out = testOut.toString();
+
+ repeat(cnt, i -> assertNotContains(log, out, "typeName=" + typeNames[i]));
+
+ repeat(cnt, i -> crd.binary().builder(typeNames[i])
+ .setField("fld1", typeValues[i])
+ .build());
+
+ repeat(cnt, i -> assertEquals(EXIT_CODE_OK, execute("--meta", "update", "--in",
+ typeBackups[i].toString())));
+
+ repeat(cnt, i -> {
+ assertEquals(EXIT_CODE_OK, execute("--meta", "details", "--typeName", typeNames[i]));
+ checkTypeDetails(log, testOut.toString(), crd.context().cacheObjects().metadata(typeIds[i]));
+ });
+ }
+ finally {
+ repeat(cnt, i -> {
+ if (typeBackups[i] != null) {
+ try {
+ Files.deleteIfExists(typeBackups[i]);
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ }
+ }
+
+ /**
+ * Check the type can't be merged after remove-recreate-update operations
+ * with incompatible changes.
+ *
+ * Steps:
+ * - creates several types with name 'TypeNameX' where X - some index.
+ * - removes the type by cmdline utility (store removed metadata to specified file).
+ * - checks that type removed.
+ * - creates new types with the same names and same fields but different field types.
+ * - try to restores and verifies that it fails.
+ */
+ @Test
+ public void testTypeCantBeMergedAfterRemoveUpdateWithIncompatibleChanges() {
+ String[] typeNames = new String[]{"TypeName0", "TypeName1"};
+
+ final int cnt = typeNames.length;
+
+ Object[] typeValues = new Object[]{0, LocalDate.now()};
+
+ repeat(cnt, i -> crd.binary().builder(typeNames[i])
+ .setField("fld0", typeValues[i])
+ .build().type().typeId());
+
+ Path[] typeBackups = new Path[typeNames.length];
+
+ try {
+ repeat(cnt, i -> {
+ Path path = FS.getPath(typeNames[i] + ".bin");
+
+ typeBackups[i] = path;
+
+ assertEquals(EXIT_CODE_OK, execute("--meta", "remove",
+ "--typeName", typeNames[i],
+ "--out", path.toString()));
+ });
+
+ assertEquals(EXIT_CODE_OK, execute("--meta", "list"));
+
+ String out = testOut.toString();
+
+ repeat(cnt, i -> assertNotContains(log, out, "typeName=" + typeNames[i]));
+
+ repeat(cnt, i -> crd.binary().builder(typeNames[i])
+ .setField("fld0", typeValues[cnt - i - 1]) // swap values
+ .build());
+
+ repeat(cnt, i -> {
+ assertEquals(EXIT_CODE_UNEXPECTED_ERROR, execute("--meta", "update", "--in",
+ typeBackups[i].toString()));
+
+ assertContains(log, testOut.toString(), "The type of an existing field can not be changed");
+ });
+ }
+ finally {
+ repeat(cnt, i -> {
+ if (typeBackups[i] != null) {
+ try {
+ Files.deleteIfExists(typeBackups[i]);
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ }
+ }
+
+ /**
+ * Check that you could remove and update type when metadata store under load.
+ *
+ * Steps:
+ * - creates type with name 'Type0'.
+ * - start another thread which generates in an infinite loop new types.
+ * - removes the type by cmdline utility (store removed metadata to specified file).
+ * - checks that type removed.
+ * - restore the type.
+ * - checks that type restored.
+ */
+ @Test
+ public void testRemoveUpdateUnderLoad() throws Exception {
+ Path typeFile = FS.getPath("type0.bin");
+
+ try {
+ createType("Type0", 0);
+
+ assertEquals(EXIT_CODE_OK, execute("--meta", "list"));
+ assertContains(log, testOut.toString(), "typeName=Type0");
+
+ AtomicBoolean stop = new AtomicBoolean(false);
+
+ Thread t = new Thread(() -> {
+ long i = 1;
+
+ while (!stop.get())
+ createType("Type" + i++, i);
+ });
+ t.start();
+
+ assertEquals(EXIT_CODE_OK, execute("--meta", "remove",
+ "--typeName", "Type0",
+ "--out", typeFile.toString()));
+
+ assertEquals(EXIT_CODE_OK, execute("--meta", "list"));
+ assertNotContains(log, testOut.toString(), "typeName=Type0");
+
+ // Restore the metadata from file.
+ assertEquals(EXIT_CODE_OK, execute("--meta", "update", "--in", typeFile.toString()));
+
+ assertEquals(EXIT_CODE_OK, execute("--meta", "list"));
+ assertContains(log, testOut.toString(), "typeName=Type0");
+
+ stop.set(true);
+
+ t.join(getTestTimeout());
+ }
+ finally {
+ if (Files.exists(typeFile))
+ Files.delete(typeFile);
+ }
+ }
+
+ /**
+ * Checks metadata list/details behaviour after a type removing.
+ *
+ * Steps:
+ * - creates some type.
+ * - checks that metadata list|details command returns proper type information.
+ * - removes the type by cmdline utility.
+ * - checks list command output not contains removed type.
+ * - checks details command fails with "Type not found" error.
+ */
+ @Test
+ public void testMetadataListDetailsAfterTypeRemoving() throws IOException {
+ Path typeFile = FS.getPath("type0.bin");
+
+ try {
+ int typeId = createType("Type0", 0);
+
+ assertEquals(EXIT_CODE_OK, execute("--meta", "list"));
+ assertContains(log, testOut.toString(), "typeName=Type0");
+
+ assertEquals(EXIT_CODE_OK, execute("--meta", "details", "--typeName", "Type0"));
+ checkTypeDetails(log, testOut.toString(), crd.binary().type(typeId));
+
+ assertEquals(EXIT_CODE_OK, execute("--meta", "remove",
+ "--typeName", "Type0",
+ "--out", typeFile.toString()));
+
+ assertEquals(EXIT_CODE_OK, execute("--meta", "list"));
+ assertNotContains(log, testOut.toString(), "typeName=Type0");
+
+ assertEquals(EXIT_CODE_UNEXPECTED_ERROR, execute("--meta", "details", "--typeName", "Type0"));
+ assertContains(log, testOut.toString(), "type not found: " + typeId);
+ }
+ finally {
+ Files.deleteIfExists(typeFile);
+ }
+ }
+
+ /**
+ * Repeats {@code cons} {@code cnt} times.
+ *
+ * @param cnt Count.
+ * @param cons Cons.
+ */
+ private void repeat(int cnt, Consumer<Integer> cons) {
+ for (int i = 0; i < cnt; i++)
+ cons.accept(i);
+ }
+
+ /**
* @param t Binary type.
*/
private void checkTypeDetails(@Nullable IgniteLogger log, String cmdOut, BinaryType t) {
@@ -459,26 +693,33 @@ public class GridCommandHandlerMetadataTest extends GridCommandHandlerClusterByC
* @param typeName Type name
* @param val Field value.
*/
- void createType(String typeName, Object val) {
- createType(crd.binary(), typeName, val);
+ int createType(String typeName, Object val) {
+ return createType(crd.binary(), typeName, val);
}
/**
* @param typeName Type name.
* @param val Field value.
*/
- void createType(IgniteBinary bin, String typeName, Object val) {
- BinaryObjectBuilder bob = bin.builder(typeName);
- bob.setField("fld", val);
- bob.build();
+ int createType(IgniteBinary bin, String typeName, Object val) {
+ return bin.builder(typeName)
+ .setField("fld", val)
+ .build()
+ .type()
+ .typeId();
}
/** */
- private ClientConfiguration clientConfiguration() {
+ protected ClientConfiguration clientConfiguration() {
return new ClientConfiguration()
.setAddresses("127.0.0.1:10800");
}
+ /** */
+ protected String jdbcThinUrl() {
+ return "jdbc:ignite:thin://127.0.0.1";
+ }
+
/**
*
*/
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/meta/tasks/MetadataInfoTask.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/meta/tasks/MetadataInfoTask.java
index f543921..4c0dfce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/meta/tasks/MetadataInfoTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/meta/tasks/MetadataInfoTask.java
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.List;
import org.apache.ignite.IgniteException;
import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.binary.BinaryMetadata;
import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.visor.VisorJob;
@@ -28,7 +29,7 @@ import org.apache.ignite.internal.visor.VisorMultiNodeTask;
import org.jetbrains.annotations.Nullable;
/**
- * Task for getting binary metadata.
+ * Task for MetadataListCommand and MetadataDetailsCommand commands.
*/
@GridInternal
public class MetadataInfoTask extends VisorMultiNodeTask<MetadataTypeArgs, MetadataListResult, MetadataListResult> {
@@ -55,7 +56,7 @@ public class MetadataInfoTask extends VisorMultiNodeTask<MetadataTypeArgs, Metad
}
/**
- * Job for getting binary metadata.
+ * Job for {@link CheckIndexInlineSizes} command.
*/
private static class MetadataListJob extends VisorJob<MetadataTypeArgs, MetadataListResult> {
/** */
@@ -80,8 +81,12 @@ public class MetadataInfoTask extends VisorMultiNodeTask<MetadataTypeArgs, Metad
// returns specified metadata
int typeId = arg.typeId(ignite.context());
- return new MetadataListResult(Collections.singleton(
- ((CacheObjectBinaryProcessorImpl)ignite.context().cacheObjects()).binaryMetadata(typeId)));
+ BinaryMetadata binMeta = ((CacheObjectBinaryProcessorImpl)ignite.context().cacheObjects()).binaryMetadata(typeId);
+
+ if (binMeta == null)
+ throw new IgniteException("Failed to get metadata, type not found: " + typeId);
+
+ return new MetadataListResult(Collections.singleton(binMeta));
}
}
}