You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sk...@apache.org on 2021/11/09 16:07:13 UTC
[ignite-3] branch main updated: IGNITE-15881 Use direct properties
in the TableManager. Fixes #431
This is an automated email from the ASF dual-hosted git repository.
sk0x50 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new e3c2d46 IGNITE-15881 Use direct properties in the TableManager. Fixes #431
e3c2d46 is described below
commit e3c2d4613e2c69206b2f0e4e7d781c0817f11c5b
Author: Alexander Polovtcev <al...@gmail.com>
AuthorDate: Tue Nov 9 19:06:39 2021 +0300
IGNITE-15881 Use direct properties in the TableManager. Fixes #431
Signed-off-by: Slava Koptilin <sl...@gmail.com>
---
.../org/apache/ignite/internal/app/IgniteImpl.java | 1 -
.../internal/table/distributed/TableManager.java | 84 ++-------------
.../ignite/internal/table/TableManagerTest.java | 115 +++++----------------
3 files changed, 34 insertions(+), 166 deletions(-)
diff --git a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java
index 7e71299..6c101d9 100644
--- a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java
+++ b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java
@@ -206,7 +206,6 @@ public class IgniteImpl implements Ignite {
raftMgr,
baselineMgr,
clusterSvc.topologyService(),
- metaStorageMgr,
getPartitionsStorePath(workDir)
);
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
index b720abb..5c8da9a 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -55,13 +54,10 @@ import org.apache.ignite.internal.configuration.schema.ExtendedTableChange;
import org.apache.ignite.internal.configuration.schema.ExtendedTableConfiguration;
import org.apache.ignite.internal.configuration.schema.ExtendedTableView;
import org.apache.ignite.internal.configuration.schema.SchemaView;
-import org.apache.ignite.internal.configuration.tree.NamedListNode;
import org.apache.ignite.internal.configuration.util.ConfigurationUtil;
import org.apache.ignite.internal.manager.EventListener;
import org.apache.ignite.internal.manager.IgniteComponent;
import org.apache.ignite.internal.manager.Producer;
-import org.apache.ignite.internal.metastorage.MetaStorageManager;
-import org.apache.ignite.internal.metastorage.client.Entry;
import org.apache.ignite.internal.raft.Loza;
import org.apache.ignite.internal.schema.SchemaDescriptor;
import org.apache.ignite.internal.schema.SchemaUtils;
@@ -79,10 +75,7 @@ import org.apache.ignite.internal.table.distributed.storage.InternalTableImpl;
import org.apache.ignite.internal.table.event.TableEvent;
import org.apache.ignite.internal.table.event.TableEventParameters;
import org.apache.ignite.internal.util.ByteUtils;
-import org.apache.ignite.internal.util.Cursor;
import org.apache.ignite.internal.util.IgniteSpinBusyLock;
-import org.apache.ignite.lang.ByteArray;
-import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteException;
import org.apache.ignite.lang.IgniteInternalCheckedException;
import org.apache.ignite.lang.IgniteInternalException;
@@ -109,19 +102,10 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp
/** The logger. */
private static final IgniteLogger LOG = IgniteLogger.forClass(TableManager.class);
- /**
- *
- */
+ /** */
private static final int INITIAL_SCHEMA_VERSION = 1;
- /** Public prefix for metastorage. */
- // TODO: IGNITE-15412 Remove after implementation. Configuration manager will be used to retrieve distributed values
- // TODO: instead of metastorage manager.
- private static final String PUBLIC_PREFIX = "dst-cfg.table.tables.";
-
- /**
- *
- */
+ /** */
private static final IgniteUuidGenerator TABLE_ID_GENERATOR = new IgniteUuidGenerator(UUID.randomUUID(), 0);
/** Tables configuration. */
@@ -136,11 +120,6 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp
/** Baseline manager. */
private final BaselineManager baselineMgr;
- // TODO: IGNITE-15412 Remove after implementation. Configuration manager will be used to retrieve distributed values
- // TODO: instead of metastorage manager.
- /** Meta storage service. */
- private final MetaStorageManager metaStorageMgr;
-
/** Storage engine instance. Only one type is available right now, which is the {@link RocksDbStorageEngine}. */
private final StorageEngine engine;
@@ -163,7 +142,7 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp
private final IgniteSpinBusyLock busyLock = new IgniteSpinBusyLock();
/** Prevents double stopping the component. */
- AtomicBoolean stopGuard = new AtomicBoolean();
+ private final AtomicBoolean stopGuard = new AtomicBoolean();
/**
* Creates a new table manager.
@@ -172,7 +151,6 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp
* @param dataStorageCfg Data storage configuration.
* @param raftMgr Raft manager.
* @param baselineMgr Baseline manager.
- * @param metaStorageMgr Meta storage manager.
* @param partitionsStoreDir Partitions store directory.
*/
public TableManager(
@@ -181,14 +159,12 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp
Loza raftMgr,
BaselineManager baselineMgr,
TopologyService topologyService,
- MetaStorageManager metaStorageMgr,
Path partitionsStoreDir
) {
this.tablesCfg = tablesCfg;
this.dataStorageCfg = dataStorageCfg;
this.raftMgr = raftMgr;
this.baselineMgr = baselineMgr;
- this.metaStorageMgr = metaStorageMgr;
this.partitionsStoreDir = partitionsStoreDir;
netAddrResolver = addr -> {
@@ -966,7 +942,7 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp
* @return Future representing pending completion of the operation.
*/
private CompletableFuture<List<Table>> tablesAsyncInternal() {
- var tableNames = tableNamesConfigured();
+ List<String> tableNames = tableNamesConfigured();
var tableFuts = new CompletableFuture[tableNames.size()];
var i = 0;
@@ -994,56 +970,12 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp
}
/**
- * Collects a set of table names from the distributed configuration storage.
- *
- * @return A set of table names.
- */
- // TODO: IGNITE-15412 Configuration manager will be used to retrieve distributed values
- // TODO: instead of metastorage manager. That will automatically resolve several bugs of current implementation.
- private Set<String> tableNamesConfigured() {
- IgniteBiTuple<ByteArray, ByteArray> range = toRange(new ByteArray(PUBLIC_PREFIX));
-
- Set<String> tableNames = new HashSet<>();
-
- try (Cursor<Entry> cursor = metaStorageMgr.range(range.get1(), range.get2())) {
- while (cursor.hasNext()) {
- Entry entry = cursor.next();
-
- List<String> keySplit = ConfigurationUtil.split(entry.key().toString());
-
- if (keySplit.size() == 5 && NamedListNode.NAME.equals(keySplit.get(4))) {
- @Nullable byte[] value = entry.value();
- if (value != null) {
- tableNames.add(ByteUtils.fromBytes(value).toString());
- }
- }
- }
- } catch (Exception e) {
- LOG.error("Can't get table names.", e);
- }
-
- return tableNames;
- }
-
- /**
- * Transforms a prefix bytes to range. This method should be replaced to direct call of range by prefix in Meta storage manager when it
- * will be implemented.
+ * Collects a list of table names from the distributed configuration storage.
*
- * @param prefixKey Prefix bytes.
- * @return Tuple with left and right borders for range.
+ * @return A list of table names.
*/
- // TODO: IGNITE-15412 Remove after implementation. Configuration manager will be used to retrieve distributed values
- // TODO: instead of metastorage manager.
- private IgniteBiTuple<ByteArray, ByteArray> toRange(ByteArray prefixKey) {
- var bytes = Arrays.copyOf(prefixKey.bytes(), prefixKey.bytes().length);
-
- if (bytes[bytes.length - 1] != Byte.MAX_VALUE) {
- bytes[bytes.length - 1]++;
- } else {
- bytes = Arrays.copyOf(bytes, bytes.length + 1);
- }
-
- return new IgniteBiTuple<>(prefixKey, new ByteArray(bytes));
+ private List<String> tableNamesConfigured() {
+ return ConfigurationUtil.directValue(tablesCfg.tables()).namedListKeys();
}
/** {@inheritDoc} */
diff --git a/modules/table/src/test/java/org/apache/ignite/internal/table/TableManagerTest.java b/modules/table/src/test/java/org/apache/ignite/internal/table/TableManagerTest.java
index ebb1a23..0d4a80d 100644
--- a/modules/table/src/test/java/org/apache/ignite/internal/table/TableManagerTest.java
+++ b/modules/table/src/test/java/org/apache/ignite/internal/table/TableManagerTest.java
@@ -24,11 +24,9 @@ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.times;
@@ -42,7 +40,6 @@ import java.util.List;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Phaser;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
import org.apache.ignite.configuration.schemas.store.DataStorageConfiguration;
import org.apache.ignite.configuration.schemas.table.TableChange;
@@ -52,18 +49,12 @@ import org.apache.ignite.internal.baseline.BaselineManager;
import org.apache.ignite.internal.configuration.schema.ExtendedTableConfigurationSchema;
import org.apache.ignite.internal.configuration.testframework.ConfigurationExtension;
import org.apache.ignite.internal.configuration.testframework.InjectConfiguration;
-import org.apache.ignite.internal.configuration.tree.NamedListNode;
-import org.apache.ignite.internal.metastorage.MetaStorageManager;
-import org.apache.ignite.internal.metastorage.client.Entry;
import org.apache.ignite.internal.raft.Loza;
import org.apache.ignite.internal.schema.SchemaDescriptor;
import org.apache.ignite.internal.schema.SchemaUtils;
import org.apache.ignite.internal.schema.configuration.SchemaConfigurationConverter;
import org.apache.ignite.internal.table.distributed.TableManager;
import org.apache.ignite.internal.testframework.IgniteAbstractTest;
-import org.apache.ignite.internal.util.ByteUtils;
-import org.apache.ignite.internal.util.Cursor;
-import org.apache.ignite.lang.ByteArray;
import org.apache.ignite.lang.IgniteException;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.lang.IgniteUuidGenerator;
@@ -95,9 +86,6 @@ import org.mockito.quality.Strictness;
@ExtendWith({MockitoExtension.class, ConfigurationExtension.class})
@MockitoSettings(strictness = Strictness.LENIENT)
public class TableManagerTest extends IgniteAbstractTest {
- /** Public prefix for metastorage. */
- private static final String PUBLIC_PREFIX = "dst-cfg.table.tables.";
-
/** The name of the table which is statically configured. */
private static final String STATIC_TABLE_NAME = "t1";
@@ -114,22 +102,18 @@ public class TableManagerTest extends IgniteAbstractTest {
private static final String NODE_NAME = "node1";
/** Count of replicas. */
- public static final int REPLICAS = 1;
-
- /** MetaStorage manager. */
- @Mock(lenient = true)
- private MetaStorageManager mm;
+ private static final int REPLICAS = 1;
/** Schema manager. */
- @Mock(lenient = true)
+ @Mock
private BaselineManager bm;
/** Topology service. */
- @Mock(lenient = true)
+ @Mock
private TopologyService ts;
/** Raft manager. */
- @Mock(lenient = true)
+ @Mock
private Loza rm;
/** Tables configuration. */
@@ -177,7 +161,6 @@ public class TableManagerTest extends IgniteAbstractTest {
rm,
bm,
ts,
- mm,
workDir
);
@@ -323,7 +306,7 @@ public class TableManagerTest extends IgniteAbstractTest {
* Instantiates a table and prepares Table manager.
*/
@Test
- public void testGetTableDuringCreation() throws Exception {
+ public void testGetTableDuringCreation() {
TableDefinition scmTbl = SchemaBuilders.tableBuilder("PUBLIC", DYNAMIC_TABLE_FOR_DROP_NAME).columns(
SchemaBuilders.column("key", ColumnType.INT64).asNonNull().build(),
SchemaBuilders.column("val", ColumnType.INT64).asNullable().build()
@@ -427,8 +410,6 @@ public class TableManagerTest extends IgniteAbstractTest {
new NetworkAddress("localhost", 47500)
));
- AtomicBoolean tableCreatedFlag = new AtomicBoolean();
-
try (MockedStatic<SchemaUtils> schemaServiceMock = mockStatic(SchemaUtils.class)) {
schemaServiceMock.when(() -> SchemaUtils.prepareSchemaDescriptor(anyInt(), any()))
.thenReturn(mock(SchemaDescriptor.class));
@@ -447,78 +428,35 @@ public class TableManagerTest extends IgniteAbstractTest {
TableManager tableManager = createTableManager(tblManagerFut);
- TableImpl tbl2 = null;
-
- try {
- when(mm.range(eq(new ByteArray(PUBLIC_PREFIX)), any())).thenAnswer(invocation -> {
- Cursor<Entry> cursor = mock(Cursor.class);
-
- when(cursor.hasNext()).thenReturn(false);
-
- return cursor;
- });
-
- final int tablesBeforeCreation = tableManager.tables().size();
-
- tblsCfg.tables().listen(ctx -> {
- boolean createTbl = ctx.newValue().get(tableDefinition.canonicalName()) != null
- && ctx.oldValue().get(tableDefinition.canonicalName()) == null;
-
- boolean dropTbl = ctx.oldValue().get(tableDefinition.canonicalName()) != null
- && ctx.newValue().get(tableDefinition.canonicalName()) == null;
-
- if (!createTbl && !dropTbl) {
- return CompletableFuture.completedFuture(null);
- }
-
- tableCreatedFlag.set(createTbl);
-
- try {
- when(mm.range(eq(new ByteArray(PUBLIC_PREFIX)), any())).thenAnswer(invocation -> {
- AtomicBoolean firstRecord = new AtomicBoolean(createTbl);
-
- Cursor<Entry> cursor = mock(Cursor.class);
+ final int tablesBeforeCreation = tableManager.tables().size();
- when(cursor.hasNext()).thenAnswer(hasNextInvocation ->
- firstRecord.compareAndSet(true, false));
+ tblsCfg.tables().listen(ctx -> {
+ boolean createTbl = ctx.newValue().get(tableDefinition.canonicalName()) != null
+ && ctx.oldValue().get(tableDefinition.canonicalName()) == null;
- Entry mockEntry = mock(Entry.class);
-
- when(mockEntry.key()).thenReturn(new ByteArray(PUBLIC_PREFIX + "uuid." + NamedListNode.NAME));
-
- when(mockEntry.value()).thenReturn(ByteUtils.toBytes(tableDefinition.canonicalName()));
-
- when(cursor.next()).thenReturn(mockEntry);
-
- return cursor;
- });
- } catch (NodeStoppingException e) {
- log.error("Node was stopped during table creation.", e);
-
- fail();
- }
-
- if (phaser != null) {
- phaser.arriveAndAwaitAdvance();
- }
+ boolean dropTbl = ctx.oldValue().get(tableDefinition.canonicalName()) != null
+ && ctx.newValue().get(tableDefinition.canonicalName()) == null;
+ if (!createTbl && !dropTbl) {
return CompletableFuture.completedFuture(null);
- });
+ }
- tbl2 = (TableImpl) tableManager.createTable(tableDefinition.canonicalName(),
- tblCh -> SchemaConfigurationConverter.convert(tableDefinition, tblCh)
- .changeReplicas(REPLICAS)
- .changePartitions(PARTITIONS)
- );
+ if (phaser != null) {
+ phaser.arriveAndAwaitAdvance();
+ }
- assertNotNull(tbl2);
+ return CompletableFuture.completedFuture(null);
+ });
+
+ TableImpl tbl2 = (TableImpl) tableManager.createTable(tableDefinition.canonicalName(),
+ tblCh -> SchemaConfigurationConverter.convert(tableDefinition, tblCh)
+ .changeReplicas(REPLICAS)
+ .changePartitions(PARTITIONS)
+ );
- assertEquals(tablesBeforeCreation + 1, tableManager.tables().size());
- } catch (NodeStoppingException e) {
- log.error("Node was stopped during table creation.", e);
+ assertNotNull(tbl2);
- fail();
- }
+ assertEquals(tablesBeforeCreation + 1, tableManager.tables().size());
return tbl2;
}
@@ -537,7 +475,6 @@ public class TableManagerTest extends IgniteAbstractTest {
rm,
bm,
ts,
- mm,
workDir
);