You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by tk...@apache.org on 2023/09/14 07:12:53 UTC
[ignite-3] branch catalog-feature updated (81cefdd6a4 -> 96f3355cac)
This is an automated email from the ASF dual-hosted git repository.
tkalkirill pushed a change to branch catalog-feature
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
from 81cefdd6a4 Catalog feature 13 09 2023 (#2586)
add 87019581cf IGNITE-20379 Concurrency issues with write intent tracking (#2562)
add edf625a5b4 IGNITE-20337 Ensure QueryChecker is properly used in tests (#2539)
add a227876c0c IGNITE-20187 Pending assignments recovery on Table Manager start (#2517)
add 1c0795eaa2 IGNITE-20345 Replace key-only rows with BinaryTuples
add cb9966e5cb IGNITE-20264 Sql. Races in BaseAggregateTest::countOfEmptyWithRewind (#2565)
add 4977689b14 IGNITE-20239: Sql. Remove row type transformations from SetOpConverterRule (#2558)
add dfd111047e IGNITE-20200 Revert PR #2442 (#2569)
add 407419d900 IGNITE-20391 Return future metastore event processing from UpdateListener#onUpdate (#2570)
add 706ce62713 IGNITE-20375 Remove arrayHeader and mapHeader usage from client protocol (#2564)
add a1172c844f IGNITE-20335 Move IgniteRunner class into internal package (#2567)
add da7bcdd9d9 IGNITE-20332 Fix flaky test testScaleUpTriggeredByFilterUpdateIsRestoredAfterRestart (#2554)
add 9b8ad39d38 IGNITE-20374 Disable tests related to POSIX file permissions on Windows (#2560)
add 6e67aa031e IGNITE-20371 Move handlers of index-related commands from CatalogManager (#2563)
add 39a3ac4ba6 IGNITE-18856 Switch primary replica calls from Raft leader to primary replica (#2488)
add 0d1ae2d1d6 IGNITE-20044 Validate schema eligibility on each read/write operation in an RW transaction (#2566)
add 9e21ae612f IGNITE-19106 Sql. Fix partially broken column namings (#2552)
add 5f1ac53b45 IGNITE-20396 Correct implementation of PartitionReplicaListener#resolvePk (#2578)
add 0d12df82e2 IGNITE-20280 Lease updater exception is not logged on initialization (#2579)
add 63d938a005 IGNITE-20017: Introduce new command to catalog to create system view (#2528)
add cf946e9f94 IGNITE-20398 Filter null values for any cfg (#2580)
add a7a6e4e93e IGNITE-20389 Sql. Avoid unconditional conversion of table row in sql engine (#2568)
add 6bc603deb6 IGNITE-20355 Fix failing RO tests in TxLocalTest (#2582)
new 96a9ed49c2 Merge branch 'ai-main' into catalog-feature
new 96f3355cac catalog-feature after merge ai-main
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../internal/catalog/CatalogManagerImpl.java | 17 +-
.../catalog/CatalogParamsValidationUtils.java | 6 +-
.../ignite/internal/catalog/CatalogService.java | 2 +
.../catalog/CatalogValidationException.java | 2 +-
.../commands/AbstractCreateIndexCommand.java | 6 +-
.../commands/AlterTableAddColumnCommand.java | 2 +-
.../commands/AlterTableAlterColumnCommand.java | 2 +-
.../commands/AlterTableDropColumnCommand.java | 2 +-
.../internal/catalog/commands/CatalogUtils.java | 22 ++
.../internal/catalog/commands/ColumnParams.java | 20 ++
.../catalog/commands/CreateSystemViewCommand.java | 206 +++++++++++
...er.java => CreateSystemViewCommandBuilder.java} | 40 ++-
.../catalog/commands/CreateTableCommand.java | 10 +-
.../catalog/commands/DropIndexCommand.java | 2 +-
.../catalog/commands/DropTableCommand.java | 4 +-
.../descriptors/CatalogObjectDescriptor.java | 5 +-
.../descriptors/CatalogSchemaDescriptor.java | 28 +-
.../descriptors/CatalogSystemViewDescriptor.java | 105 ++++++
.../internal/catalog/events/CatalogEvent.java | 5 +-
...s.java => CreateSystemViewEventParameters.java} | 20 +-
.../internal/catalog/storage/AlterColumnEntry.java | 18 +-
.../internal/catalog/storage/DropColumnsEntry.java | 17 +-
.../internal/catalog/storage/DropIndexEntry.java | 19 +-
.../internal/catalog/storage/DropTableEntry.java | 19 +-
.../internal/catalog/storage/NewColumnsEntry.java | 16 +-
.../internal/catalog/storage/NewIndexEntry.java | 19 +-
.../catalog/storage/NewSystemViewEntry.java | 99 ++++++
.../internal/catalog/storage/NewTableEntry.java | 26 +-
.../internal/catalog/CatalogManagerSelfTest.java | 43 ++-
.../internal/catalog/CatalogSystemViewTest.java | 381 +++++++++++++++++++++
.../commands/AbstractCommandValidationTest.java | 10 +-
.../CreateSystemViewCommandValidationTest.java | 141 ++++++++
.../internal/catalog/BaseCatalogManagerTest.java | 12 +-
.../processor/ConfigurationProcessor.java | 3 +-
.../configuration/ConfigurationTreeGenerator.java | 10 +-
.../ConfigurationNotificationUtils.java | 42 ++-
.../notifications/ConfigurationNotifier.java | 44 +--
.../configuration/util/ConfigurationUtil.java | 138 +++++++-
.../configuration/util/ConfigurationUtilTest.java | 106 ++++++
.../ignite/internal/util/CollectionUtils.java | 184 ++--------
.../ignite/internal/util/CollectionUtilsTest.java | 88 +----
.../persistence/compaction/Compactor.java | 2 +-
.../persistence/store/FilePageStoreManager.java | 5 +-
.../persistence/store/GroupPageStoresMap.java | 7 +-
.../store/FilePageStoreManagerTest.java | 8 +-
.../persistence/store/GroupPageStoresMapTest.java | 9 +-
.../placementdriver/PlacementDriverManager.java | 2 +
.../internal/benchmark/SqlOneNodeBenchmark.java | 147 ++++++++
.../ignite/internal/schema/BinaryTupleSchema.java | 3 +-
.../ignite/internal/schema/SchemaTestUtils.java | 5 +-
modules/sql-engine/build.gradle | 1 +
.../internal/sql/api/AsyncResultSetImpl.java | 5 +
.../engine/exec/ExecutableTableRegistryImpl.java | 25 +-
.../sql/engine/exec/ScannableTableImpl.java | 18 +-
.../sql/engine/exec/TableRowConverter.java | 15 +-
...cleAware.java => TableRowConverterFactory.java} | 21 +-
.../sql/engine/exec/TableRowConverterImpl.java | 86 +++--
.../sql/engine/exec/UpdatableTableImpl.java | 2 +-
.../sql/engine/schema/TableDescriptor.java | 2 +-
.../sql/engine/schema/TableDescriptorImpl.java | 7 +
.../sql/engine/util/AbstractProjectedTuple.java | 227 ++++++++++++
.../util/FieldDeserializingProjectedTuple.java | 75 ++++
.../sql/engine/util/FormatAwareProjectedTuple.java | 93 +++++
.../exec/ExecutableTableRegistrySelfTest.java | 3 +
.../engine/exec/rel/ScannableTableSelfTest.java | 31 +-
.../exec/rel/TableScanNodeExecutionTest.java | 5 +-
.../sql/engine/planner/AbstractPlannerTest.java | 6 +
.../engine/schema/CatalogSqlSchemaManagerTest.java | 5 +-
.../sql/engine/util/ProjectedTupleTest.java | 179 ++++++++++
.../apache/ignite/internal/table/TxLocalTest.java | 13 +-
.../table/impl/DummyInternalTableImpl.java | 13 +-
71 files changed, 2389 insertions(+), 572 deletions(-)
create mode 100644 modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/CreateSystemViewCommand.java
copy modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/{CreateTableCommandBuilder.java => CreateSystemViewCommandBuilder.java} (52%)
create mode 100644 modules/catalog/src/main/java/org/apache/ignite/internal/catalog/descriptors/CatalogSystemViewDescriptor.java
copy modules/catalog/src/main/java/org/apache/ignite/internal/catalog/events/{CreateIndexEventParameters.java => CreateSystemViewEventParameters.java} (61%)
create mode 100644 modules/catalog/src/main/java/org/apache/ignite/internal/catalog/storage/NewSystemViewEntry.java
create mode 100644 modules/catalog/src/test/java/org/apache/ignite/internal/catalog/CatalogSystemViewTest.java
create mode 100644 modules/catalog/src/test/java/org/apache/ignite/internal/catalog/commands/CreateSystemViewCommandValidationTest.java
create mode 100644 modules/runner/src/integrationTest/java/org/apache/ignite/internal/benchmark/SqlOneNodeBenchmark.java
copy modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/{LifecycleAware.java => TableRowConverterFactory.java} (72%)
create mode 100644 modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/AbstractProjectedTuple.java
create mode 100644 modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/FieldDeserializingProjectedTuple.java
create mode 100644 modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/FormatAwareProjectedTuple.java
create mode 100644 modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/util/ProjectedTupleTest.java
[ignite-3] 01/02: Merge branch 'ai-main' into catalog-feature
Posted by tk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tkalkirill pushed a commit to branch catalog-feature
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit 96a9ed49c214af3b6df22032b04572559f48c3da
Merge: 81cefdd6a4 6bc603deb6
Author: Kirill Tkalenko <tk...@yandex.ru>
AuthorDate: Thu Sep 14 09:50:11 2023 +0300
Merge branch 'ai-main' into catalog-feature
.../internal/catalog/CatalogManagerImpl.java | 17 +-
.../catalog/CatalogParamsValidationUtils.java | 6 +-
.../ignite/internal/catalog/CatalogService.java | 2 +
.../catalog/CatalogValidationException.java | 2 +-
.../commands/AbstractCreateIndexCommand.java | 5 +-
.../commands/AlterTableAddColumnCommand.java | 2 +-
.../commands/AlterTableAlterColumnCommand.java | 2 +-
.../commands/AlterTableDropColumnCommand.java | 2 +-
.../internal/catalog/commands/CatalogUtils.java | 20 ++
.../internal/catalog/commands/ColumnParams.java | 20 ++
.../catalog/commands/CreateSystemViewCommand.java | 206 +++++++++++
.../commands/CreateSystemViewCommandBuilder.java | 62 ++++
.../catalog/commands/CreateTableCommand.java | 10 +-
.../catalog/commands/DropIndexCommand.java | 2 +-
.../catalog/commands/DropTableCommand.java | 4 +-
.../descriptors/CatalogObjectDescriptor.java | 5 +-
.../descriptors/CatalogSchemaDescriptor.java | 28 +-
.../descriptors/CatalogSystemViewDescriptor.java | 105 ++++++
.../internal/catalog/events/CatalogEvent.java | 5 +-
.../events/CreateSystemViewEventParameters.java | 48 +++
.../internal/catalog/storage/AlterColumnEntry.java | 18 +-
.../internal/catalog/storage/DropColumnsEntry.java | 17 +-
.../internal/catalog/storage/DropIndexEntry.java | 19 +-
.../internal/catalog/storage/DropTableEntry.java | 19 +-
.../internal/catalog/storage/NewColumnsEntry.java | 16 +-
.../internal/catalog/storage/NewIndexEntry.java | 19 +-
.../catalog/storage/NewSystemViewEntry.java | 99 ++++++
.../internal/catalog/storage/NewTableEntry.java | 26 +-
.../internal/catalog/CatalogManagerSelfTest.java | 43 ++-
.../internal/catalog/CatalogSystemViewTest.java | 381 +++++++++++++++++++++
.../commands/AbstractCommandValidationTest.java | 10 +-
.../CreateSystemViewCommandValidationTest.java | 141 ++++++++
.../internal/catalog/BaseCatalogManagerTest.java | 12 +-
.../processor/ConfigurationProcessor.java | 3 +-
.../configuration/ConfigurationTreeGenerator.java | 9 +-
.../ConfigurationNotificationUtils.java | 42 ++-
.../notifications/ConfigurationNotifier.java | 44 +--
.../configuration/util/ConfigurationUtil.java | 138 +++++++-
.../configuration/util/ConfigurationUtilTest.java | 106 ++++++
.../ignite/internal/util/CollectionUtils.java | 184 ++--------
.../ignite/internal/util/CollectionUtilsTest.java | 88 +----
.../persistence/compaction/Compactor.java | 2 +-
.../persistence/store/FilePageStoreManager.java | 5 +-
.../persistence/store/GroupPageStoresMap.java | 7 +-
.../store/FilePageStoreManagerTest.java | 8 +-
.../persistence/store/GroupPageStoresMapTest.java | 9 +-
.../placementdriver/PlacementDriverManager.java | 2 +
.../internal/benchmark/SqlOneNodeBenchmark.java | 147 ++++++++
.../internal/runner/app/ItTablesApiTest.java | 1 +
.../ignite/internal/schema/BinaryTupleSchema.java | 3 +-
.../ignite/internal/schema/SchemaTestUtils.java | 5 +-
modules/sql-engine/build.gradle | 1 +
.../internal/sql/api/AsyncResultSetImpl.java | 5 +
.../engine/exec/ExecutableTableRegistryImpl.java | 9 +-
.../sql/engine/exec/ScannableTableImpl.java | 18 +-
.../sql/engine/exec/TableRowConverter.java | 15 +-
...onverter.java => TableRowConverterFactory.java} | 24 +-
.../sql/engine/exec/TableRowConverterImpl.java | 85 +++--
.../sql/engine/exec/UpdatableTableImpl.java | 2 +-
.../sql/engine/schema/TableDescriptor.java | 2 +-
.../sql/engine/schema/TableDescriptorImpl.java | 7 +
.../sql/engine/util/AbstractProjectedTuple.java | 227 ++++++++++++
.../util/FieldDeserializingProjectedTuple.java | 75 ++++
.../sql/engine/util/FormatAwareProjectedTuple.java | 93 +++++
.../exec/ExecutableTableRegistrySelfTest.java | 3 +
.../engine/exec/rel/ScannableTableSelfTest.java | 31 +-
.../exec/rel/TableScanNodeExecutionTest.java | 5 +-
.../sql/engine/planner/AbstractPlannerTest.java | 6 +
.../engine/schema/CatalogSqlSchemaManagerTest.java | 5 +-
.../sql/engine/util/ProjectedTupleTest.java | 179 ++++++++++
.../internal/table/distributed/TableManager.java | 3 +-
.../replicator/PartitionReplicaListener.java | 1 +
.../apache/ignite/internal/table/TxLocalTest.java | 13 +-
.../table/impl/DummyInternalTableImpl.java | 15 +-
74 files changed, 2464 insertions(+), 536 deletions(-)
diff --cc modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/AbstractCreateIndexCommand.java
index 70c2225baf,471c3332e1..bee55a4c18
--- a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/AbstractCreateIndexCommand.java
+++ b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/AbstractCreateIndexCommand.java
@@@ -17,7 -17,7 +17,8 @@@
package org.apache.ignite.internal.catalog.commands;
+import static org.apache.ignite.internal.catalog.CatalogParamsValidationUtils.ensureNoTableOrIndexExistsWithGivenName;
+ import static org.apache.ignite.internal.catalog.CatalogParamsValidationUtils.ensureNoTableIndexOrSysViewExistsWithGivenName;
import static org.apache.ignite.internal.catalog.CatalogParamsValidationUtils.validateIdentifier;
import static org.apache.ignite.internal.catalog.commands.CatalogUtils.schemaOrThrow;
import static org.apache.ignite.internal.catalog.commands.CatalogUtils.tableOrThrow;
diff --cc modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/CatalogUtils.java
index f8a6553284,fae6e961b5..d34b705ed9
--- a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/CatalogUtils.java
+++ b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/CatalogUtils.java
@@@ -17,10 -17,13 +17,11 @@@
package org.apache.ignite.internal.catalog.commands;
-import static java.util.stream.Collectors.toList;
import static org.apache.ignite.lang.IgniteStringFormatter.format;
+ import java.util.Collection;
import java.util.EnumMap;
import java.util.EnumSet;
-import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
diff --cc modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItTablesApiTest.java
index ac1c41e638,0acac49c2c..9947f25bd8
--- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItTablesApiTest.java
+++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItTablesApiTest.java
@@@ -39,17 -44,23 +39,18 @@@ import java.util.stream.IntStream
import org.apache.ignite.Ignite;
import org.apache.ignite.IgnitionManager;
import org.apache.ignite.InitParameters;
+ import org.apache.ignite.internal.catalog.IndexExistsValidationException;
-import org.apache.ignite.internal.schema.testutils.builder.SchemaBuilders;
-import org.apache.ignite.internal.schema.testutils.definition.ColumnDefinition;
-import org.apache.ignite.internal.schema.testutils.definition.ColumnType;
+import org.apache.ignite.internal.catalog.CatalogValidationException;
+import org.apache.ignite.internal.catalog.IndexExistsValidationException;
+import org.apache.ignite.internal.catalog.TableExistsValidationException;
+import org.apache.ignite.internal.catalog.TableNotFoundValidationException;
import org.apache.ignite.internal.table.IgniteTablesInternal;
import org.apache.ignite.internal.table.TableImpl;
-import org.apache.ignite.internal.table.distributed.TableManager;
import org.apache.ignite.internal.test.WatchListenerInhibitor;
import org.apache.ignite.internal.testframework.IgniteAbstractTest;
-import org.apache.ignite.internal.testframework.IgniteTestUtils;
import org.apache.ignite.internal.testframework.TestIgnitionManager;
import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.lang.ColumnAlreadyExistsException;
-import org.apache.ignite.lang.IndexAlreadyExistsException;
import org.apache.ignite.lang.NodeStoppingException;
-import org.apache.ignite.lang.TableAlreadyExistsException;
-import org.apache.ignite.lang.TableNotFoundException;
import org.apache.ignite.sql.Session;
import org.apache.ignite.table.Table;
import org.apache.ignite.table.Tuple;
diff --cc modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterImpl.java
index 7fe1f5f3e0,355a0b42c4..8801dc69a0
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterImpl.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterImpl.java
@@@ -18,17 -18,16 +18,19 @@@
package org.apache.ignite.internal.sql.engine.exec;
import java.util.BitSet;
- import org.apache.ignite.internal.binarytuple.BinaryTupleBuilder;
import org.apache.ignite.internal.schema.BinaryRow;
- import org.apache.ignite.internal.schema.BinaryRowConverter;
import org.apache.ignite.internal.schema.BinaryTuple;
import org.apache.ignite.internal.schema.BinaryTupleSchema;
+import org.apache.ignite.internal.schema.Column;
import org.apache.ignite.internal.schema.SchemaDescriptor;
import org.apache.ignite.internal.schema.SchemaRegistry;
+ import org.apache.ignite.internal.schema.row.InternalTuple;
+ import org.apache.ignite.internal.sql.engine.schema.ColumnDescriptor;
+import org.apache.ignite.internal.schema.row.Row;
+import org.apache.ignite.internal.sql.engine.schema.ColumnDescriptor;
import org.apache.ignite.internal.sql.engine.schema.TableDescriptor;
+ import org.apache.ignite.internal.sql.engine.util.FieldDeserializingProjectedTuple;
+ import org.apache.ignite.internal.sql.engine.util.FormatAwareProjectedTuple;
import org.jetbrains.annotations.Nullable;
/**
diff --cc modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistrySelfTest.java
index 937cc182ff,6d99328dce..99b56ab6a8
--- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistrySelfTest.java
+++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistrySelfTest.java
@@@ -29,7 -29,9 +30,8 @@@ import org.apache.ignite.internal.schem
import org.apache.ignite.internal.schema.Column;
import org.apache.ignite.internal.schema.NativeTypes;
import org.apache.ignite.internal.schema.SchemaDescriptor;
-import org.apache.ignite.internal.schema.SchemaManager;
import org.apache.ignite.internal.schema.SchemaRegistry;
+ import org.apache.ignite.internal.sql.engine.schema.ColumnDescriptor;
import org.apache.ignite.internal.sql.engine.schema.TableDescriptor;
import org.apache.ignite.internal.table.InternalTable;
import org.apache.ignite.internal.table.TableImpl;
@@@ -151,8 -152,9 +153,9 @@@ public class ExecutableTableRegistrySel
when(tableManager.tableAsync(tableId)).thenReturn(CompletableFuture.completedFuture(table));
when(schemaManager.schemaRegistry(tableId)).thenReturn(schemaRegistry);
when(schemaRegistry.schema()).thenReturn(schemaDescriptor);
+ when(descriptor.iterator()).thenReturn(List.<ColumnDescriptor>of().iterator());
- return registry.getTable(tableId, descriptor);
+ return registry.getTable(tableId, tableVersion, descriptor);
}
}
}
diff --cc modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
index 66af7c86cf,1bbb0b1e45..c83f5d6666
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
@@@ -148,7 -167,7 +148,8 @@@ import org.apache.ignite.internal.table
import org.apache.ignite.internal.table.distributed.raft.snapshot.PartitionSnapshotStorageFactory;
import org.apache.ignite.internal.table.distributed.raft.snapshot.outgoing.OutgoingSnapshotsManager;
import org.apache.ignite.internal.table.distributed.raft.snapshot.outgoing.SnapshotAwarePartitionDataStorage;
+import org.apache.ignite.internal.table.distributed.replicator.DirectCatalogTables;
+ import org.apache.ignite.internal.table.distributed.replicator.CatalogTablesWithIdConversion;
import org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener;
import org.apache.ignite.internal.table.distributed.replicator.TransactionStateResolver;
import org.apache.ignite.internal.table.distributed.schema.NonHistoricSchemas;
@@@ -491,48 -538,80 +492,48 @@@ public class TableManager extends Produ
@Override
public void start() {
- mvGc.start();
+ inBusyLock(busyLock, () -> {
+ mvGc.start();
- lowWatermark.start();
+ lowWatermark.start();
- try {
- metaStorageMgr.recoveryFinishedFuture()
- .thenComposeAsync(this::performRebalanceOnRecovery, ioExecutor)
- .get();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
-
- throw new IgniteInternalException(INTERNAL_ERR, e);
- } catch (ExecutionException e) {
- throw new IgniteInternalException(INTERNAL_ERR, e);
- }
-
- metaStorageMgr.registerPrefixWatch(ByteArray.fromString(PENDING_ASSIGNMENTS_PREFIX), pendingAssignmentsRebalanceListener);
- metaStorageMgr.registerPrefixWatch(ByteArray.fromString(STABLE_ASSIGNMENTS_PREFIX), stableAssignmentsRebalanceListener);
- metaStorageMgr.registerPrefixWatch(ByteArray.fromString(ASSIGNMENTS_SWITCH_REDUCE_PREFIX), assignmentsSwitchRebalanceListener);
+ startTables();
- tablesCfg.tables().listenElements(new ConfigurationNamedListListener<>() {
- @Override
- public CompletableFuture<?> onCreate(ConfigurationNotificationEvent<TableView> ctx) {
- return onTableCreate(ctx);
- }
-
- @Override
- public CompletableFuture<?> onRename(ConfigurationNotificationEvent<TableView> ctx) {
- // TODO: IGNITE-15485 Support table rename operation.
-
- return completedFuture(null);
- }
-
- @Override
- public CompletableFuture<?> onDelete(ConfigurationNotificationEvent<TableView> ctx) {
- return onTableDelete(ctx);
- }
- });
-
- schemaManager.listen(SchemaEvent.CREATE, new EventListener<>() {
- @Override
- public CompletableFuture<Boolean> notify(SchemaEventParameters parameters, @Nullable Throwable exception) {
- var eventParameters = new TableEventParameters(parameters.causalityToken(), parameters.tableId());
-
- return fireEvent(TableEvent.ALTER, eventParameters).thenApply(v -> false);
+ try {
+ metaStorageMgr.recoveryFinishedFuture()
+ .thenComposeAsync(this::performRebalanceOnRecovery, ioExecutor)
+ .get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IgniteInternalException(INTERNAL_ERR, e);
+ } catch (ExecutionException e) {
+ throw new IgniteInternalException(INTERNAL_ERR, e);
}
- });
- addMessageHandler(clusterService.messagingService());
+ metaStorageMgr.registerPrefixWatch(ByteArray.fromString(PENDING_ASSIGNMENTS_PREFIX), pendingAssignmentsRebalanceListener);
+ metaStorageMgr.registerPrefixWatch(ByteArray.fromString(STABLE_ASSIGNMENTS_PREFIX), stableAssignmentsRebalanceListener);
+ metaStorageMgr.registerPrefixWatch(ByteArray.fromString(ASSIGNMENTS_SWITCH_REDUCE_PREFIX), assignmentsSwitchRebalanceListener);
- // TODO: IGNITE-19499 - remove when switched to the Catalog.
- recoverTableIdsMapping();
+ catalogService.listen(CatalogEvent.TABLE_CREATE, (parameters, exception) -> {
+ assert exception == null : parameters;
- // TODO: IGNITE-19499 - remove when switched to the Catalog.
- catalogService.listen(CatalogEvent.TABLE_CREATE, (parameters, exception) -> {
- CreateTableEventParameters event = (CreateTableEventParameters) parameters;
-
- TableView tableConfig = tablesCfg.tables().value().get(event.tableDescriptor().name());
-
- assert tableConfig != null : "No table config found by name " + event.tableDescriptor().name();
+ return onTableCreate((CreateTableEventParameters) parameters).thenApply(unused -> false);
+ });
- tableIdTranslator.registerMapping(tableConfig.id(), event.tableId());
+ catalogService.listen(CatalogEvent.TABLE_DROP, (parameters, exception) -> {
+ assert exception == null : parameters;
- return completedFuture(false);
- });
- }
+ return onTableDelete(((DropTableEventParameters) parameters)).thenApply(unused -> false);
+ });
- private void recoverTableIdsMapping() {
- tablesCfg.tables().value().forEach(tableView -> {
- CatalogTableDescriptor tableDescriptor = catalogService.table(tableView.name(), Long.MAX_VALUE);
+ schemaManager.listen(SchemaEvent.CREATE, (parameters, exception) -> inBusyLockAsync(busyLock, () -> {
+ var eventParameters = new TableEventParameters(parameters.causalityToken(), parameters.tableId());
- assert tableDescriptor != null : "No table in the Catalog with name " + tableView.name();
+ return fireEvent(TableEvent.ALTER, eventParameters).thenApply(v -> false);
+ }));
- tableIdTranslator.registerMapping(tableView.id(), tableDescriptor.id());
+ addMessageHandler(clusterService.messagingService());
});
}
diff --cc modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionReplicaListener.java
index fba6bec0e7,8b785381c4..a649d0ad6c
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionReplicaListener.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionReplicaListener.java
@@@ -279,8 -281,8 +279,9 @@@ public class PartitionReplicaListener i
* @param localNode Instance of the local node.
* @param mvTableStorage Table storage.
* @param indexBuilder Index builder.
- * @param tablesConfig Tables configuration.
+ * @param catalogService Catalog service.
+ * @param placementDriver Placement driver.
+ * @param placementDriver Placement driver.
*/
public PartitionReplicaListener(
MvPartitionStorage mvDataStorage,
diff --cc modules/table/src/testFixtures/java/org/apache/ignite/internal/table/impl/DummyInternalTableImpl.java
index bcc7c184b6,69476824e4..89ca256b7e
--- a/modules/table/src/testFixtures/java/org/apache/ignite/internal/table/impl/DummyInternalTableImpl.java
+++ b/modules/table/src/testFixtures/java/org/apache/ignite/internal/table/impl/DummyInternalTableImpl.java
@@@ -113,12 -113,12 +112,14 @@@ public class DummyInternalTableImpl ext
public static final ClusterNode LOCAL_NODE = new ClusterNodeImpl("id", "node", ADDR);
- public static final HybridClock CLOCK = new TestHybridClock(new LongSupplier() {
- @Override
- public long getAsLong() {
- return 0;
- }
- });
++ public static final ClusterNode LOCAL_NODE = new ClusterNodeImpl("id", "node", ADDR);
++
+ // 2000 was picked to avoid negative time that we get when building read timestamp
+ // in TxManagerImpl.currentReadTimestamp.
+ // We subtract (ReplicaManager.IDLE_SAFE_TIME_PROPAGATION_PERIOD_MILLISECONDS + HybridTimestamp.CLOCK_SKEW) = (1000 + 7) = 1007
+ // from the current time.
+ // Any value greater than that will work, hence 2000.
+ public static final HybridClock CLOCK = new TestHybridClock(() -> 2000);
private static final int PART_ID = 0;
[ignite-3] 02/02: catalog-feature after merge ai-main
Posted by tk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tkalkirill pushed a commit to branch catalog-feature
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit 96f3355cac1d54377b5d54559ff3e48e35a77033
Author: Kirill Tkalenko <tk...@yandex.ru>
AuthorDate: Thu Sep 14 10:12:43 2023 +0300
catalog-feature after merge ai-main
---
.../catalog/commands/AbstractCreateIndexCommand.java | 1 -
.../ignite/internal/catalog/commands/CatalogUtils.java | 2 ++
.../configuration/ConfigurationTreeGenerator.java | 1 -
.../ignite/internal/runner/app/ItTablesApiTest.java | 1 -
.../sql/engine/exec/ExecutableTableRegistryImpl.java | 16 ----------------
.../internal/sql/engine/exec/TableRowConverterImpl.java | 5 +----
.../ignite/internal/table/distributed/TableManager.java | 3 +--
.../distributed/replicator/PartitionReplicaListener.java | 1 -
.../internal/table/impl/DummyInternalTableImpl.java | 2 --
9 files changed, 4 insertions(+), 28 deletions(-)
diff --git a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/AbstractCreateIndexCommand.java b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/AbstractCreateIndexCommand.java
index bee55a4c18..471c3332e1 100644
--- a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/AbstractCreateIndexCommand.java
+++ b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/AbstractCreateIndexCommand.java
@@ -17,7 +17,6 @@
package org.apache.ignite.internal.catalog.commands;
-import static org.apache.ignite.internal.catalog.CatalogParamsValidationUtils.ensureNoTableOrIndexExistsWithGivenName;
import static org.apache.ignite.internal.catalog.CatalogParamsValidationUtils.ensureNoTableIndexOrSysViewExistsWithGivenName;
import static org.apache.ignite.internal.catalog.CatalogParamsValidationUtils.validateIdentifier;
import static org.apache.ignite.internal.catalog.commands.CatalogUtils.schemaOrThrow;
diff --git a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/CatalogUtils.java b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/CatalogUtils.java
index d34b705ed9..eba42a5cc2 100644
--- a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/CatalogUtils.java
+++ b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/CatalogUtils.java
@@ -17,11 +17,13 @@
package org.apache.ignite.internal.catalog.commands;
+import static java.util.stream.Collectors.toList;
import static org.apache.ignite.lang.IgniteStringFormatter.format;
import java.util.Collection;
import java.util.EnumMap;
import java.util.EnumSet;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
diff --git a/modules/configuration/src/main/java/org/apache/ignite/internal/configuration/ConfigurationTreeGenerator.java b/modules/configuration/src/main/java/org/apache/ignite/internal/configuration/ConfigurationTreeGenerator.java
index 0984d66bb0..13d7761d65 100644
--- a/modules/configuration/src/main/java/org/apache/ignite/internal/configuration/ConfigurationTreeGenerator.java
+++ b/modules/configuration/src/main/java/org/apache/ignite/internal/configuration/ConfigurationTreeGenerator.java
@@ -31,7 +31,6 @@ import static org.apache.ignite.internal.configuration.util.ConfigurationUtil.sc
import static org.apache.ignite.internal.configuration.util.ConfigurationUtil.schemaFields;
import static org.apache.ignite.internal.util.CollectionUtils.difference;
import static org.apache.ignite.internal.util.CollectionUtils.first;
-import static org.apache.ignite.internal.util.CollectionUtils.viewReadOnly;
import java.lang.reflect.Field;
import java.util.Collection;
diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItTablesApiTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItTablesApiTest.java
index 9947f25bd8..ac1c41e638 100644
--- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItTablesApiTest.java
+++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItTablesApiTest.java
@@ -39,7 +39,6 @@ import java.util.stream.IntStream;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgnitionManager;
import org.apache.ignite.InitParameters;
-import org.apache.ignite.internal.catalog.IndexExistsValidationException;
import org.apache.ignite.internal.catalog.CatalogValidationException;
import org.apache.ignite.internal.catalog.IndexExistsValidationException;
import org.apache.ignite.internal.catalog.TableExistsValidationException;
diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistryImpl.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistryImpl.java
index ecd7d29ab8..a807a38bbd 100644
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistryImpl.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistryImpl.java
@@ -78,22 +78,6 @@ public class ExecutableTableRegistryImpl implements ExecutableTableRegistry, Sch
tableCache.clear();
}
- private CompletableFuture<ExecutableTable> loadTable(String tableName, TableDescriptor tableDescriptor) {
- return tableManager.tableAsyncInternal(tableName.toUpperCase())
- .thenApply(table -> {
- InternalTable internalTable = table.internalTable();
- SchemaRegistry schemaRegistry = schemaManager.schemaRegistry(table.tableId());
- SchemaDescriptor schemaDescriptor = schemaRegistry.schema();
- TableRowConverter rowConverter = new TableRowConverterImpl(schemaRegistry, schemaDescriptor, tableDescriptor);
- ScannableTable scannableTable = new ScannableTableImpl(internalTable, rowConverter, tableDescriptor);
-
- UpdatableTableImpl updatableTable = new UpdatableTableImpl(table.tableId(), tableDescriptor, internalTable.partitions(),
- replicaService, clock, rowConverter, schemaDescriptor);
-
- return new ExecutableTableImpl(internalTable, scannableTable, updatableTable);
- });
- }
-
private CompletableFuture<ExecutableTable> loadTable(int tableId, TableDescriptor tableDescriptor) {
CompletableFuture<Map.Entry<InternalTable, SchemaRegistry>> f = tableManager.tableAsync(tableId)
.thenApply(table -> {
diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterImpl.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterImpl.java
index 8801dc69a0..9af3e25aa9 100644
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterImpl.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterImpl.java
@@ -21,13 +21,10 @@ import java.util.BitSet;
import org.apache.ignite.internal.schema.BinaryRow;
import org.apache.ignite.internal.schema.BinaryTuple;
import org.apache.ignite.internal.schema.BinaryTupleSchema;
-import org.apache.ignite.internal.schema.Column;
import org.apache.ignite.internal.schema.SchemaDescriptor;
import org.apache.ignite.internal.schema.SchemaRegistry;
import org.apache.ignite.internal.schema.row.InternalTuple;
import org.apache.ignite.internal.sql.engine.schema.ColumnDescriptor;
-import org.apache.ignite.internal.schema.row.Row;
-import org.apache.ignite.internal.sql.engine.schema.ColumnDescriptor;
import org.apache.ignite.internal.sql.engine.schema.TableDescriptor;
import org.apache.ignite.internal.sql.engine.util.FieldDeserializingProjectedTuple;
import org.apache.ignite.internal.sql.engine.util.FormatAwareProjectedTuple;
@@ -67,7 +64,7 @@ public class TableRowConverterImpl implements TableRowConverter {
continue;
}
- mapping[currentIdx++] = column.physicalIndex();
+ mapping[currentIdx++] = schemaDescriptor.column(column.name()).schemaIndex();
}
}
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
index c83f5d6666..66af7c86cf 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
@@ -149,7 +149,6 @@ import org.apache.ignite.internal.table.distributed.raft.snapshot.PartitionSnaps
import org.apache.ignite.internal.table.distributed.raft.snapshot.outgoing.OutgoingSnapshotsManager;
import org.apache.ignite.internal.table.distributed.raft.snapshot.outgoing.SnapshotAwarePartitionDataStorage;
import org.apache.ignite.internal.table.distributed.replicator.DirectCatalogTables;
-import org.apache.ignite.internal.table.distributed.replicator.CatalogTablesWithIdConversion;
import org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener;
import org.apache.ignite.internal.table.distributed.replicator.TransactionStateResolver;
import org.apache.ignite.internal.table.distributed.schema.NonHistoricSchemas;
@@ -495,7 +494,7 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp
inBusyLock(busyLock, () -> {
mvGc.start();
- lowWatermark.start();
+ lowWatermark.start();
startTables();
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionReplicaListener.java b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionReplicaListener.java
index a649d0ad6c..fba6bec0e7 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionReplicaListener.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionReplicaListener.java
@@ -281,7 +281,6 @@ public class PartitionReplicaListener implements ReplicaListener {
* @param indexBuilder Index builder.
* @param catalogService Catalog service.
* @param placementDriver Placement driver.
- * @param placementDriver Placement driver.
*/
public PartitionReplicaListener(
MvPartitionStorage mvDataStorage,
diff --git a/modules/table/src/testFixtures/java/org/apache/ignite/internal/table/impl/DummyInternalTableImpl.java b/modules/table/src/testFixtures/java/org/apache/ignite/internal/table/impl/DummyInternalTableImpl.java
index 89ca256b7e..89f2812e0b 100644
--- a/modules/table/src/testFixtures/java/org/apache/ignite/internal/table/impl/DummyInternalTableImpl.java
+++ b/modules/table/src/testFixtures/java/org/apache/ignite/internal/table/impl/DummyInternalTableImpl.java
@@ -112,8 +112,6 @@ public class DummyInternalTableImpl extends InternalTableImpl {
public static final ClusterNode LOCAL_NODE = new ClusterNodeImpl("id", "node", ADDR);
- public static final ClusterNode LOCAL_NODE = new ClusterNodeImpl("id", "node", ADDR);
-
// 2000 was picked to avoid negative time that we get when building read timestamp
// in TxManagerImpl.currentReadTimestamp.
// We subtract (ReplicaManager.IDLE_SAFE_TIME_PROPAGATION_PERIOD_MILLISECONDS + HybridTimestamp.CLOCK_SKEW) = (1000 + 7) = 1007