You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@ignite.apache.org by GitBox <gi...@apache.org> on 2022/09/06 14:29:23 UTC
[GitHub] [ignite-3] SammyVimes commented on a diff in pull request #1055: IGNITE-17339 Hash index storage for page-memory based engines.
SammyVimes commented on code in PR #1055:
URL: https://github.com/apache/ignite-3/pull/1055#discussion_r963695402
##########
modules/storage-api/src/test/java/org/apache/ignite/internal/storage/index/AbstractHashIndexStorageTest.java:
##########
@@ -210,13 +214,32 @@ void testDestroy() {
put(row2);
put(row3);
- indexStorage.destroy();
+ CompletableFuture<Void> destroyFuture = tableStorage.destroyIndex(indexStorage.indexDescriptor().id());
+
+ waitForDurableCompletion(destroyFuture, 1L);
+ //TODO IGNITE-17626 Index must be invalid, we should assert that getIndex returns null and that in won't surface upon restart.
+ // "destroy" is not "clear", you know. Maybe "getAndCreateIndex" will do it for the test, idk
assertThat(getAll(row1), is(empty()));
assertThat(getAll(row2), is(empty()));
assertThat(getAll(row3), is(empty()));
}
+ @SuppressWarnings("BusyWait")
+ private void waitForDurableCompletion(CompletableFuture<?> future, long timeout) throws Exception {
+ while (true) {
+ if (future.isDone()) {
+ return;
+ }
+
+ CompletableFuture<Void> flushFuture = partitionStorage.flush();
+
+ Thread.sleep(timeout);
+
+ flushFuture.join();
+ }
Review Comment:
Do we need to sleep if we do a join on a future?
##########
modules/storage-page-memory/src/main/java/org/apache/ignite/internal/storage/pagememory/mv/AbstractPageMemoryMvPartitionStorage.java:
##########
@@ -52,39 +71,134 @@ public abstract class AbstractPageMemoryMvPartitionStorage implements MvPartitio
private static final Predicate<HybridTimestamp> ALWAYS_LOAD_VALUE = timestamp -> true;
- private final int partitionId;
- private final int groupId;
+ protected final int partitionId;
+
+ protected final AbstractPageMemoryTableStorage tableStorage;
+
+ protected final VersionChainTree versionChainTree;
- private final VersionChainTree versionChainTree;
protected final RowVersionFreeList rowVersionFreeList;
- private final DataPageReader rowVersionDataPageReader;
+
+ protected final IndexColumnsFreeList indexFreeList;
+
+ protected final IndexMetaTree indexMetaTree;
+
+ protected final DataPageReader rowVersionDataPageReader;
+
+ protected final ConcurrentMap<UUID, HashIndexStorage> indexes = new ConcurrentHashMap<>();
/**
* Constructor.
*
* @param partitionId Partition id.
- * @param tableView Table configuration.
- * @param pageMemory Page memory.
+ * @param tableStorage Table storage instance.
* @param rowVersionFreeList Free list for {@link RowVersion}.
+ * @param indexFreeList Free list fot {@link IndexColumns}.
* @param versionChainTree Table tree for {@link VersionChain}.
+ * @param indexMetaTree Tree that contains SQL indexes.
*/
protected AbstractPageMemoryMvPartitionStorage(
int partitionId,
- TableView tableView,
- PageMemory pageMemory,
+ AbstractPageMemoryTableStorage tableStorage,
RowVersionFreeList rowVersionFreeList,
- VersionChainTree versionChainTree
+ IndexColumnsFreeList indexFreeList,
+ VersionChainTree versionChainTree,
+ IndexMetaTree indexMetaTree
) {
this.partitionId = partitionId;
+ this.tableStorage = tableStorage;
this.rowVersionFreeList = rowVersionFreeList;
+ this.indexFreeList = indexFreeList;
+
this.versionChainTree = versionChainTree;
+ this.indexMetaTree = indexMetaTree;
+
+ PageMemory pageMemory = tableStorage.dataRegion().pageMemory();
- groupId = tableView.tableId();
+ int groupId = tableStorage.configuration().value().tableId();
rowVersionDataPageReader = new DataPageReader(pageMemory, groupId, IoStatisticsHolderNoOp.INSTANCE);
}
+ /**
+ * Starts a partition by initializing its internal structures.
+ */
+ public void start() {
+ try {
+ IgniteCursor<IndexMeta> cursor = indexMetaTree.find(null, null);
+
+ NamedListView<? extends TableIndexView> indicesCfgView = tableStorage.configuration().value().indices();
+
+ while (cursor.next()) {
+ IndexMeta indexMeta = cursor.get();
+
+ TableIndexView indexCfgView = getByInternalId(indicesCfgView, indexMeta.id());
+
+ if (indexCfgView instanceof HashIndexView) {
+ createHashIndex(indexMeta);
+ } else if (indexCfgView instanceof SortedIndexView) {
+ throw new UnsupportedOperationException("Not implemented yet");
+ } else {
+ assert indexCfgView == null;
+
+ //TODO IGNITE-17626 Drop the index synchronously.
+ }
+ }
+ } catch (IgniteInternalCheckedException e) {
+ throw new StorageException("", e);
+ }
+ }
+
+ /**
+ * Returns a hash index instance, creating index it if necessary.
+ *
+ * @param indexId Index UUID.
+ */
+ public HashIndexStorage getOrCreateHashIndex(UUID indexId) {
+ return indexes.computeIfAbsent(indexId, uuid -> createHashIndex(new IndexMeta(indexId, 0L)));
+ }
+
+ private PageMemoryHashIndexStorage createHashIndex(IndexMeta indexMeta) {
Review Comment:
More like createOrRestore, right?
##########
modules/storage-page-memory/src/test/java/org/apache/ignite/internal/storage/pagememory/index/PersistentPageMemoryHashIndexStorageTest.java:
##########
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.storage.pagememory.index;
+
+import java.nio.file.Path;
+import org.apache.ignite.configuration.schemas.table.ConstantValueDefaultConfigurationSchema;
+import org.apache.ignite.configuration.schemas.table.EntryCountBudgetConfigurationSchema;
+import org.apache.ignite.configuration.schemas.table.FunctionCallDefaultConfigurationSchema;
+import org.apache.ignite.configuration.schemas.table.HashIndexConfigurationSchema;
+import org.apache.ignite.configuration.schemas.table.NullValueDefaultConfigurationSchema;
+import org.apache.ignite.configuration.schemas.table.TableConfiguration;
+import org.apache.ignite.configuration.schemas.table.UnlimitedBudgetConfigurationSchema;
+import org.apache.ignite.internal.configuration.testframework.ConfigurationExtension;
+import org.apache.ignite.internal.configuration.testframework.InjectConfiguration;
+import org.apache.ignite.internal.pagememory.configuration.schema.UnsafeMemoryAllocatorConfigurationSchema;
+import org.apache.ignite.internal.pagememory.io.PageIoRegistry;
+import org.apache.ignite.internal.storage.index.AbstractHashIndexStorageTest;
+import org.apache.ignite.internal.storage.pagememory.PersistentPageMemoryStorageEngine;
+import org.apache.ignite.internal.storage.pagememory.PersistentPageMemoryTableStorage;
+import org.apache.ignite.internal.storage.pagememory.configuration.schema.PersistentPageMemoryDataStorageConfigurationSchema;
+import org.apache.ignite.internal.storage.pagememory.configuration.schema.PersistentPageMemoryStorageEngineConfiguration;
+import org.apache.ignite.internal.testframework.WorkDirectory;
+import org.apache.ignite.internal.testframework.WorkDirectoryExtension;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/**
+ * Hash index test implementation for persistent page memory storage.
+ */
+@ExtendWith({ConfigurationExtension.class, WorkDirectoryExtension.class})
+class PersistentPageMemoryHashIndexStorageTest extends AbstractHashIndexStorageTest {
+ @WorkDirectory
+ private Path workDir;
+
+ @InjectConfiguration(polymorphicExtensions = UnsafeMemoryAllocatorConfigurationSchema.class)
+ private PersistentPageMemoryStorageEngineConfiguration engineConfig;
+
+ @InjectConfiguration(
+ name = "table",
+ value = "mock.dataStorage.name = " + PersistentPageMemoryStorageEngine.ENGINE_NAME,
+ polymorphicExtensions = {
Review Comment:
Sidenote: we wanted to make testing easier and yet we have to provide a giant list of classes smh
##########
modules/storage-page-memory/src/main/java/org/apache/ignite/internal/storage/pagememory/mv/AbstractPageMemoryMvPartitionStorage.java:
##########
@@ -52,39 +71,134 @@ public abstract class AbstractPageMemoryMvPartitionStorage implements MvPartitio
private static final Predicate<HybridTimestamp> ALWAYS_LOAD_VALUE = timestamp -> true;
- private final int partitionId;
- private final int groupId;
+ protected final int partitionId;
+
+ protected final AbstractPageMemoryTableStorage tableStorage;
+
+ protected final VersionChainTree versionChainTree;
- private final VersionChainTree versionChainTree;
protected final RowVersionFreeList rowVersionFreeList;
- private final DataPageReader rowVersionDataPageReader;
+
+ protected final IndexColumnsFreeList indexFreeList;
+
+ protected final IndexMetaTree indexMetaTree;
+
+ protected final DataPageReader rowVersionDataPageReader;
+
+ protected final ConcurrentMap<UUID, HashIndexStorage> indexes = new ConcurrentHashMap<>();
/**
* Constructor.
*
* @param partitionId Partition id.
- * @param tableView Table configuration.
- * @param pageMemory Page memory.
+ * @param tableStorage Table storage instance.
* @param rowVersionFreeList Free list for {@link RowVersion}.
+ * @param indexFreeList Free list fot {@link IndexColumns}.
* @param versionChainTree Table tree for {@link VersionChain}.
+ * @param indexMetaTree Tree that contains SQL indexes.
*/
protected AbstractPageMemoryMvPartitionStorage(
int partitionId,
- TableView tableView,
- PageMemory pageMemory,
+ AbstractPageMemoryTableStorage tableStorage,
RowVersionFreeList rowVersionFreeList,
- VersionChainTree versionChainTree
+ IndexColumnsFreeList indexFreeList,
+ VersionChainTree versionChainTree,
+ IndexMetaTree indexMetaTree
) {
this.partitionId = partitionId;
+ this.tableStorage = tableStorage;
this.rowVersionFreeList = rowVersionFreeList;
+ this.indexFreeList = indexFreeList;
+
this.versionChainTree = versionChainTree;
+ this.indexMetaTree = indexMetaTree;
+
+ PageMemory pageMemory = tableStorage.dataRegion().pageMemory();
- groupId = tableView.tableId();
+ int groupId = tableStorage.configuration().value().tableId();
rowVersionDataPageReader = new DataPageReader(pageMemory, groupId, IoStatisticsHolderNoOp.INSTANCE);
}
+ /**
+ * Starts a partition by initializing its internal structures.
+ */
+ public void start() {
Review Comment:
I feel like there are too much index specifics now (not in method start but in storage class itself).
Probably it will be bloated in a year. Do you think index stuff can be extracted somehow? I don't have any suggestions though
##########
modules/storage-page-memory/src/main/java/org/apache/ignite/internal/storage/pagememory/mv/AbstractPageMemoryMvPartitionStorage.java:
##########
@@ -52,39 +71,134 @@ public abstract class AbstractPageMemoryMvPartitionStorage implements MvPartitio
private static final Predicate<HybridTimestamp> ALWAYS_LOAD_VALUE = timestamp -> true;
- private final int partitionId;
- private final int groupId;
+ protected final int partitionId;
+
+ protected final AbstractPageMemoryTableStorage tableStorage;
+
+ protected final VersionChainTree versionChainTree;
- private final VersionChainTree versionChainTree;
protected final RowVersionFreeList rowVersionFreeList;
- private final DataPageReader rowVersionDataPageReader;
+
+ protected final IndexColumnsFreeList indexFreeList;
+
+ protected final IndexMetaTree indexMetaTree;
+
+ protected final DataPageReader rowVersionDataPageReader;
+
+ protected final ConcurrentMap<UUID, HashIndexStorage> indexes = new ConcurrentHashMap<>();
/**
* Constructor.
*
* @param partitionId Partition id.
- * @param tableView Table configuration.
- * @param pageMemory Page memory.
+ * @param tableStorage Table storage instance.
* @param rowVersionFreeList Free list for {@link RowVersion}.
+ * @param indexFreeList Free list fot {@link IndexColumns}.
* @param versionChainTree Table tree for {@link VersionChain}.
+ * @param indexMetaTree Tree that contains SQL indexes.
*/
protected AbstractPageMemoryMvPartitionStorage(
int partitionId,
- TableView tableView,
- PageMemory pageMemory,
+ AbstractPageMemoryTableStorage tableStorage,
RowVersionFreeList rowVersionFreeList,
- VersionChainTree versionChainTree
+ IndexColumnsFreeList indexFreeList,
+ VersionChainTree versionChainTree,
+ IndexMetaTree indexMetaTree
) {
this.partitionId = partitionId;
+ this.tableStorage = tableStorage;
this.rowVersionFreeList = rowVersionFreeList;
+ this.indexFreeList = indexFreeList;
+
this.versionChainTree = versionChainTree;
+ this.indexMetaTree = indexMetaTree;
+
+ PageMemory pageMemory = tableStorage.dataRegion().pageMemory();
- groupId = tableView.tableId();
+ int groupId = tableStorage.configuration().value().tableId();
rowVersionDataPageReader = new DataPageReader(pageMemory, groupId, IoStatisticsHolderNoOp.INSTANCE);
}
+ /**
+ * Starts a partition by initializing its internal structures.
+ */
+ public void start() {
+ try {
+ IgniteCursor<IndexMeta> cursor = indexMetaTree.find(null, null);
+
+ NamedListView<? extends TableIndexView> indicesCfgView = tableStorage.configuration().value().indices();
+
+ while (cursor.next()) {
+ IndexMeta indexMeta = cursor.get();
+
+ TableIndexView indexCfgView = getByInternalId(indicesCfgView, indexMeta.id());
+
+ if (indexCfgView instanceof HashIndexView) {
+ createHashIndex(indexMeta);
+ } else if (indexCfgView instanceof SortedIndexView) {
+ throw new UnsupportedOperationException("Not implemented yet");
+ } else {
+ assert indexCfgView == null;
+
+ //TODO IGNITE-17626 Drop the index synchronously.
+ }
+ }
+ } catch (IgniteInternalCheckedException e) {
+ throw new StorageException("", e);
Review Comment:
Stub exception message
##########
modules/storage-page-memory/src/main/java/org/apache/ignite/internal/storage/pagememory/mv/PersistentPageMemoryMvPartitionStorage.java:
##########
@@ -192,6 +198,14 @@ private void syncMetadataOnCheckpoint(@Nullable Executor executor) throws Ignite
throw new IgniteInternalException("Failed to save RowVersionFreeList metadata", e);
}
});
+
+ executor.execute(() -> {
+ try {
+ indexFreeList.saveMetadata();
+ } catch (IgniteInternalCheckedException e) {
+ throw new IgniteInternalException("Failed to save RowVersionFreeList metadata", e);
Review Comment:
```suggestion
throw new IgniteInternalException("Failed to save IndexColumnsFreeList metadata", e);
```
##########
modules/storage-page-memory/src/main/java/org/apache/ignite/internal/storage/pagememory/mv/PersistentPageMemoryMvPartitionStorage.java:
##########
@@ -136,7 +130,9 @@ public CompletableFuture<Void> flush() {
if (lastCheckpoint != null && meta.metaSnapshot(lastCheckpoint.id()).lastAppliedIndex() == meta.lastAppliedIndex()) {
scheduledCheckpoint = lastCheckpoint;
} else {
- PersistentPageMemoryStorageEngineView engineCfg = tableStorage.engine().configuration().value();
+ var persistentTableStorage = (PersistentPageMemoryTableStorage) tableStorage;
Review Comment:
is `var` ok here because of the cast?
##########
modules/storage-page-memory/src/test/java/org/apache/ignite/internal/storage/pagememory/mv/AbstractPageMemoryMvPartitionStorageTest.java:
##########
@@ -35,8 +35,7 @@
/**
* Base test for MV partition storages based on PageMemory.
*/
-@ExtendWith(ConfigurationExtension.class)
-@ExtendWith(WorkDirectoryExtension.class)
+@ExtendWith({ConfigurationExtension.class, WorkDirectoryExtension.class})
Review Comment:
So they made it both `@Repeatable` and array-based. I blame junit!
##########
modules/storage-page-memory/src/main/java/org/apache/ignite/internal/storage/pagememory/index/meta/io/IndexMetaIo.java:
##########
@@ -112,6 +112,6 @@ default void storeByOffset(long pageAddr, int off, IndexMeta row) {
putLong(pageAddr, off + INDEX_ID_MSB_OFFSET, row.id().getMostSignificantBits());
putLong(pageAddr, off + INDEX_ID_LSB_OFFSET, row.id().getLeastSignificantBits());
- putLong(pageAddr, off + INDEX_ROOT_PAGE_ID_OFFSET, row.rootPageId());
+ putLong(pageAddr, off + INDEX_ROOT_PAGE_ID_OFFSET, row.metaPageId());
Review Comment:
Let's rename the offset (and change the comment)
##########
modules/storage-page-memory/src/main/java/org/apache/ignite/internal/storage/pagememory/index/freelist/io/IndexColumnsDataIo.java:
##########
@@ -48,6 +49,10 @@ protected IndexColumnsDataIo(int ver) {
protected void writeRowData(long pageAddr, int dataOff, int payloadSize, IndexColumns row, boolean newRow) {
assertPageType(pageAddr);
+ putShort(pageAddr, dataOff, (short) payloadSize);
Review Comment:
I wonder if we check payloadSize to be in range of the unsigned short somewhere
##########
modules/storage-page-memory/src/main/java/org/apache/ignite/internal/storage/pagememory/mv/VolatilePageMemoryMvPartitionStorage.java:
##########
@@ -34,20 +34,25 @@ public class VolatilePageMemoryMvPartitionStorage extends AbstractPageMemoryMvPa
/**
* Constructor.
*
- * @param partId Partition id.
- * @param tableView Table configuration.
- * @param pageMemory Page memory.
- * @param rowVersionFreeList Free list for {@link RowVersion}.
+ * @param tableStorage Table storage instance.
+ * @param partitionId Partition id.
* @param versionChainTree Table tree for {@link VersionChain}.
+ * @param indexMetaTree Tree that contains SQL indexes.
*/
Review Comment:
```suggestion
* @param indexMetaTree Tree that contains SQL indexes' metadata.
```
Or something like that?
##########
modules/storage-page-memory/src/main/java/org/apache/ignite/internal/storage/pagememory/mv/AbstractPageMemoryMvPartitionStorage.java:
##########
@@ -52,39 +71,134 @@ public abstract class AbstractPageMemoryMvPartitionStorage implements MvPartitio
private static final Predicate<HybridTimestamp> ALWAYS_LOAD_VALUE = timestamp -> true;
- private final int partitionId;
- private final int groupId;
+ protected final int partitionId;
+
+ protected final AbstractPageMemoryTableStorage tableStorage;
+
+ protected final VersionChainTree versionChainTree;
- private final VersionChainTree versionChainTree;
protected final RowVersionFreeList rowVersionFreeList;
- private final DataPageReader rowVersionDataPageReader;
+
+ protected final IndexColumnsFreeList indexFreeList;
+
+ protected final IndexMetaTree indexMetaTree;
+
+ protected final DataPageReader rowVersionDataPageReader;
+
+ protected final ConcurrentMap<UUID, HashIndexStorage> indexes = new ConcurrentHashMap<>();
/**
* Constructor.
*
* @param partitionId Partition id.
- * @param tableView Table configuration.
- * @param pageMemory Page memory.
+ * @param tableStorage Table storage instance.
* @param rowVersionFreeList Free list for {@link RowVersion}.
+ * @param indexFreeList Free list fot {@link IndexColumns}.
* @param versionChainTree Table tree for {@link VersionChain}.
+ * @param indexMetaTree Tree that contains SQL indexes.
*/
protected AbstractPageMemoryMvPartitionStorage(
int partitionId,
- TableView tableView,
- PageMemory pageMemory,
+ AbstractPageMemoryTableStorage tableStorage,
RowVersionFreeList rowVersionFreeList,
- VersionChainTree versionChainTree
+ IndexColumnsFreeList indexFreeList,
+ VersionChainTree versionChainTree,
+ IndexMetaTree indexMetaTree
) {
this.partitionId = partitionId;
+ this.tableStorage = tableStorage;
this.rowVersionFreeList = rowVersionFreeList;
+ this.indexFreeList = indexFreeList;
+
this.versionChainTree = versionChainTree;
+ this.indexMetaTree = indexMetaTree;
+
+ PageMemory pageMemory = tableStorage.dataRegion().pageMemory();
- groupId = tableView.tableId();
+ int groupId = tableStorage.configuration().value().tableId();
rowVersionDataPageReader = new DataPageReader(pageMemory, groupId, IoStatisticsHolderNoOp.INSTANCE);
}
+ /**
+ * Starts a partition by initializing its internal structures.
+ */
+ public void start() {
+ try {
+ IgniteCursor<IndexMeta> cursor = indexMetaTree.find(null, null);
+
+ NamedListView<? extends TableIndexView> indicesCfgView = tableStorage.configuration().value().indices();
+
+ while (cursor.next()) {
+ IndexMeta indexMeta = cursor.get();
+
+ TableIndexView indexCfgView = getByInternalId(indicesCfgView, indexMeta.id());
+
+ if (indexCfgView instanceof HashIndexView) {
+ createHashIndex(indexMeta);
+ } else if (indexCfgView instanceof SortedIndexView) {
+ throw new UnsupportedOperationException("Not implemented yet");
+ } else {
+ assert indexCfgView == null;
+
+ //TODO IGNITE-17626 Drop the index synchronously.
+ }
+ }
+ } catch (IgniteInternalCheckedException e) {
+ throw new StorageException("", e);
+ }
+ }
+
+ /**
+ * Returns a hash index instance, creating index it if necessary.
+ *
+ * @param indexId Index UUID.
+ */
+ public HashIndexStorage getOrCreateHashIndex(UUID indexId) {
+ return indexes.computeIfAbsent(indexId, uuid -> createHashIndex(new IndexMeta(indexId, 0L)));
+ }
+
+ private PageMemoryHashIndexStorage createHashIndex(IndexMeta indexMeta) {
+ TableView tableView = tableStorage.configuration().value();
+
+ var indexDescriptor = new HashIndexDescriptor(indexMeta.id(), tableView);
+
+ try {
+ PageMemory pageMemory = tableStorage.dataRegion().pageMemory();
+
+ int groupId = tableView.tableId();
Review Comment:
Why don't we store groupId in a field anymore?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org