You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by am...@apache.org on 2023/05/11 12:49:53 UTC

[ignite-3] 01/02: Wip. Add catalog events.

This is an automated email from the ASF dual-hosted git repository.

amashenkov pushed a commit to branch ignite-19081
in repository https://gitbox.apache.org/repos/asf/ignite-3.git

commit 56c5bc76bac6ff921d1326e97380b3b9ecd3c34b
Author: amashenkov <an...@gmail.com>
AuthorDate: Wed May 10 19:19:54 2023 +0300

    Wip. Add catalog events.
---
 .../ignite/internal/catalog/CatalogManager.java    |  2 +-
 .../ignite/internal/catalog/CatalogService.java    |  5 +++
 .../internal/catalog/CatalogServiceImpl.java       | 28 +++++++++----
 .../internal/catalog/events/CatalogEvent.java      | 31 ++++++++++++++
 .../catalog/events/CatalogEventParameters.java     | 34 ++++++++++++++++
 .../catalog/events/CreateTableEventParameters.java | 47 ++++++++++++++++++++++
 .../internal/catalog/CatalogServiceSelfTest.java   | 35 ++++++++++++++++
 modules/index/build.gradle                         |  1 +
 .../org/apache/ignite/internal/app/IgniteImpl.java |  8 ++--
 .../engine/exec/ddl/DdlCommandHandlerWrapper.java  |  1 +
 modules/table/build.gradle                         |  1 +
 11 files changed, 181 insertions(+), 12 deletions(-)

diff --git a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/CatalogManager.java b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/CatalogManager.java
index 47f883180f..2724d5d965 100644
--- a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/CatalogManager.java
+++ b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/CatalogManager.java
@@ -27,7 +27,7 @@ import org.apache.ignite.internal.manager.IgniteComponent;
 /**
  * The catalog manager provides schema manipulation methods and is responsible for managing distributed operations.
  */
-public interface CatalogManager extends IgniteComponent {
+public interface CatalogManager extends IgniteComponent, CatalogService {
     /**
      * Creates new table.
      *
diff --git a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/CatalogService.java b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/CatalogService.java
index 507eefde03..a23b23da38 100644
--- a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/CatalogService.java
+++ b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/CatalogService.java
@@ -21,6 +21,9 @@ import java.util.Collection;
 import org.apache.ignite.internal.catalog.descriptors.IndexDescriptor;
 import org.apache.ignite.internal.catalog.descriptors.SchemaDescriptor;
 import org.apache.ignite.internal.catalog.descriptors.TableDescriptor;
+import org.apache.ignite.internal.catalog.events.CatalogEvent;
+import org.apache.ignite.internal.catalog.events.CatalogEventParameters;
+import org.apache.ignite.internal.manager.EventListener;
 
 /**
  * Catalog service provides methods to access schema object's descriptors of exact version and/or last actual version at given timestamp,
@@ -54,4 +57,6 @@ public interface CatalogService {
     SchemaDescriptor schema(int version);
 
     SchemaDescriptor activeSchema(long timestamp);
+
+    void listen(CatalogEvent evt, EventListener<CatalogEventParameters> closure);
 }
diff --git a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/CatalogServiceImpl.java b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/CatalogServiceImpl.java
index 00097301b9..dadcee6fef 100644
--- a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/CatalogServiceImpl.java
+++ b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/CatalogServiceImpl.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.catalog;
 import static java.util.concurrent.CompletableFuture.completedFuture;
 import static java.util.concurrent.CompletableFuture.failedFuture;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map.Entry;
@@ -35,12 +36,16 @@ import org.apache.ignite.internal.catalog.commands.DropTableParams;
 import org.apache.ignite.internal.catalog.descriptors.IndexDescriptor;
 import org.apache.ignite.internal.catalog.descriptors.SchemaDescriptor;
 import org.apache.ignite.internal.catalog.descriptors.TableDescriptor;
+import org.apache.ignite.internal.catalog.events.CatalogEvent;
+import org.apache.ignite.internal.catalog.events.CatalogEventParameters;
+import org.apache.ignite.internal.catalog.events.CreateTableEventParameters;
 import org.apache.ignite.internal.catalog.storage.NewTableEntry;
 import org.apache.ignite.internal.catalog.storage.ObjectIdGenUpdateEntry;
 import org.apache.ignite.internal.catalog.storage.UpdateEntry;
 import org.apache.ignite.internal.catalog.storage.UpdateLog;
 import org.apache.ignite.internal.catalog.storage.UpdateLog.OnUpdateHandler;
 import org.apache.ignite.internal.catalog.storage.VersionedUpdate;
+import org.apache.ignite.internal.manager.Producer;
 import org.apache.ignite.internal.util.ArrayUtils;
 import org.apache.ignite.internal.util.PendingComparableValuesTracker;
 import org.apache.ignite.lang.ErrorGroups.Common;
@@ -52,7 +57,7 @@ import org.jetbrains.annotations.Nullable;
  * Catalog service implementation.
  * TODO: IGNITE-19081 Introduce catalog events and make CatalogServiceImpl extends Producer.
  */
-public class CatalogServiceImpl implements CatalogService, CatalogManager {
+public class CatalogServiceImpl extends Producer<CatalogEvent, CatalogEventParameters> implements CatalogManager {
     private static final int MAX_RETRY_COUNT = 10;
 
     /** Versioned catalog descriptors. */
@@ -229,36 +234,45 @@ public class CatalogServiceImpl implements CatalogService, CatalogManager {
     class OnUpdateHandlerImpl implements OnUpdateHandler {
         @Override
         public void handle(VersionedUpdate update) {
-            Catalog catalog = catalogByVer.get(update.version() - 1);
+            int version = update.version();
+            Catalog catalog = catalogByVer.get(version - 1);
 
             assert catalog != null;
 
+            List<CompletableFuture<?>> eventFutures = new ArrayList<>(update.entries().size());
+
             for (UpdateEntry entry : update.entries()) {
                 String schemaName = CatalogService.PUBLIC;
                 SchemaDescriptor schema = Objects.requireNonNull(catalog.schema(schemaName), "No schema found: " + schemaName);
 
                 if (entry instanceof NewTableEntry) {
                     catalog = new Catalog(
-                            update.version(),
+                            version,
                             System.currentTimeMillis(),
                             catalog.objectIdGenState(),
                             new SchemaDescriptor(
                                     schema.id(),
                                     schema.name(),
-                                    update.version(),
+                                    version,
                                     ArrayUtils.concat(schema.tables(), ((NewTableEntry) entry).descriptor()),
                                     schema.indexes()
                             )
                     );
+
+                    eventFutures.add(fireEvent(
+                            CatalogEvent.TABLE_CREATE,
+                            new CreateTableEventParameters(version, ((NewTableEntry) entry).descriptor())
+                    ));
+
                 } else if (entry instanceof ObjectIdGenUpdateEntry) {
                     catalog = new Catalog(
-                            update.version(),
+                            version,
                             System.currentTimeMillis(),
                             catalog.objectIdGenState() + ((ObjectIdGenUpdateEntry) entry).delta(),
                             new SchemaDescriptor(
                                     schema.id(),
                                     schema.name(),
-                                    update.version(),
+                                    version,
                                     schema.tables(),
                                     schema.indexes()
                             )
@@ -270,7 +284,7 @@ public class CatalogServiceImpl implements CatalogService, CatalogManager {
 
             registerCatalog(catalog);
 
-            versionTracker.update(catalog.version());
+            CompletableFuture.allOf(eventFutures.toArray(CompletableFuture[]::new)).thenRun(() -> versionTracker.update(version));
         }
     }
 
diff --git a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/events/CatalogEvent.java b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/events/CatalogEvent.java
new file mode 100644
index 0000000000..d88347aeca
--- /dev/null
+++ b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/events/CatalogEvent.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.catalog.events;
+
+import org.apache.ignite.internal.manager.Event;
+
+/**
+ * Catalog management events.
+ */
+public enum CatalogEvent implements Event {
+    /** This event is fired, when a table was created in Catalog. */
+    TABLE_CREATE,
+
+    /** This event is fired, when a table was dropped in Catalog. */
+    TABLE_DROP
+}
diff --git a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/events/CatalogEventParameters.java b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/events/CatalogEventParameters.java
new file mode 100644
index 0000000000..11c03c885e
--- /dev/null
+++ b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/events/CatalogEventParameters.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.catalog.events;
+
+import org.apache.ignite.internal.manager.EventParameters;
+
+/**
+ * Base class for Catalog event parameters.
+ */
+public abstract class CatalogEventParameters extends EventParameters {
+    /**
+     * Constructor.
+     *
+     * @param causalityToken Causality token.
+     */
+    public CatalogEventParameters(long causalityToken) {
+        super(causalityToken);
+    }
+}
diff --git a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/events/CreateTableEventParameters.java b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/events/CreateTableEventParameters.java
new file mode 100644
index 0000000000..345956f672
--- /dev/null
+++ b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/events/CreateTableEventParameters.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.catalog.events;
+
+import org.apache.ignite.internal.catalog.descriptors.TableDescriptor;
+
+/**
+ * Create table event parameters contains a table descriptor for newly created table.
+ */
+public class CreateTableEventParameters extends CatalogEventParameters {
+
+    private final TableDescriptor tableDescriptor;
+
+    /**
+     * Constructor.
+     *
+     * @param causalityToken Causality token.
+     * @param tableDescriptor Newly created table descriptor.
+     */
+    public CreateTableEventParameters(long causalityToken, TableDescriptor tableDescriptor) {
+        super(causalityToken);
+
+        this.tableDescriptor = tableDescriptor;
+    }
+
+    /**
+     * Gets table descriptor for newly created table.
+     */
+    public TableDescriptor tableDescriptor() {
+        return tableDescriptor;
+    }
+}
diff --git a/modules/catalog/src/test/java/org/apache/ignite/internal/catalog/CatalogServiceSelfTest.java b/modules/catalog/src/test/java/org/apache/ignite/internal/catalog/CatalogServiceSelfTest.java
index b532dc5e7a..e6dccd4fb9 100644
--- a/modules/catalog/src/test/java/org/apache/ignite/internal/catalog/CatalogServiceSelfTest.java
+++ b/modules/catalog/src/test/java/org/apache/ignite/internal/catalog/CatalogServiceSelfTest.java
@@ -31,6 +31,7 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
 import java.util.List;
@@ -40,11 +41,15 @@ import org.apache.ignite.internal.catalog.commands.CreateTableParams;
 import org.apache.ignite.internal.catalog.commands.DefaultValue;
 import org.apache.ignite.internal.catalog.descriptors.SchemaDescriptor;
 import org.apache.ignite.internal.catalog.descriptors.TableDescriptor;
+import org.apache.ignite.internal.catalog.events.CatalogEvent;
+import org.apache.ignite.internal.catalog.events.CatalogEventParameters;
+import org.apache.ignite.internal.catalog.events.CreateTableEventParameters;
 import org.apache.ignite.internal.catalog.storage.ObjectIdGenUpdateEntry;
 import org.apache.ignite.internal.catalog.storage.UpdateLog;
 import org.apache.ignite.internal.catalog.storage.UpdateLog.OnUpdateHandler;
 import org.apache.ignite.internal.catalog.storage.UpdateLogImpl;
 import org.apache.ignite.internal.catalog.storage.VersionedUpdate;
+import org.apache.ignite.internal.manager.EventListener;
 import org.apache.ignite.internal.metastorage.MetaStorageManager;
 import org.apache.ignite.internal.metastorage.impl.StandaloneMetaStorageManager;
 import org.apache.ignite.internal.metastorage.server.SimpleInMemoryKeyValueStorage;
@@ -58,6 +63,7 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.mockito.ArgumentCaptor;
+import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
 
 /**
@@ -247,6 +253,35 @@ public class CatalogServiceSelfTest {
         verify(updateLogMock).stop();
     }
 
+    @Test
+    public void testCreateTableEvents() {
+        CreateTableParams params = CreateTableParams.builder()
+                .schemaName("PUBLIC")
+                .tableName(TABLE_NAME)
+                .ifTableExists(true)
+                .zone("ZONE")
+                .columns(List.of(
+                        new ColumnParams("key1", ColumnType.INT32, DefaultValue.constant(null), false),
+                        new ColumnParams("key2", ColumnType.INT32, DefaultValue.constant(null), false),
+                        new ColumnParams("val", ColumnType.INT32, DefaultValue.constant(null), true)
+                ))
+                .primaryKeyColumns(List.of("key1", "key2"))
+                .colocationColumns(List.of("key2"))
+                .build();
+
+        EventListener<CatalogEventParameters> eventListener = Mockito.mock(EventListener.class);
+        when(eventListener.notify(any(), any())).thenReturn(completedFuture(false));
+
+        service.listen(CatalogEvent.TABLE_CREATE, eventListener);
+
+        CompletableFuture<Void> fut = service.createTable(params);
+
+        assertThat(fut, willBe((Object) null));
+
+        verify(eventListener).notify(any(CreateTableEventParameters.class), ArgumentMatchers.isNull());
+        verifyNoMoreInteractions(eventListener);
+    }
+
     private static CreateTableParams simpleTable(String name) {
         return CreateTableParams.builder()
                 .schemaName("PUBLIC")
diff --git a/modules/index/build.gradle b/modules/index/build.gradle
index 33c33510ba..cb1ae8e24f 100644
--- a/modules/index/build.gradle
+++ b/modules/index/build.gradle
@@ -21,6 +21,7 @@ apply from: "$rootDir/buildscripts/java-junit5.gradle"
 
 dependencies {
     implementation project(':ignite-api')
+    implementation project(':ignite-catalog')
     implementation project(':ignite-core')
     implementation project(':ignite-configuration-api')
     implementation project(':ignite-schema')
diff --git a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java
index 8fcff1a9ae..be4ca9c411 100644
--- a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java
+++ b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java
@@ -479,6 +479,10 @@ public class IgniteImpl implements Ignite {
 
         outgoingSnapshotsManager = new OutgoingSnapshotsManager(clusterSvc.messagingService());
 
+        catalogManager = new CatalogServiceImpl(
+                new UpdateLogImpl(metaStorageMgr, vaultMgr)
+        );
+
         distributedTblMgr = new TableManager(
                 name,
                 registry,
@@ -507,10 +511,6 @@ public class IgniteImpl implements Ignite {
 
         indexManager = new IndexManager(name, tablesConfiguration, schemaManager, distributedTblMgr, clusterSvc);
 
-        catalogManager = new CatalogServiceImpl(
-                new UpdateLogImpl(metaStorageMgr, vaultMgr)
-        );
-
         qryEngine = new SqlQueryProcessor(
                 registry,
                 clusterSvc,
diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ddl/DdlCommandHandlerWrapper.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ddl/DdlCommandHandlerWrapper.java
index 5ca2457242..4fd386f44d 100644
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ddl/DdlCommandHandlerWrapper.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ddl/DdlCommandHandlerWrapper.java
@@ -54,6 +54,7 @@ public class DdlCommandHandlerWrapper extends DdlCommandHandler {
     public CompletableFuture<Boolean> handle(DdlCommand cmd) {
         if (cmd instanceof CreateTableCommand) {
             return catalogManager.createTable(DdlToCatalogCommandConverter.convert((CreateTableCommand) cmd))
+                    .handle((res, err) -> Boolean.TRUE)
                     .thenCompose(res -> super.handle(cmd));
         }
 
diff --git a/modules/table/build.gradle b/modules/table/build.gradle
index 70611297f3..8aa95a0473 100644
--- a/modules/table/build.gradle
+++ b/modules/table/build.gradle
@@ -28,6 +28,7 @@ dependencies {
 
     implementation project(':ignite-api')
     implementation project(':ignite-schema')
+    implementation project(':ignite-catalog')
     implementation project(':ignite-core')
     implementation project(':ignite-metastorage-api')
     implementation project(':ignite-affinity')