You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by za...@apache.org on 2025/02/05 11:39:56 UTC

(flink) branch master updated (a4eb67642a8 -> ac4f5952d5d)

This is an automated email from the ASF dual-hosted git repository.

zakelly pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


    from a4eb67642a8 [FLINK-36929][table] Add SQL connector for keyed savepoint data
     new b88a6cac675 [FLINK-37222][Test] Introduce simple test submitting nexmark sql via sql client
     new ac4f5952d5d [FLINK-37222] Do not reuse views across TableEnvironments in SQL client

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../apache/flink/table/client/SqlClientTest.java   |   8 ++
 .../src/test/resources/nexmark.sql                 | 116 +++++++++++++++++++++
 .../EnvironmentReusableInMemoryCatalog.java        |  68 ++++++++++++
 .../gateway/service/context/SessionContext.java    |   3 +-
 .../table/catalog/QueryOperationCatalogView.java   |   5 +
 5 files changed, 198 insertions(+), 2 deletions(-)
 create mode 100644 flink-table/flink-sql-client/src/test/resources/nexmark.sql
 create mode 100644 flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/EnvironmentReusableInMemoryCatalog.java


(flink) 01/02: [FLINK-37222][Test] Introduce simple test submitting nexmark sql via sql client

Posted by za...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

zakelly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit b88a6cac675a6acf6e2e8a62d225f5db2057f709
Author: Zakelly <za...@gmail.com>
AuthorDate: Fri Jan 31 00:38:31 2025 +0800

    [FLINK-37222][Test] Introduce simple test submitting nexmark sql via sql client
---
 .../apache/flink/table/client/SqlClientTest.java   |   8 ++
 .../src/test/resources/nexmark.sql                 | 116 +++++++++++++++++++++
 2 files changed, 124 insertions(+)

diff --git a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/SqlClientTest.java b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/SqlClientTest.java
index d8f270a8dbe..653d1528620 100644
--- a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/SqlClientTest.java
+++ b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/SqlClientTest.java
@@ -214,6 +214,14 @@ class SqlClientTest extends SqlClientTestBase {
         }
     }
 
+    @Test
+    void testExecuteNexmark() throws Exception {
+        final URL sqlFile = getClass().getClassLoader().getResource("nexmark.sql");
+        String[] args = new String[] {"-f", sqlFile.getPath()};
+        String output = runSqlClient(args);
+        assertThat(output).doesNotContain("java.lang.AssertionError");
+    }
+
     @Test
     void testDisplayMultiLineSqlInInteractiveMode() throws Exception {
         List<String> statements =
diff --git a/flink-table/flink-sql-client/src/test/resources/nexmark.sql b/flink-table/flink-sql-client/src/test/resources/nexmark.sql
new file mode 100644
index 00000000000..5475f471dd9
--- /dev/null
+++ b/flink-table/flink-sql-client/src/test/resources/nexmark.sql
@@ -0,0 +1,116 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE TABLE datagen
+(
+    event_type   int,
+    person       ROW<
+        id  BIGINT,
+        name         VARCHAR,
+        emailAddress VARCHAR,
+        creditCard   VARCHAR,
+        city         VARCHAR,
+        state        VARCHAR,
+        `dateTime`   TIMESTAMP(3),
+        extra        VARCHAR>,
+    auction      ROW<
+        id  BIGINT,
+        itemName     VARCHAR,
+        description  VARCHAR,
+        initialBid   BIGINT,
+        reserve      BIGINT,
+        `dateTime`   TIMESTAMP(3),
+        expires      TIMESTAMP(3),
+        seller       BIGINT,
+        category     BIGINT,
+        extra        VARCHAR>,
+    bid          ROW<
+        auction  BIGINT,
+        bidder       BIGINT,
+        price        BIGINT,
+        channel      VARCHAR,
+        url          VARCHAR,
+        `dateTime`   TIMESTAMP(3),
+        extra        VARCHAR>,
+    `dateTime` AS
+        CASE
+            WHEN event_type = 0 THEN person.`dateTime`
+            WHEN event_type = 1 THEN auction.`dateTime`
+            ELSE bid.`dateTime`
+        END,
+    WATERMARK FOR `dateTime` AS `dateTime` - INTERVAL '4' SECOND
+) WITH (
+      'connector' = 'datagen',
+      'number-of-rows' = '10'
+);
+CREATE VIEW person AS
+SELECT person.id,
+       person.name,
+       person.emailAddress,
+       person.creditCard,
+       person.city,
+       person.state,
+       `dateTime`,
+       person.extra
+FROM datagen
+WHERE event_type = 0;
+
+CREATE VIEW auction AS
+SELECT auction.id,
+       auction.itemName,
+       auction.description,
+       auction.initialBid,
+       auction.reserve,
+       `dateTime`,
+       auction.expires,
+       auction.seller,
+       auction.category,
+       auction.extra
+FROM datagen
+WHERE event_type = 1;
+
+CREATE VIEW bid AS
+SELECT bid.auction,
+       bid.bidder,
+       bid.price,
+       bid.channel,
+       bid.url,
+       `dateTime`,
+       bid.extra
+FROM datagen
+WHERE event_type = 2;
+
+
+CREATE TABLE nexmark_q7
+(
+    auction    BIGINT,
+    bidder     BIGINT,
+    price      BIGINT,
+    `dateTime` TIMESTAMP(3),
+    extra      VARCHAR
+) WITH (
+      'connector' = 'blackhole'
+);
+
+INSERT INTO nexmark_q7
+SELECT B.auction, B.price, B.bidder, B.`dateTime`, B.extra
+from bid B
+         JOIN (SELECT MAX(price) AS maxprice, window_end as `dateTime`
+               FROM TABLE(
+                       TUMBLE(TABLE bid, DESCRIPTOR(`dateTime`), INTERVAL '10' SECOND))
+               GROUP BY window_start, window_end) B1
+              ON B.price = B1.maxprice
+WHERE B.`dateTime` BETWEEN B1.`dateTime` - INTERVAL '10' SECOND AND B1.`dateTime`;


(flink) 02/02: [FLINK-37222] Do not reuse views across TableEnvironments in SQL client

Posted by za...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

zakelly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit ac4f5952d5dd47e09644681b8f0fbf2b46a76509
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Mon Feb 3 11:35:14 2025 +0100

    [FLINK-37222] Do not reuse views across TableEnvironments in SQL client
---
 .../EnvironmentReusableInMemoryCatalog.java        | 68 ++++++++++++++++++++++
 .../gateway/service/context/SessionContext.java    |  3 +-
 .../table/catalog/QueryOperationCatalogView.java   |  5 ++
 3 files changed, 74 insertions(+), 2 deletions(-)

diff --git a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/EnvironmentReusableInMemoryCatalog.java b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/EnvironmentReusableInMemoryCatalog.java
new file mode 100644
index 00000000000..ae8efc1a3e7
--- /dev/null
+++ b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/EnvironmentReusableInMemoryCatalog.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.gateway.service.context;
+
+import org.apache.flink.table.api.TableEnvironment;
+import org.apache.flink.table.catalog.CatalogBaseTable;
+import org.apache.flink.table.catalog.CatalogView;
+import org.apache.flink.table.catalog.GenericInMemoryCatalog;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.QueryOperationCatalogView;
+import org.apache.flink.table.catalog.ResolvedCatalogView;
+import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException;
+import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
+
+import java.util.Optional;
+
+/**
+ * An in-memory catalog that can be reused across different {@link TableEnvironment}. The SQL client
+ * works against {@link TableEnvironment} design and reuses some of the components (e.g.
+ * CatalogManager), but not all (e.g. Planner) which causes e.g. views registered in an in-memory
+ * catalog to fail. This class is a workaround not to keep Planner bound parts of a view reused
+ * across different {@link TableEnvironment}.
+ */
+public class EnvironmentReusableInMemoryCatalog extends GenericInMemoryCatalog {
+    public EnvironmentReusableInMemoryCatalog(String name, String defaultDatabase) {
+        super(name, defaultDatabase);
+    }
+
+    @Override
+    public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ignoreIfExists)
+            throws TableAlreadyExistException, DatabaseNotExistException {
+        CatalogBaseTable tableToRegister =
+                extractView(table)
+                        .flatMap(QueryOperationCatalogView::getOriginalView)
+                        .map(v -> (CatalogBaseTable) v)
+                        .orElse(table);
+        super.createTable(tablePath, tableToRegister, ignoreIfExists);
+    }
+
+    private Optional<QueryOperationCatalogView> extractView(CatalogBaseTable table) {
+        if (table instanceof ResolvedCatalogView) {
+            final CatalogView origin = ((ResolvedCatalogView) table).getOrigin();
+            if (origin instanceof QueryOperationCatalogView) {
+                return Optional.of((QueryOperationCatalogView) origin);
+            }
+            return Optional.empty();
+        } else if (table instanceof QueryOperationCatalogView) {
+            return Optional.of((QueryOperationCatalogView) table);
+        }
+        return Optional.empty();
+    }
+}
diff --git a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/SessionContext.java b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/SessionContext.java
index 12bfc2d8e5a..c0dfad56f92 100644
--- a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/SessionContext.java
+++ b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/SessionContext.java
@@ -30,7 +30,6 @@ import org.apache.flink.table.catalog.Catalog;
 import org.apache.flink.table.catalog.CatalogManager;
 import org.apache.flink.table.catalog.CatalogStoreHolder;
 import org.apache.flink.table.catalog.FunctionCatalog;
-import org.apache.flink.table.catalog.GenericInMemoryCatalog;
 import org.apache.flink.table.factories.CatalogStoreFactory;
 import org.apache.flink.table.factories.FactoryUtil;
 import org.apache.flink.table.factories.TableFactoryUtil;
@@ -446,7 +445,7 @@ public class SessionContext {
                                                     catalogStore.config(),
                                                     catalogStore.classLoader()))
                             .orElse(
-                                    new GenericInMemoryCatalog(
+                                    new EnvironmentReusableInMemoryCatalog(
                                             defaultCatalogName, settings.getBuiltInDatabaseName()));
         }
         defaultCatalog.open();
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/QueryOperationCatalogView.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/QueryOperationCatalogView.java
index 02e31d5112b..fd2b0a850c3 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/QueryOperationCatalogView.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/QueryOperationCatalogView.java
@@ -116,4 +116,9 @@ public final class QueryOperationCatalogView implements CatalogView {
     public boolean supportsShowCreateView() {
         return originalView != null;
     }
+
+    @Internal
+    public Optional<CatalogView> getOriginalView() {
+        return Optional.ofNullable(originalView);
+    }
 }