You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@flink.apache.org by GitBox <gi...@apache.org> on 2022/06/23 08:27:09 UTC
[GitHub] [flink-table-store] JingsongLi opened a new pull request, #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
JingsongLi opened a new pull request, #173:
URL: https://github.com/apache/flink-table-store/pull/173
We currently have developed a Flink's Catalog.
If we expose this Catalog directly to other connector developers, it is not good and there will be many unsupported interfaces and capabilities.
So here we create a tablestore dedicated catalog.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [flink-table-store] LadyForest commented on a diff in pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
Posted by GitBox <gi...@apache.org>.
LadyForest commented on code in PR #173:
URL: https://github.com/apache/flink-table-store/pull/173#discussion_r905064602
##########
flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/FlinkCatalogTest.java:
##########
@@ -0,0 +1,341 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.connector;
+
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.Schema;
+import org.apache.flink.table.catalog.Catalog;
+import org.apache.flink.table.catalog.CatalogBaseTable;
+import org.apache.flink.table.catalog.CatalogTable;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
+import org.apache.flink.table.catalog.ResolvedSchema;
+import org.apache.flink.table.catalog.exceptions.CatalogException;
+import org.apache.flink.table.catalog.exceptions.DatabaseNotEmptyException;
+import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
+import org.apache.flink.table.catalog.exceptions.TableNotExistException;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Test for {@link FlinkCatalog}. */
+public class FlinkCatalogTest {
+
+ @Rule public ExpectedException exception = ExpectedException.none();
+
+ private final ObjectPath path1 = new ObjectPath("db1", "t1");
+ private final ObjectPath path3 = new ObjectPath("db1", "t2");
+ private final ObjectPath nonExistDbPath = ObjectPath.fromString("non.exist");
+ private final ObjectPath nonExistObjectPath = ObjectPath.fromString("db1.nonexist");
+ private Catalog catalog;
+
+ @ClassRule public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder();
+
+ @Before
+ public void beforeEach() throws IOException {
+ String path = TEMPORARY_FOLDER.newFolder().toURI().toString();
+ catalog = FlinkCatalogFactory.createCatalog(new Path(path), "test-catalog");
+ }
+
+ private ResolvedSchema createSchema() {
+ return new ResolvedSchema(
+ Arrays.asList(
+ Column.physical("first", DataTypes.STRING()),
+ Column.physical("second", DataTypes.INT()),
+ Column.physical("third", DataTypes.STRING())),
+ Collections.emptyList(),
+ null);
+ }
+
+ private CatalogTable createStreamingTable() {
+ ResolvedSchema resolvedSchema = this.createSchema();
+ CatalogTable origin =
+ CatalogTable.of(
+ Schema.newBuilder().fromResolvedSchema(resolvedSchema).build(),
+ "test comment",
+ Collections.emptyList(),
+ this.getStreamingTableProperties());
+ return new ResolvedCatalogTable(origin, resolvedSchema);
+ }
+
+ private List<String> createPartitionKeys() {
+ return Arrays.asList("second", "third");
+ }
+
+ private Map<String, String> getBatchTableProperties() {
+ return new HashMap<String, String>() {
+ {
+ this.put("is_streaming", "false");
+ }
+ };
+ }
+
+ private Map<String, String> getStreamingTableProperties() {
+ return new HashMap<String, String>() {
+ {
+ this.put("is_streaming", "true");
+ }
+ };
+ }
+
+ private CatalogTable createAnotherTable() {
+ // TODO support change schema, modify it to createAnotherSchema
+ ResolvedSchema resolvedSchema = this.createSchema();
+ CatalogTable origin =
+ CatalogTable.of(
+ Schema.newBuilder().fromResolvedSchema(resolvedSchema).build(),
+ "test comment",
+ Collections.emptyList(),
+ this.getBatchTableProperties());
+ return new ResolvedCatalogTable(origin, resolvedSchema);
+ }
+
+ private CatalogTable createAnotherPartitionedTable() {
+ // TODO support change schema, modify it to createAnotherSchema
+ ResolvedSchema resolvedSchema = this.createSchema();
+ CatalogTable origin =
+ CatalogTable.of(
+ Schema.newBuilder().fromResolvedSchema(resolvedSchema).build(),
+ "test comment",
+ this.createPartitionKeys(),
+ this.getBatchTableProperties());
+ return new ResolvedCatalogTable(origin, resolvedSchema);
+ }
+
+ private CatalogTable createTable() {
+ ResolvedSchema resolvedSchema = this.createSchema();
+ CatalogTable origin =
+ CatalogTable.of(
+ Schema.newBuilder().fromResolvedSchema(resolvedSchema).build(),
+ "test comment",
+ Collections.emptyList(),
+ this.getBatchTableProperties());
+ return new ResolvedCatalogTable(origin, resolvedSchema);
+ }
+
+ private CatalogTable createPartitionedTable() {
+ ResolvedSchema resolvedSchema = this.createSchema();
+ CatalogTable origin =
+ CatalogTable.of(
+ Schema.newBuilder().fromResolvedSchema(resolvedSchema).build(),
+ "test comment",
+ this.createPartitionKeys(),
+ this.getBatchTableProperties());
+ return new ResolvedCatalogTable(origin, resolvedSchema);
+ }
+
+ @Test
+ public void testAlterTable() throws Exception {
+ CatalogTable table = this.createTable();
+ catalog.createTable(this.path1, table, false);
+ checkEquals(path1, table, (CatalogTable) catalog.getTable(this.path1));
+ CatalogTable newTable = this.createAnotherTable();
+ catalog.alterTable(this.path1, newTable, false);
+ Assert.assertNotEquals(table, catalog.getTable(this.path1));
+ checkEquals(path1, newTable, (CatalogTable) catalog.getTable(this.path1));
+ catalog.dropTable(this.path1, false);
+
+ // Not support views
+ }
+
+ @Test
+ public void testListTables() throws Exception {
+ catalog.createTable(this.path1, this.createTable(), false);
+ catalog.createTable(this.path3, this.createTable(), false);
+ Assert.assertEquals(2L, catalog.listTables("db1").size());
+
+ // Not support views
+ }
+
+ @Test
+ public void testAlterTable_differentTypedTable() {
+ // TODO support this
+ }
+
+ @Test
+ public void testCreateFlinkTable() {
+ // create a flink table
+ CatalogTable table = createTable();
+ HashMap<String, String> newOptions = new HashMap<>(table.getOptions());
+ newOptions.put("connector", "filesystem");
+ CatalogTable newTable = table.copy(newOptions);
+
+ assertThatThrownBy(() -> catalog.createTable(this.path1, newTable, false))
+ .isInstanceOf(CatalogException.class)
+ .hasMessageContaining(
+ "Table Store Catalog only supports table store tables, not Flink connector: filesystem");
+ }
+
+ @Test
+ public void testCreateTable_Streaming() throws Exception {
+ CatalogTable table = createStreamingTable();
+ catalog.createTable(path1, table, false);
+ checkEquals(path1, table, (CatalogTable) catalog.getTable(path1));
+ }
+
+ @Test
+ public void testAlterPartitionedTable() throws Exception {
+ CatalogTable table = this.createPartitionedTable();
+ catalog.createTable(this.path1, table, false);
+ checkEquals(path1, table, (CatalogTable) catalog.getTable(this.path1));
+ CatalogTable newTable = this.createAnotherPartitionedTable();
+ catalog.alterTable(this.path1, newTable, false);
+ checkEquals(path1, newTable, (CatalogTable) catalog.getTable(this.path1));
+ }
+
+ @Test
+ public void testCreateTable_Batch() throws Exception {
+ CatalogTable table = this.createTable();
+ catalog.createTable(this.path1, table, false);
+ CatalogBaseTable tableCreated = catalog.getTable(this.path1);
+ checkEquals(path1, table, (CatalogTable) tableCreated);
+ Assert.assertEquals("test comment", tableCreated.getDescription().get());
+ List<String> tables = catalog.listTables("db1");
+ Assert.assertEquals(1L, tables.size());
+ Assert.assertEquals(this.path1.getObjectName(), tables.get(0));
+ catalog.dropTable(this.path1, false);
+ }
+
+ @Test
+ public void testCreateTable_TableAlreadyExist_ignored() throws Exception {
+ CatalogTable table = this.createTable();
+ catalog.createTable(this.path1, table, false);
+ checkEquals(path1, table, (CatalogTable) catalog.getTable(this.path1));
+ catalog.createTable(this.path1, this.createAnotherTable(), true);
+ checkEquals(path1, table, (CatalogTable) catalog.getTable(this.path1));
+ }
+
+ @Test
+ public void testCreatePartitionedTable_Batch() throws Exception {
+ CatalogTable table = this.createPartitionedTable();
+ catalog.createTable(this.path1, table, false);
+ checkEquals(path1, table, (CatalogTable) catalog.getTable(this.path1));
+ List<String> tables = catalog.listTables("db1");
+ Assert.assertEquals(1L, tables.size());
+ Assert.assertEquals(this.path1.getObjectName(), tables.get(0));
+ }
+
+ @Test
+ public void testDropDb_DatabaseNotEmptyException() throws Exception {
+ catalog.createTable(this.path1, this.createTable(), false);
+ this.exception.expect(DatabaseNotEmptyException.class);
+ this.exception.expectMessage("Database db1 in catalog test-catalog is not empty");
Review Comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [flink-table-store] LadyForest commented on a diff in pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
Posted by GitBox <gi...@apache.org>.
LadyForest commented on code in PR #173:
URL: https://github.com/apache/flink-table-store/pull/173#discussion_r905038905
##########
flink-table-store-core/src/main/java/org/apache/flink/table/store/file/catalog/Catalog.java:
##########
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.file.catalog;
+
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.store.file.schema.TableSchema;
+import org.apache.flink.table.store.file.schema.UpdateSchema;
+
+import java.util.List;
+
+/**
+ * This interface is responsible for reading and writing metadata such as database/table from a
+ * table store catalog.
+ */
+public interface Catalog extends AutoCloseable {
+
+ /**
+ * Get the names of all databases in this catalog.
+ *
+ * @return a list of the names of all databases
+ */
+ List<String> listDatabases();
+
+ /**
+ * Drop a database.
+ *
+ * @param name Name of the database to be dropped.
+ * @param cascade Flag to specify behavior when the database contains table or function: if set
+ * to true, delete all tables and functions in the database and then delete the database, if
+ * set to false, throw an exception.
+ * @throws DatabaseNotEmptyException if the given database is not empty and isRestrict is true
+ */
+ void dropDatabase(String name, boolean cascade) throws DatabaseNotEmptyException;
+
+ /**
+ * Get names of all tables under this database. An empty list is returned if none exists.
+ *
+ * @return a list of the names of all tables in this database
+ */
+ List<String> listTables(String databaseName);
+
+ /**
+ * Returns the table location path identified by the given {@link ObjectPath}.
Review Comment:
Nit: Return
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [flink-table-store] LadyForest commented on a diff in pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
Posted by GitBox <gi...@apache.org>.
LadyForest commented on code in PR #173:
URL: https://github.com/apache/flink-table-store/pull/173#discussion_r905040965
##########
flink-table-store-core/src/main/java/org/apache/flink/table/store/file/catalog/Catalog.java:
##########
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.file.catalog;
+
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.store.file.schema.TableSchema;
+import org.apache.flink.table.store.file.schema.UpdateSchema;
+
+import java.util.List;
+
+/**
+ * This interface is responsible for reading and writing metadata such as database/table from a
+ * table store catalog.
+ */
+public interface Catalog extends AutoCloseable {
+
+ /**
+ * Get the names of all databases in this catalog.
+ *
+ * @return a list of the names of all databases
+ */
+ List<String> listDatabases();
+
+ /**
+ * Drop a database.
+ *
+ * @param name Name of the database to be dropped.
+ * @param cascade Flag to specify behavior when the database contains table or function: if set
+ * to true, delete all tables and functions in the database and then delete the database, if
+ * set to false, throw an exception.
+ * @throws DatabaseNotEmptyException if the given database is not empty and isRestrict is true
Review Comment:
isRestrict?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [flink-table-store] LadyForest commented on a diff in pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
Posted by GitBox <gi...@apache.org>.
LadyForest commented on code in PR #173:
URL: https://github.com/apache/flink-table-store/pull/173#discussion_r905058925
##########
flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/FlinkCatalog.java:
##########
@@ -16,39 +16,192 @@
* limitations under the License.
*/
-package org.apache.flink.table.store.file.catalog;
+package org.apache.flink.table.store.connector;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.core.fs.Path;
import org.apache.flink.table.catalog.AbstractCatalog;
+import org.apache.flink.table.catalog.CatalogBaseTable;
import org.apache.flink.table.catalog.CatalogDatabase;
import org.apache.flink.table.catalog.CatalogFunction;
import org.apache.flink.table.catalog.CatalogPartition;
import org.apache.flink.table.catalog.CatalogPartitionSpec;
+import org.apache.flink.table.catalog.CatalogTable;
import org.apache.flink.table.catalog.ObjectPath;
import org.apache.flink.table.catalog.exceptions.CatalogException;
+import org.apache.flink.table.catalog.exceptions.DatabaseNotEmptyException;
+import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException;
import org.apache.flink.table.catalog.exceptions.FunctionNotExistException;
import org.apache.flink.table.catalog.exceptions.PartitionNotExistException;
+import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
+import org.apache.flink.table.catalog.exceptions.TableNotExistException;
import org.apache.flink.table.catalog.stats.CatalogColumnStatistics;
import org.apache.flink.table.catalog.stats.CatalogTableStatistics;
import org.apache.flink.table.expressions.Expression;
+import org.apache.flink.table.factories.DynamicTableFactory;
+import org.apache.flink.table.factories.Factory;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.table.store.file.catalog.Catalog;
+import org.apache.flink.table.store.file.schema.TableSchema;
+import org.apache.flink.table.store.file.schema.UpdateSchema;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.apache.flink.table.factories.FactoryUtil.CONNECTOR;
+import static org.apache.flink.table.store.connector.FlinkCatalogFactory.IDENTIFIER;
+import static org.apache.flink.table.store.file.FileStoreOptions.PATH;
/** Catalog for table store. */
-public abstract class TableStoreCatalog extends AbstractCatalog {
+public class FlinkCatalog extends AbstractCatalog {
+
+ private final Catalog catalog;
- public TableStoreCatalog(String name, String defaultDatabase) {
+ public FlinkCatalog(Catalog catalog, String name, String defaultDatabase) {
super(name, defaultDatabase);
+ this.catalog = catalog;
+ }
+
+ @VisibleForTesting
+ Catalog catalog() {
+ return catalog;
+ }
+
+ @Override
+ public Optional<Factory> getFactory() {
+ return Optional.of(
+ FactoryUtil.discoverFactory(classLoader(), DynamicTableFactory.class, IDENTIFIER));
+ }
+
+ @Override
+ public List<String> listDatabases() throws CatalogException {
+ return catalog.listDatabases();
+ }
+
+ @Override
+ public boolean databaseExists(String databaseName) throws CatalogException {
+ return true;
+ }
+
+ @Override
+ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade)
+ throws DatabaseNotEmptyException, CatalogException {
+ try {
+ catalog.dropDatabase(name, cascade);
+ } catch (Catalog.DatabaseNotEmptyException e) {
+ throw new DatabaseNotEmptyException(getName(), e.database());
+ }
+ }
+
+ @Override
+ public List<String> listTables(String databaseName) throws CatalogException {
+ return catalog.listTables(databaseName);
+ }
+
+ @Override
+ public CatalogBaseTable getTable(ObjectPath tablePath)
+ throws TableNotExistException, CatalogException {
+ TableSchema schema;
+ try {
+ schema = catalog.getTable(tablePath);
+ } catch (Catalog.TableNotExistException e) {
+ throw new TableNotExistException(getName(), e.tablePath());
+ }
+
+ CatalogTable table = schema.toUpdateSchema().toCatalogTable();
+ // add path to source and sink
+ table.getOptions().put(PATH.key(), catalog.getTableLocation(tablePath).toString());
+ return table;
+ }
+
+ @Override
+ public boolean tableExists(ObjectPath tablePath) throws CatalogException {
+ return catalog.tableExists(tablePath);
+ }
+
+ @Override
+ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists)
+ throws TableNotExistException, CatalogException {
+ try {
+ catalog.dropTable(tablePath, ignoreIfNotExists);
+ } catch (Catalog.TableNotExistException e) {
+ throw new TableNotExistException(getName(), e.tablePath());
+ }
+ }
+
+ @Override
+ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ignoreIfExists)
+ throws TableAlreadyExistException, DatabaseNotExistException, CatalogException {
+ try {
+ catalog.createTable(tablePath, convertTableToSchema(tablePath, table), ignoreIfExists);
+ } catch (Catalog.TableAlreadyExistException e) {
+ throw new TableAlreadyExistException(getName(), e.tablePath());
+ }
+ }
+
+ @Override
+ public void alterTable(
+ ObjectPath tablePath, CatalogBaseTable newTable, boolean ignoreIfNotExists)
+ throws TableNotExistException, CatalogException {
+ try {
+ catalog.alterTable(
+ tablePath, convertTableToSchema(tablePath, newTable), ignoreIfNotExists);
+ } catch (Catalog.TableNotExistException e) {
+ throw new TableNotExistException(getName(), e.tablePath());
+ }
+ }
+
+ private static ClassLoader classLoader() {
+ return FlinkCatalog.class.getClassLoader();
+ }
+
+ private UpdateSchema convertTableToSchema(ObjectPath tablePath, CatalogBaseTable baseTable) {
+ if (!(baseTable instanceof CatalogTable)) {
+ throw new UnsupportedOperationException(
+ "Only support CatalogTable, but is: " + baseTable.getClass());
+ }
+ CatalogTable table = (CatalogTable) baseTable;
+ Map<String, String> options = table.getOptions();
+ if (options.containsKey(CONNECTOR.key())) {
+ throw new CatalogException(
+ "Table Store Catalog only supports table store tables, not Flink connector: "
+ + options.get(CONNECTOR.key()));
+ }
+
+ // remove table path
+ String specific = options.remove(PATH.key());
+ if (specific != null) {
+ if (!catalog.getTableLocation(tablePath).equals(new Path(specific))) {
+ throw new IllegalArgumentException(
+ "Illegal table path in table options: " + specific);
+ }
+ table = table.copy(options);
+ }
+
+ return UpdateSchema.fromCatalogTable(table);
}
// --------------------- unsupported methods ----------------------------
+ @Override
+ public void createDatabase(String name, CatalogDatabase database, boolean ignoreIfExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
Review Comment:
just wondered why
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [flink-table-store] JingsongLi merged pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
Posted by GitBox <gi...@apache.org>.
JingsongLi merged PR #173:
URL: https://github.com/apache/flink-table-store/pull/173
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [flink-table-store] LadyForest commented on a diff in pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
Posted by GitBox <gi...@apache.org>.
LadyForest commented on code in PR #173:
URL: https://github.com/apache/flink-table-store/pull/173#discussion_r905044294
##########
flink-table-store-core/src/main/java/org/apache/flink/table/store/file/catalog/Catalog.java:
##########
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.file.catalog;
+
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.store.file.schema.TableSchema;
+import org.apache.flink.table.store.file.schema.UpdateSchema;
+
+import java.util.List;
+
+/**
+ * This interface is responsible for reading and writing metadata such as database/table from a
+ * table store catalog.
+ */
+public interface Catalog extends AutoCloseable {
+
+ /**
+ * Get the names of all databases in this catalog.
+ *
+ * @return a list of the names of all databases
+ */
+ List<String> listDatabases();
+
+ /**
+ * Drop a database.
+ *
+ * @param name Name of the database to be dropped.
+ * @param cascade Flag to specify behavior when the database contains table or function: if set
+ * to true, delete all tables and functions in the database and then delete the database, if
+ * set to false, throw an exception.
+ * @throws DatabaseNotEmptyException if the given database is not empty and isRestrict is true
Review Comment:
Missing a param like `ignoreIfNotExists`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [flink-table-store] LadyForest commented on a diff in pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
Posted by GitBox <gi...@apache.org>.
LadyForest commented on code in PR #173:
URL: https://github.com/apache/flink-table-store/pull/173#discussion_r905039540
##########
flink-table-store-core/src/main/java/org/apache/flink/table/store/file/catalog/Catalog.java:
##########
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.file.catalog;
+
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.store.file.schema.TableSchema;
+import org.apache.flink.table.store.file.schema.UpdateSchema;
+
+import java.util.List;
+
+/**
+ * This interface is responsible for reading and writing metadata such as database/table from a
+ * table store catalog.
+ */
+public interface Catalog extends AutoCloseable {
+
+ /**
+ * Get the names of all databases in this catalog.
+ *
+ * @return a list of the names of all databases
+ */
+ List<String> listDatabases();
+
+ /**
+ * Drop a database.
+ *
+ * @param name Name of the database to be dropped.
+ * @param cascade Flag to specify behavior when the database contains table or function: if set
+ * to true, delete all tables and functions in the database and then delete the database, if
+ * set to false, throw an exception.
+ * @throws DatabaseNotEmptyException if the given database is not empty and isRestrict is true
+ */
+ void dropDatabase(String name, boolean cascade) throws DatabaseNotEmptyException;
+
+ /**
+ * Get names of all tables under this database. An empty list is returned if none exists.
+ *
+ * @return a list of the names of all tables in this database
+ */
+ List<String> listTables(String databaseName);
+
+ /**
+ * Returns the table location path identified by the given {@link ObjectPath}.
+ *
+ * @param tablePath Path of the table
+ * @return The requested table location
+ */
+ Path getTableLocation(ObjectPath tablePath);
+
+ /**
+ * Returns a {@link TableSchema} identified by the given {@link ObjectPath}.
+ *
+ * @param tablePath Path of the table
+ * @return The requested table
+ * @throws TableNotExistException if the target does not exist
+ */
+ TableSchema getTable(ObjectPath tablePath) throws TableNotExistException;
+
+ /**
+ * Check if a table exists in this catalog.
+ *
+ * @param tablePath Path of the table
+ * @return true if the given table exists in the catalog false otherwise
+ */
+ boolean tableExists(ObjectPath tablePath);
+
+ /**
+ * Drop a table.
+ *
+ * @param tablePath Path of the table to be dropped
+ * @param ignoreIfNotExists Flag to specify behavior when the table does not exist: if set to
+ * false, throw an exception, if set to true, do nothing.
+ * @throws TableNotExistException if the table does not exist
+ */
+ void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) throws TableNotExistException;
+
+ /**
+ * Creates a new table.
+ *
+ * @param tablePath path of the table to be created
+ * @param tableSchema the table definition
+ * @param ignoreIfExists flag to specify behavior when a table already exists at the given path:
+ * if set to false, it throws a TableAlreadyExistException, if set to true, do nothing.
+ * @throws TableAlreadyExistException if table already exists and ignoreIfExists is false
+ */
+ void createTable(ObjectPath tablePath, UpdateSchema tableSchema, boolean ignoreIfExists)
+ throws TableAlreadyExistException;
+
+ /**
+ * Modifies an existing table.
Review Comment:
Ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [flink-table-store] LadyForest commented on a diff in pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
Posted by GitBox <gi...@apache.org>.
LadyForest commented on code in PR #173:
URL: https://github.com/apache/flink-table-store/pull/173#discussion_r905039136
##########
flink-table-store-core/src/main/java/org/apache/flink/table/store/file/catalog/Catalog.java:
##########
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.file.catalog;
+
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.store.file.schema.TableSchema;
+import org.apache.flink.table.store.file.schema.UpdateSchema;
+
+import java.util.List;
+
+/**
+ * This interface is responsible for reading and writing metadata such as database/table from a
+ * table store catalog.
+ */
+public interface Catalog extends AutoCloseable {
+
+ /**
+ * Get the names of all databases in this catalog.
+ *
+ * @return a list of the names of all databases
+ */
+ List<String> listDatabases();
+
+ /**
+ * Drop a database.
+ *
+ * @param name Name of the database to be dropped.
+ * @param cascade Flag to specify behavior when the database contains table or function: if set
+ * to true, delete all tables and functions in the database and then delete the database, if
+ * set to false, throw an exception.
+ * @throws DatabaseNotEmptyException if the given database is not empty and isRestrict is true
+ */
+ void dropDatabase(String name, boolean cascade) throws DatabaseNotEmptyException;
+
+ /**
+ * Get names of all tables under this database. An empty list is returned if none exists.
+ *
+ * @return a list of the names of all tables in this database
+ */
+ List<String> listTables(String databaseName);
+
+ /**
+ * Returns the table location path identified by the given {@link ObjectPath}.
+ *
+ * @param tablePath Path of the table
+ * @return The requested table location
+ */
+ Path getTableLocation(ObjectPath tablePath);
+
+ /**
+ * Returns a {@link TableSchema} identified by the given {@link ObjectPath}.
Review Comment:
Ditto
##########
flink-table-store-core/src/main/java/org/apache/flink/table/store/file/catalog/Catalog.java:
##########
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.file.catalog;
+
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.store.file.schema.TableSchema;
+import org.apache.flink.table.store.file.schema.UpdateSchema;
+
+import java.util.List;
+
+/**
+ * This interface is responsible for reading and writing metadata such as database/table from a
+ * table store catalog.
+ */
+public interface Catalog extends AutoCloseable {
+
+ /**
+ * Get the names of all databases in this catalog.
+ *
+ * @return a list of the names of all databases
+ */
+ List<String> listDatabases();
+
+ /**
+ * Drop a database.
+ *
+ * @param name Name of the database to be dropped.
+ * @param cascade Flag to specify behavior when the database contains table or function: if set
+ * to true, delete all tables and functions in the database and then delete the database, if
+ * set to false, throw an exception.
+ * @throws DatabaseNotEmptyException if the given database is not empty and isRestrict is true
+ */
+ void dropDatabase(String name, boolean cascade) throws DatabaseNotEmptyException;
+
+ /**
+ * Get names of all tables under this database. An empty list is returned if none exists.
+ *
+ * @return a list of the names of all tables in this database
+ */
+ List<String> listTables(String databaseName);
+
+ /**
+ * Returns the table location path identified by the given {@link ObjectPath}.
+ *
+ * @param tablePath Path of the table
+ * @return The requested table location
+ */
+ Path getTableLocation(ObjectPath tablePath);
+
+ /**
+ * Returns a {@link TableSchema} identified by the given {@link ObjectPath}.
+ *
+ * @param tablePath Path of the table
+ * @return The requested table
+ * @throws TableNotExistException if the target does not exist
+ */
+ TableSchema getTable(ObjectPath tablePath) throws TableNotExistException;
+
+ /**
+ * Check if a table exists in this catalog.
+ *
+ * @param tablePath Path of the table
+ * @return true if the given table exists in the catalog false otherwise
+ */
+ boolean tableExists(ObjectPath tablePath);
+
+ /**
+ * Drop a table.
+ *
+ * @param tablePath Path of the table to be dropped
+ * @param ignoreIfNotExists Flag to specify behavior when the table does not exist: if set to
+ * false, throw an exception, if set to true, do nothing.
+ * @throws TableNotExistException if the table does not exist
+ */
+ void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) throws TableNotExistException;
+
+ /**
+ * Creates a new table.
Review Comment:
Ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [flink-table-store] LadyForest commented on a diff in pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
Posted by GitBox <gi...@apache.org>.
LadyForest commented on code in PR #173:
URL: https://github.com/apache/flink-table-store/pull/173#discussion_r905064240
##########
flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/FlinkCatalogTest.java:
##########
@@ -0,0 +1,341 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.connector;
+
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.Schema;
+import org.apache.flink.table.catalog.Catalog;
+import org.apache.flink.table.catalog.CatalogBaseTable;
+import org.apache.flink.table.catalog.CatalogTable;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
+import org.apache.flink.table.catalog.ResolvedSchema;
+import org.apache.flink.table.catalog.exceptions.CatalogException;
+import org.apache.flink.table.catalog.exceptions.DatabaseNotEmptyException;
+import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
+import org.apache.flink.table.catalog.exceptions.TableNotExistException;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Test for {@link FlinkCatalog}. */
+public class FlinkCatalogTest {
+
+ @Rule public ExpectedException exception = ExpectedException.none();
+
+ private final ObjectPath path1 = new ObjectPath("db1", "t1");
+ private final ObjectPath path3 = new ObjectPath("db1", "t2");
+ private final ObjectPath nonExistDbPath = ObjectPath.fromString("non.exist");
+ private final ObjectPath nonExistObjectPath = ObjectPath.fromString("db1.nonexist");
+ private Catalog catalog;
+
+ @ClassRule public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder();
+
+ @Before
+ public void beforeEach() throws IOException {
+ String path = TEMPORARY_FOLDER.newFolder().toURI().toString();
+ catalog = FlinkCatalogFactory.createCatalog(new Path(path), "test-catalog");
+ }
+
+ private ResolvedSchema createSchema() {
+ return new ResolvedSchema(
+ Arrays.asList(
+ Column.physical("first", DataTypes.STRING()),
+ Column.physical("second", DataTypes.INT()),
+ Column.physical("third", DataTypes.STRING())),
+ Collections.emptyList(),
+ null);
+ }
+
+ private CatalogTable createStreamingTable() {
+ ResolvedSchema resolvedSchema = this.createSchema();
+ CatalogTable origin =
+ CatalogTable.of(
+ Schema.newBuilder().fromResolvedSchema(resolvedSchema).build(),
+ "test comment",
+ Collections.emptyList(),
+ this.getStreamingTableProperties());
+ return new ResolvedCatalogTable(origin, resolvedSchema);
+ }
+
+ private List<String> createPartitionKeys() {
+ return Arrays.asList("second", "third");
+ }
+
+ private Map<String, String> getBatchTableProperties() {
+ return new HashMap<String, String>() {
+ {
+ this.put("is_streaming", "false");
+ }
+ };
+ }
+
+ private Map<String, String> getStreamingTableProperties() {
+ return new HashMap<String, String>() {
+ {
+ this.put("is_streaming", "true");
+ }
+ };
+ }
+
+ private CatalogTable createAnotherTable() {
+ // TODO support change schema, modify it to createAnotherSchema
+ ResolvedSchema resolvedSchema = this.createSchema();
+ CatalogTable origin =
+ CatalogTable.of(
+ Schema.newBuilder().fromResolvedSchema(resolvedSchema).build(),
+ "test comment",
+ Collections.emptyList(),
+ this.getBatchTableProperties());
+ return new ResolvedCatalogTable(origin, resolvedSchema);
+ }
+
+ private CatalogTable createAnotherPartitionedTable() {
+ // TODO support change schema, modify it to createAnotherSchema
+ ResolvedSchema resolvedSchema = this.createSchema();
+ CatalogTable origin =
+ CatalogTable.of(
+ Schema.newBuilder().fromResolvedSchema(resolvedSchema).build(),
+ "test comment",
+ this.createPartitionKeys(),
+ this.getBatchTableProperties());
+ return new ResolvedCatalogTable(origin, resolvedSchema);
+ }
+
+ private CatalogTable createTable() {
+ ResolvedSchema resolvedSchema = this.createSchema();
+ CatalogTable origin =
+ CatalogTable.of(
+ Schema.newBuilder().fromResolvedSchema(resolvedSchema).build(),
+ "test comment",
+ Collections.emptyList(),
+ this.getBatchTableProperties());
+ return new ResolvedCatalogTable(origin, resolvedSchema);
+ }
+
+ private CatalogTable createPartitionedTable() {
+ ResolvedSchema resolvedSchema = this.createSchema();
+ CatalogTable origin =
+ CatalogTable.of(
+ Schema.newBuilder().fromResolvedSchema(resolvedSchema).build(),
+ "test comment",
+ this.createPartitionKeys(),
+ this.getBatchTableProperties());
+ return new ResolvedCatalogTable(origin, resolvedSchema);
+ }
+
+ @Test
+ public void testAlterTable() throws Exception {
+ CatalogTable table = this.createTable();
+ catalog.createTable(this.path1, table, false);
+ checkEquals(path1, table, (CatalogTable) catalog.getTable(this.path1));
+ CatalogTable newTable = this.createAnotherTable();
+ catalog.alterTable(this.path1, newTable, false);
+ Assert.assertNotEquals(table, catalog.getTable(this.path1));
+ checkEquals(path1, newTable, (CatalogTable) catalog.getTable(this.path1));
+ catalog.dropTable(this.path1, false);
+
+ // Not support views
+ }
+
+ @Test
+ public void testListTables() throws Exception {
+ catalog.createTable(this.path1, this.createTable(), false);
+ catalog.createTable(this.path3, this.createTable(), false);
+ Assert.assertEquals(2L, catalog.listTables("db1").size());
+
+ // Not support views
+ }
+
+ @Test
+ public void testAlterTable_differentTypedTable() {
+ // TODO support this
+ }
+
+ @Test
+ public void testCreateFlinkTable() {
+ // create a flink table
+ CatalogTable table = createTable();
+ HashMap<String, String> newOptions = new HashMap<>(table.getOptions());
+ newOptions.put("connector", "filesystem");
+ CatalogTable newTable = table.copy(newOptions);
+
+ assertThatThrownBy(() -> catalog.createTable(this.path1, newTable, false))
+ .isInstanceOf(CatalogException.class)
+ .hasMessageContaining(
+ "Table Store Catalog only supports table store tables, not Flink connector: filesystem");
+ }
+
+ @Test
+ public void testCreateTable_Streaming() throws Exception {
+ CatalogTable table = createStreamingTable();
+ catalog.createTable(path1, table, false);
+ checkEquals(path1, table, (CatalogTable) catalog.getTable(path1));
+ }
+
+ @Test
+ public void testAlterPartitionedTable() throws Exception {
+ CatalogTable table = this.createPartitionedTable();
+ catalog.createTable(this.path1, table, false);
+ checkEquals(path1, table, (CatalogTable) catalog.getTable(this.path1));
+ CatalogTable newTable = this.createAnotherPartitionedTable();
+ catalog.alterTable(this.path1, newTable, false);
+ checkEquals(path1, newTable, (CatalogTable) catalog.getTable(this.path1));
+ }
+
+ @Test
+ public void testCreateTable_Batch() throws Exception {
+ CatalogTable table = this.createTable();
+ catalog.createTable(this.path1, table, false);
+ CatalogBaseTable tableCreated = catalog.getTable(this.path1);
+ checkEquals(path1, table, (CatalogTable) tableCreated);
+ Assert.assertEquals("test comment", tableCreated.getDescription().get());
+ List<String> tables = catalog.listTables("db1");
+ Assert.assertEquals(1L, tables.size());
+ Assert.assertEquals(this.path1.getObjectName(), tables.get(0));
+ catalog.dropTable(this.path1, false);
+ }
+
+ @Test
+ public void testCreateTable_TableAlreadyExist_ignored() throws Exception {
+ CatalogTable table = this.createTable();
+ catalog.createTable(this.path1, table, false);
+ checkEquals(path1, table, (CatalogTable) catalog.getTable(this.path1));
+ catalog.createTable(this.path1, this.createAnotherTable(), true);
+ checkEquals(path1, table, (CatalogTable) catalog.getTable(this.path1));
+ }
+
+ @Test
+ public void testCreatePartitionedTable_Batch() throws Exception {
+ CatalogTable table = this.createPartitionedTable();
+ catalog.createTable(this.path1, table, false);
+ checkEquals(path1, table, (CatalogTable) catalog.getTable(this.path1));
+ List<String> tables = catalog.listTables("db1");
+ Assert.assertEquals(1L, tables.size());
+ Assert.assertEquals(this.path1.getObjectName(), tables.get(0));
+ }
+
+ @Test
+ public void testDropDb_DatabaseNotEmptyException() throws Exception {
+ catalog.createTable(this.path1, this.createTable(), false);
+ this.exception.expect(DatabaseNotEmptyException.class);
+ this.exception.expectMessage("Database db1 in catalog test-catalog is not empty");
Review Comment:
mixed-use of two assertions?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [flink-table-store] JingsongLi closed pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
Posted by GitBox <gi...@apache.org>.
JingsongLi closed pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
URL: https://github.com/apache/flink-table-store/pull/173
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [flink-table-store] JingsongLi commented on pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
Posted by GitBox <gi...@apache.org>.
JingsongLi commented on PR #173:
URL: https://github.com/apache/flink-table-store/pull/173#issuecomment-1165255264
> Q1: What are the criteria for choosing which interface not to implement. E.g. createDatabase is unsupported, which is confusing to me.
I was previously considering database creation all the way to lazy, but exposing it is more comprehensive.
So yes, I will change this.
> O2: It seems like FlinkCatalog serves as a wrapper to interact with Flink. Does it only accept the 3-rd party ecosystem's database as the inner impl?
See above comment, we should change back to support the previous functionality
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [flink-table-store] JingsongLi commented on a diff in pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
Posted by GitBox <gi...@apache.org>.
JingsongLi commented on code in PR #173:
URL: https://github.com/apache/flink-table-store/pull/173#discussion_r905728693
##########
flink-table-store-core/src/main/java/org/apache/flink/table/store/file/catalog/Catalog.java:
##########
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.file.catalog;
+
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.store.file.schema.TableSchema;
+import org.apache.flink.table.store.file.schema.UpdateSchema;
+
+import java.util.List;
+
+/**
+ * This interface is responsible for reading and writing metadata such as database/table from a
+ * table store catalog.
+ */
+public interface Catalog extends AutoCloseable {
+
+ /**
+ * Get the names of all databases in this catalog.
+ *
+ * @return a list of the names of all databases
+ */
+ List<String> listDatabases();
+
+ /**
+ * Drop a database.
+ *
+ * @param name Name of the database to be dropped.
+ * @param cascade Flag to specify behavior when the database contains table or function: if set
+ * to true, delete all tables and functions in the database and then delete the database, if
+ * set to false, throw an exception.
+ * @throws DatabaseNotEmptyException if the given database is not empty and isRestrict is true
Review Comment:
I was previously considering database creation all the way to lazy, but exposing it is more comprehensive.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [flink-table-store] JingsongLi commented on a diff in pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
Posted by GitBox <gi...@apache.org>.
JingsongLi commented on code in PR #173:
URL: https://github.com/apache/flink-table-store/pull/173#discussion_r905726586
##########
flink-table-store-core/src/main/java/org/apache/flink/table/store/file/catalog/Catalog.java:
##########
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.file.catalog;
+
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.store.file.schema.TableSchema;
+import org.apache.flink.table.store.file.schema.UpdateSchema;
+
+import java.util.List;
+
+/**
+ * This interface is responsible for reading and writing metadata such as database/table from a
+ * table store catalog.
+ */
+public interface Catalog extends AutoCloseable {
Review Comment:
We don't public any interfaces now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [flink-table-store] LadyForest commented on a diff in pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
Posted by GitBox <gi...@apache.org>.
LadyForest commented on code in PR #173:
URL: https://github.com/apache/flink-table-store/pull/173#discussion_r905059331
##########
flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/FlinkCatalog.java:
##########
@@ -16,39 +16,192 @@
* limitations under the License.
*/
-package org.apache.flink.table.store.file.catalog;
+package org.apache.flink.table.store.connector;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.core.fs.Path;
import org.apache.flink.table.catalog.AbstractCatalog;
+import org.apache.flink.table.catalog.CatalogBaseTable;
import org.apache.flink.table.catalog.CatalogDatabase;
import org.apache.flink.table.catalog.CatalogFunction;
import org.apache.flink.table.catalog.CatalogPartition;
import org.apache.flink.table.catalog.CatalogPartitionSpec;
+import org.apache.flink.table.catalog.CatalogTable;
import org.apache.flink.table.catalog.ObjectPath;
import org.apache.flink.table.catalog.exceptions.CatalogException;
+import org.apache.flink.table.catalog.exceptions.DatabaseNotEmptyException;
+import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException;
import org.apache.flink.table.catalog.exceptions.FunctionNotExistException;
import org.apache.flink.table.catalog.exceptions.PartitionNotExistException;
+import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
+import org.apache.flink.table.catalog.exceptions.TableNotExistException;
import org.apache.flink.table.catalog.stats.CatalogColumnStatistics;
import org.apache.flink.table.catalog.stats.CatalogTableStatistics;
import org.apache.flink.table.expressions.Expression;
+import org.apache.flink.table.factories.DynamicTableFactory;
+import org.apache.flink.table.factories.Factory;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.table.store.file.catalog.Catalog;
+import org.apache.flink.table.store.file.schema.TableSchema;
+import org.apache.flink.table.store.file.schema.UpdateSchema;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.apache.flink.table.factories.FactoryUtil.CONNECTOR;
+import static org.apache.flink.table.store.connector.FlinkCatalogFactory.IDENTIFIER;
+import static org.apache.flink.table.store.file.FileStoreOptions.PATH;
/** Catalog for table store. */
-public abstract class TableStoreCatalog extends AbstractCatalog {
+public class FlinkCatalog extends AbstractCatalog {
+
+ private final Catalog catalog;
- public TableStoreCatalog(String name, String defaultDatabase) {
+ public FlinkCatalog(Catalog catalog, String name, String defaultDatabase) {
super(name, defaultDatabase);
+ this.catalog = catalog;
+ }
+
+ @VisibleForTesting
+ Catalog catalog() {
+ return catalog;
+ }
+
+ @Override
+ public Optional<Factory> getFactory() {
+ return Optional.of(
+ FactoryUtil.discoverFactory(classLoader(), DynamicTableFactory.class, IDENTIFIER));
+ }
+
+ @Override
+ public List<String> listDatabases() throws CatalogException {
+ return catalog.listDatabases();
+ }
+
+ @Override
+ public boolean databaseExists(String databaseName) throws CatalogException {
+ return true;
Review Comment:
always `true`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [flink-table-store] LadyForest commented on a diff in pull request #173: [FLINK-28219] Refactor TableStoreCatalog: Introduce a dedicated Catalog for table store
Posted by GitBox <gi...@apache.org>.
LadyForest commented on code in PR #173:
URL: https://github.com/apache/flink-table-store/pull/173#discussion_r905037522
##########
flink-table-store-core/src/main/java/org/apache/flink/table/store/file/catalog/Catalog.java:
##########
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.file.catalog;
+
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.store.file.schema.TableSchema;
+import org.apache.flink.table.store.file.schema.UpdateSchema;
+
+import java.util.List;
+
+/**
+ * This interface is responsible for reading and writing metadata such as database/table from a
+ * table store catalog.
+ */
+public interface Catalog extends AutoCloseable {
Review Comment:
Nit: add `@PublicEvolving` annotation?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org